#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Oct 26 22:23:36 2021

@author: wangshiyuan
"""
import xarray as xr

import pandas as pd
import numpy as np
# import datetime as dt
from time import time
from glob import glob
import os
# import matplotlib.pyplot as plt

from multiprocessing import Pool

'''===================basic settings=================='''

N = 4
# means the value of each pp would be determined by 4 nearest pp's value.
EXPONENT = 2
LATBOUND = [-10, 60]
LONBOUND = [95, 180]

STARTDATE = '1960-01-01'
ENDDATE = '2018-01-01'

grid_dir = '/home/waterlab/Wang/bachelor_thesis/downloads/JRA/rawdownloads/uas/anl_mdl.033_ugrd.reg_tl319.196101_196112.wang528867'

rp_dir = '../downloads/GCMs/MRI/*.nc'

write_dir_base = '../interpolated_gcms_mon'

'''===================for testing=================='''

'''----0-----'''
'''values in each Map'''
# the default setting shoud be [None, None]
bnd = [None, None]
# bnd = [10, 10]
# these limit the program only to predict few points for each time step

'''-----1-----'''
STARTDATE = '1960-01-01'
ENDDATE = '2018-01-01'
# the default setting shoud be theirselves.
# the default should be like: STARTDATE = STARTDATE

'''-----2-----'''
timeMeasurement = True
# determine whether to measure the run time or a specific function or not

'''-----3-----'''
outputDataType = 'dataset'
# can only set as 'dataarray' or 'dataset'

cnt = 0

def measureRunTime(func):
    def timer(*args, **kwargs):
        start = time()
        ret = func(*args, **kwargs)
        end = time()
        print(
            f'func \'{func.__name__ }\' run time {end - start:04f} seconds, {(end-start)/60 :03f} min')
        return ret
    return timer if timeMeasurement is True else func

class Point():
    def __init__(self, lat=None, lon=None, source=None, variableType=None):
        self.lat = float(lat)
        self.lon = float(lon)
        self.dist = float('inf')
        self.value = None
        self.loc = np.array([lat, lon])
        # self.idx = []
        self.source = source
        self.type = variableType

    def getValue(self):
        # print(self.source, self.lat, self.lon)

        if self.source and self.lat and self.lon and self.type:

            value = float(self.source.sel(
                lat=self.lat, lon=self.lon, method='nearest')[self.type])
        else:
            raise ValueError(
                'need to know the lat, lon, source and variableType to get the value. Now the three are {bool(self.source)}, {bool(self.lat)}, {bool(self.lon)}, {bool(self.type)!')
        self.value = value

def newVal(weights, values):
    return (weights @ values)/sum(weights)

def droppingVariablesOfds(ds, distToBound=0):

    return ds.sel(lat=slice(LATBOUND[0]-distToBound, LATBOUND[1]+distToBound), lon=slice(LONBOUND[0]-distToBound, LONBOUND[1]+distToBound))

def find4Nearest(rp_ds, pp_loc, variableType="uas"):

    pass

def findNearestPoints(rp_ds, pp_loc, N=N, variableType="uas"):
    # given a rp map at one time step, and a exact location to predict, reuturn an array concluds 4 nearest Points
    # pp_loc is also a one-dim
    # means the value of each pp would be determined by 4 nearest ensional array and has two numbers (lat, lon)
    points, res = [], []
    lat, lon = pp_loc[0], pp_loc[1]
    # print(pp_loc)

    try:
        rp_ds = rp_ds.sel(lat=slice(lat-2, lat+2), lon=slice(lon-2, lon+2))
    except:
        with open(f'/home/waterlab/Wang/bachelor_thesis/interpolated_gcms_mon/MRI/{variableType}/error.txt', 'a') as f:
            info = [rp_ds, rp_ds.lat, rp_ds.lon, lat, lon, variableType]
            for s in info:
                f.write(str(s)+'\n')

        rp_ds = rp_ds

    # this is because we don't neet to check evey point in the grid to find 4 nearest points

    for lat in rp_ds.lat:
        for lon in rp_ds.lon:
            point = Point(lat, lon, source=rp_ds, variableType=variableType)

    # for i in lat_idxs:
    #     for j in lon_idxs:
    #         point = Point(float(rp_ds.lat[i]), float(rp_ds.lon[j]))
            point.dist = np.linalg.norm(point.loc - pp_loc)
    #         point.idx = [i, j]
    #         # print(point.idx)
            points.append(point)

            # print(f' for {pp_loc} appened the rp at {point.loc}')

    if len(points) == N:
        for point in points:
            point.getValue()
        res = points
    else:

        for point in sorted(points, key=lambda point: point.dist)[:N]:
            point.getValue()
            res.append(point)

        # points = sorted(points, key=lambda point: point.dist)

        # for i in range(N):
        #     points[i].getValue()
        #     res.append(points[i])
            # print(
            #     f'for pp at {pp_loc} found No.{i} nearest rp, at {res[i].loc} with distance {res[i].dist}')
    return res

@ measureRunTime
def interpolatedMap(grid_ds, rp_ds, time, bnd: list = bnd, exponent=EXPONENT, modelType=None, variableType="uas"):
    # given (a grid map at one time step) and (a rp map at one time step), pridict the value of each point on the grid

    # grid_ds = grid_ds.isel(time=0)
    xbnd, ybnd = bnd[0], bnd[1]
    data = np.zeros((len(grid_ds.lat), len(grid_ds.lon)))
    # print(data.shape)
    for i, lat in enumerate(grid_ds.lat):
        for j, lon in enumerate(grid_ds.lon):

            pp_loc = np.array((float(lat), float(lon)))

            nearestNPoints = findNearestPoints(
                rp_ds, pp_loc, variableType=variableType)

            if nearestNPoints[0].dist == 0.0:
                # to avoid dividing by zero
                data[i][j] = nearestNPoints[0].value
            else:
                values = [point.value for point in nearestNPoints]
                # print(values)
                dists = [point.dist for point in nearestNPoints]
                weights = 1 / np.power(dists, exponent)
                data[i][j] = newVal(weights, values)

            # print(
            #     f'the value at {pp_loc} has been predicted to {data[i][j]}, in [{i}][{j}]. And its 4 nearest is {values}')
            # print(f"[{i}][{j}] is over with value {data[i][j]}")
            # print(f"[{i}][{j}] is over")

            if ybnd and j >= ybnd:
                break

        if xbnd and i >= xbnd:
            break
        if i % 10 == 0 or i == 1:
            print(f"gcm {modelType} [{i} of {len(data)}] is over")
    # print(data.shape)
    assert data.size > 100, 'interpolate error'

    data = data[:, :, np.newaxis]
    if outputDataType == 'dataarray':

        new = xr.DataArray(data, coords=[
            grid_ds.lat, grid_ds.lon, time], dims=['lat', 'lon', 'time'])
    elif outputDataType == 'dataset':
        new = xr.Dataset(
            {
                variableType: (["lat", "lon", "time"], data),
            },
            coords={
                "lat": (["lat"], grid_ds.lat.to_index()),
                "lon": (["lon"], grid_ds.lon.to_index()),
                'time': (['time'], time)
            },
        )
    else:
        new = None
        raise ValueError(
            'need to set the outputDataType to either \'dataarray\' or \'dataset\'')

    print(f'interpolation for {time.strftime("%Y-%m-%d")[0]} is over')
    return new

@ measureRunTime
def interpolatedDataset(grid_ds, rp_ds, startTime=STARTDATE, endTime=ENDDATE, modelType=None, variableType="uas"):
    assert len(rp_ds.lat) > 5, 'lat error'
    timeIdx = 0
    rp_ds = rp_ds.sel(time=slice(startTime, endTime))

    # print(rp_ds)
    times = rp_ds['time'].to_index()

    concacted = interpolatedMap(grid_ds, rp_ds.sel(
        time=times[0]), times[:1], modelType=modelType, variableType=variableType)
    assert concacted[variableType].data.size > 100, 'interpolatedMap error'
    for timeIdx in range(1, len(times)):

        new = interpolatedMap(grid_ds, rp_ds.sel(
            time=times[timeIdx]), times[timeIdx:timeIdx+1], modelType=modelType, variableType=variableType)
        concacted = xr.concat([concacted, new], dim='time')
        print(
            f"interpolation and concaction for {startTime} to {times[timeIdx].strftime('%Y-%m-%d')} is over")
    return concacted

def preprocessed(ds, modelType='', startdate=None, enddate=None, variableType=None):
    def JRA_preprocess(ds):
        if "u" in ds:
            ds = ds.rename({"u": "uas"})
        if "v" in ds:
            ds = ds.rename({"v": "vas"})
        ds = ds.rename(
            {'latitude': 'lat', 'longitude': 'lon'})
        ds = ds.sortby('lat')
        ds = droppingVariablesOfds(ds)
        ds = ds.transpose('lat', 'lon', 'time')
        return ds

    def MRI_preprocess(ds):
        # print('here\n\n\n\n')
        # ds = ds.drop_vars(['time_bnds', 'lon_bnds', 'lat_bnds'])
        ds["time"] = ds.time.to_index().map(lambda t: t.replace(day=1, hour=0))

        ds = ds.transpose('lat', 'lon', 'time')

        return ds

    def MIROC6_preprpocess(ds):
        return MRI_preprocess(ds)

    def CNRM_ESM_preprocess(ds):
        # ds = ds.drop_vars(['time_bounds'])
        ds["time"] = ds.time.to_index().map(lambda t: t.replace(day=1, hour=0))

        ds = ds.transpose('lat', 'lon', 'time')
        return ds

    def unknown_preprocess(ds):

        raise ValueError(
            'The model cannot be preprocessed. Please do preprocessing by yourself')

        return ds

    model_list = {'JRA': JRA_preprocess,
                  'MRI': MRI_preprocess,
                  'MIROC6': MIROC6_preprpocess,
                  'CNRM-ESM2-1': CNRM_ESM_preprocess}

    def main_preprocess(ds, modelType=modelType, startdate=startdate, enddate=enddate, variableType=variableType):
        if startdate and enddate:
            ds = ds.sel(time=slice(startdate, enddate))
            # print('/n/n/n/n', 'cutdate',)

        if variableType:
            ds = ds[[variableType]]
        # print(ds)

        ds = model_list.get(modelType, unknown_preprocess)(ds)

        return ds
    print(f'{modelType} has been preprocessed')

    return main_preprocess(ds)

def main(args):
    # interpolate one month of data
    grid_ds, filepath, modelType, startdate, enddate, variableType = args[
        0], args[1], args[2], args[3], args[4], args[5]

    rp_ds = xr.open_dataset(filepath, engine='h5netcdf')

    rp_ds = preprocessed(rp_ds, modelType=modelType,
                         startdate=startdate, enddate=enddate, variableType=variableType)

    assert len(rp_ds.lat) > 5, 'preprocess'

    try:
        out = interpolatedDataset(grid_ds, rp_ds, modelType=modelType,
                                  startTime=startdate, endTime=enddate, variableType=variableType)
    except:
        return

    # print(out)

    def checked(out):
        flag = [True]

        if out[variableType].data.size < 100:
            flag.append(False)

        if np.isnan(out[variableType].data).any():
            flag.append(False)

        return flag

    if all(checked(out)):

        write_dir = write_dir_base + f"/{modelType}/{variableType}"
        if not os.path.exists(write_dir):
            os.makedirs(write_dir)

        out.to_netcdf(
            write_dir + f"/{modelType}_{startdate.year}_{variableType}.nc", engine="h5netcdf")

        global cnt
        cnt += 1

    else:
        with open(f"{write_dir}/message.txt", 'a') as f:
            f.write(f"{startdate.year} had error!\n", checked(out), '\n')

if __name__ == '__main__':
    for file in glob('../downloads/JRA/rawdownloads/**/*.idx', recursive=True):
        os.remove(file)

    grid_ds = xr.open_dataset(grid_dir, engine='cfgrib')
    grid_ds = preprocessed(grid_ds, modelType='JRA')
    grid_ds = grid_ds.isel(time=0)
    # print(grid_ds, '\n\n\n\n\n')
    arg_list = []

    for filepath in glob(rp_dir):
        ds = xr.open_dataset(filepath, engine="h5netcdf")

        modelType = os.path.basename(os.path.dirname(filepath))
        ds = preprocessed(ds, modelType=modelType,
                          startdate=STARTDATE, enddate=ENDDATE)
        if len(ds.time) == 0:
            continue

        timeIdx = ds.time.to_index()
        startdate_list = pd.date_range(timeIdx[0], timeIdx[-1], freq="YS")
        enddate_list = pd.date_range(timeIdx[0], timeIdx[-1], freq="Y")
        # print(startdate_list, enddate_list)
        for startdate, enddate in zip(startdate_list, enddate_list):
            for variableType in ds.data_vars:

                args = (grid_ds, filepath, modelType,
                        startdate, enddate, variableType)
                if not os.path.exists(write_dir_base + f"/{modelType}/{variableType}/{modelType}_{startdate.year}_{variableType}.nc"):

                    arg_list.append(args)

    # print(grid_ds)
    # for args in arg_list:
        # print(args[3:])
    #     print(args[2:], "\n\n")
    #     # print(args[3:5])
    print(len(arg_list))
    expectH = len(arg_list)*164.5*12/60/60
    print(f"the time would be about {expectH:.2f} hours")

    t1 = time()

    for args in arg_list:

        main(args)

        if (time() - t1) > 2*60*60:
            break

    # Pool().map(main, arg_list)
    t2 = time()

    print(
        f'run time is {t2-t1:.04f} sec, {(t2-t1)/60/60:.2f} hours, for {cnt} loops. while the predicted time is {expectH:.2f} hours')


import pytest
from pyscfad import gto

@pytest.fixture(scope="module")
def get_mol():
    mol = gto.Mole()
    mol.atom = 'H 0. 0. 0.; F 0. 0. 1.1'
    mol.basis = '631g'
    mol.verbose = 0
    mol.incore_anyway = True
    mol.build(trace_exp=False, trace_ctr_coeff=False)
    return mol


import torch
import gym
from torch import nn
from typing import Union, Dict, List, Any
import ray
from ray.rllib.models.torch.torch_modelv2 import TorchModelV2
from ray.rllib.models.modelv2 import ModelV2, restore_original_dimensions
from ray.rllib.utils.typing import ModelConfigDict, TensorType
from ray.rllib.models.torch.fcnet import FullyConnectedNetwork as TorchFC

from models.depth_vae import DepthVAE

# from models.rgbd_resnet18_vae import VAE

DEFAULTS = {"z_dim": 64, "depth_weight": 1, "rgb_weight": 1, "elbo_beta": 1}

class DepthRayVAE(TorchModelV2, DepthVAE):
    def __init__(
        self,
        obs_space: gym.spaces.Space,
        action_space: gym.spaces.Space,
        num_outputs: int,
        model_config: ModelConfigDict,
        name: str,
        **custom_model_kwargs,
    ):
        super().__init__(obs_space, action_space, num_outputs, model_config, name)
        self.cfg = dict(DEFAULTS, **custom_model_kwargs)
        # VAE.__init__(self, z_dim=self.cfg["z_dim"], nc=4)
        DepthVAE.__init__(self, z_dim=self.cfg["z_dim"])
        self.rgb_loss_fn = nn.MSELoss(reduction="mean")
        self.depth_loss_fn = nn.MSELoss(reduction="mean")
        self.act_space = gym.spaces.utils.flatdim(action_space)
        self.visdom_imgs: Dict[str, Any] = {}

    def forward(self, input_dict, state, seq_lens):
        """Compute autoencoded image. Note the returned "logits"
        are random, as we want random actions"""
        self.curr_ae_input = self.to_img(input_dict)
        # if torch.any(self.curr_ae_input > 0):
        #        ray.util.pdb.set_trace()
        self.curr_ae_output, self.curr_mu, self.curr_logvar = DepthVAE.forward(
            self, self.curr_ae_input
        )
        batch = input_dict["obs_flat"].shape[0]
        device = input_dict["obs_flat"].device
        self.device = device
        self._curr_value = torch.zeros((batch,), device=device)
        out = torch.zeros((batch, self.act_space), device=device)

        self.visdom_imgs.clear()
        di = self.curr_ae_input[:64].cpu().detach()
        do = self.curr_ae_output[:64].cpu().detach()
        self.visdom_imgs[f"depth_in-{self.cfg}"] = torch.cat(
            (di, di, di), dim=1
        ).numpy()
        self.visdom_imgs[f"depth_out-{self.cfg}"] = torch.cat(
            (do, do, do), dim=1
        ).numpy()
        # Return [batch, action_space]
        return out, state

    def value_function(self):
        return self._curr_value

    def to_img(self, input_dict):
        """Build obs into an image tensor for feeding to nn"""
        depth = input_dict["obs"]["depth"]
        # [batch, channel, cols, rows]
        # To B, dim, h, w
        depth = depth.permute(0, 3, 1, 2)
        """
        x = torch.zeros(
            (batch, channels, cols, rows), dtype=torch.float32, device=rgb.device
        )
        x[:, 0:semantic_tgt_channel] = semantic
        # x[:, semantic_channels:semantic_tgt_channel] = sem_tgt
        x[:, depth_channel] = torch.squeeze(depth)
        """
        return depth

    def custom_loss(
        self, policy_loss: List[torch.Tensor], loss_inputs
    ) -> List[torch.Tensor]:

        self.sem_loss = torch.tensor([0], device=self.curr_ae_output.device)

        self.depth_loss = self.cfg["depth_weight"] * self.depth_loss_fn(
            self.curr_ae_output[:, -1, :, :],
            self.curr_ae_input[:, -1, :, :],
        )
        self.recon_loss = self.sem_loss + self.depth_loss
        self.kld_loss = self.cfg["elbo_beta"] * DepthVAE.kld_loss(
            self, self.curr_mu, self.curr_logvar
        )
        self.combined_loss = self.recon_loss + self.kld_loss

        return [self.combined_loss]

    def metrics(self):
        return {
            "depth_loss": self.depth_loss.detach().item(),
            "semantic_loss": self.sem_loss.detach().item(),
            "combined_loss": self.combined_loss.detach().item(),
        }


# System imports
import datetime
import json
import os
import re
import shutil
import zipfile

from gettext import gettext as _
from pathlib import Path

# Included imports

from loguru import logger
from utility import cprint, cstrip

# Module imports
from .config import *
from .hardware import *
from .util import *

class PlatformBase():
    MOVE_PM_BASH = False

    def __init__(self, hm):
        self.hm = hm

    def first_run(self):
        """
        Called on first run, this can be used to add custom sources for your platform.
        """
        logger.debug(f"{self.__class__.__name__}: First Run")

    def port_install(self, port_name, port_info, port_files):
        """
        Called on after a port is installed, this can be used to check permissions, possibly augment the bash scripts.
        """
        logger.debug(f"{self.__class__.__name__}: Port Install {port_name}")

    def runtime_install(self, runtime_name, runtime_files):
        """
        Called on after a port is installed, this can be used to check permissions, possibly augment the bash scripts.
        """
        logger.debug(f"{self.__class__.__name__}: Runtime Install {runtime_name}")

    def port_uninstall(self, port_name, port_info, port_files):
        """
        Called on after a port is uninstalled, this can be used clean up special files.
        """
        logger.debug(f"{self.__class__.__name__}: Port Uninstall {port_name}")

    def portmaster_install(self):
        """
        Called on after portmaster is updated, this can be used clean up special files.
        """
        logger.debug(f"{self.__class__.__name__}: PortMaster Install")

    def set_gcd_mode(self, mode=None):
        logger.debug(f"{self.__class__.__name__}: Set GCD Mode {mode}")

    def get_gcd_modes(self):
        return tuple()

    def get_gcd_mode(self):
        logger.debug(f"{self.__class__.__name__}: Get GCD Mode")
        return None

class PlatformGCD_PortMaster:
    """
    gamecontrollerdb standard / xbox mode
    """
    def set_gcd_mode(self, gcd_mode=None):
        gamecontroller_file = self.hm.tools_dir / "PortMaster" / "gamecontrollerdb.txt"
        mode_files = {
            'standard': self.hm.tools_dir / "PortMaster" / ".Backup" / "donottouch.txt",
            'xbox': self.hm.tools_dir / "PortMaster" / ".Backup" / "donottouch_x.txt",
            }

        logger.debug(f"{self.__class__.__name__}: Set GCD Mode: {gcd_mode}")

        if gcd_mode:
            if gcd_mode not in mode_files:
                logger.debug(f"Unknown gcd_mode {gcd_mode}")
                return

            if not mode_files[gcd_mode].is_file():
                logger.debug(f"Unknown gcd_mode {gcd_mode} file.")
                return

            shutil.copy(mode_files[gcd_mode], gamecontroller_file)

            self.hm.cfg_data['gcd-mode'] = gcd_mode
            self.hm.save_config()

        else:
            logger.debug(f"Weird {gcd_mode}")

    def get_gcd_modes(self):
        return ('standard', 'xbox')

    def get_gcd_mode(self):
        gamecontroller_file = self.hm.tools_dir / "PortMaster" / "gamecontrollerdb.txt"

        gcd_mode = self.hm.cfg_data.get('gcd-mode', None)

        if gcd_mode not in self.get_gcd_modes():
            gcd_mode = None

        if gcd_mode is None:
            if gamecontroller_file.is_file():
                if "# Xbox 360 Layout" in gamecontroller_file.read_text():
                    gcd_mode = 'xbox'

            if gcd_mode is None:
                gcd_mode = 'standard'

            self.hm.cfg_data['gcd-mode'] = gcd_mode
            self.hm.save_config()

        logger.debug(f"{self.__class__.__name__}: Get GCD Mode: {gcd_mode}")
        return gcd_mode

class PlatformUOS(PlatformGCD_PortMaster, PlatformBase):
    ...

class PlatformJELOS(PlatformBase):
    def first_run(self):
        self.portmaster_install()

    def portmaster_install(self):
        """
        Copy JELOS PortMaster files here.
        """

        ## Copy the JELOS portmaster stuff into the right place.
        JELOS_PM_DIR = Path("/storage/.config/PortMaster")

        if not JELOS_PM_DIR.is_dir():
            shutil.copytree("/usr/config/PortMaster", JELOS_PM_DIR)

        ## This fixes it if you have the older mapper.txt
        BROKEN_MAPPER = JELOS_PM_DIR / "mapper.txt"
        BROKEN_MAPPER_MD5 = "edb6c56435798cd8a5bc15be71a3c124"
        PM_DIR = HM_TOOLS_DIR / "PortMaster"

        if hash_file(BROKEN_MAPPER) == BROKEN_MAPPER_MD5:
            logger.debug("Replacing broken mapper.txt if it is still there.")
            FIXED_MAPPER = PM_DIR / ".Backup" / "mapper.txt"
            shutil.copy(FIXED_MAPPER, BROKEN_MAPPER)

        ## Copy the files as per usual.
        shutil.copy(JELOS_PM_DIR / "control.txt", PM_DIR / "control.txt")
        shutil.copy(JELOS_PM_DIR / "gptokeyb", PM_DIR / "gptokeyb")
        shutil.copy(JELOS_PM_DIR / "gamecontrollerdb.txt", PM_DIR / "gamecontrollerdb.txt")
        shutil.copy(JELOS_PM_DIR / "mapper.txt", PM_DIR / "mapper.txt")

        for oga_control in JELOS_PM_DIR.glob("oga_controls*"):
            shutil.copy(oga_control, PM_DIR / oga_control.name)

class PlatformArkOS(PlatformGCD_PortMaster, PlatformBase):
    MOVE_PM_BASH = True

class PlatformAmberELEC(PlatformGCD_PortMaster, PlatformBase):
    ...

HM_PLATFORMS = {
    'jelos': PlatformJELOS,
    'arkos': PlatformArkOS,
    'amberelec': PlatformAmberELEC,
    'unofficialos': PlatformUOS,
    'default': PlatformBase,
    # 'default': PlatformAmberELEC,
    }

__all__ = (
    'PlatformBase',
    'HM_PLATFORMS',
    )


# coding=utf-8

import json
import os

import datasets
import gdown

logger = datasets.logging.get_logger(__name__)

_CITATION = """\
@article{park2019cord,
  title={CORD: A Consolidated Receipt Dataset for Post-OCR Parsing},
  author={Park, Seunghyun and Shin, Seung and Lee, Bado and Lee, Junyeop and Surh, Jaeheung and Seo, Minjoon and Lee, Hwalsuk}
  booktitle={Document Intelligence Workshop at Neural Information Processing Systems}
  year={2019}
}
"""

_DESCRIPTION = """\
https://github.com/clovaai/cord
"""
_URL = "https://drive.google.com/uc?id=1MqhTbcj-AHXOqYoeoh12aRUwIprzTJYI"

def gdrive_downloader(url, path):
    gdown.download(url, path, quiet=False)

class CordConfig(datasets.BuilderConfig):
    """BuilderConfig for CORD"""

    def __init__(self, **kwargs):
        """BuilderConfig for CORD.
        Args:
          **kwargs: keyword arguments forwarded to super.
        """
        super(CordConfig, self).__init__(**kwargs)

class Cord(datasets.GeneratorBasedBuilder):
    """Conll2003 dataset."""

    BUILDER_CONFIGS = [
        CordConfig(name="cord", version=datasets.Version(
            "1.0.0"), description="FUNSD dataset"),
    ]

    def _info(self):
        return datasets.DatasetInfo(
            description=_DESCRIPTION,
            features=datasets.Features(
                {
                    "id": datasets.Value("string"),
                    "tokens": datasets.Sequence(datasets.Value("string")),
                    "bboxes": datasets.Sequence(datasets.Sequence(datasets.Value("int64"))),
                    "roi": datasets.Sequence(datasets.Sequence(datasets.Value("int64"))),
                    "ner_tags": datasets.Sequence(
                        datasets.features.ClassLabel(
                            names=['menu.cnt',
                                   'menu.discountprice',
                                   'menu.etc',
                                   'menu.itemsubtotal',
                                   'menu.nm',
                                   'menu.num',
                                   'menu.price',
                                   'menu.sub_cnt',
                                   'menu.sub_etc',
                                   'menu.sub_nm',
                                   'menu.sub_price',
                                   'menu.sub_unitprice',
                                   'menu.unitprice',
                                   'menu.vatyn',
                                   'sub_total.discount_price',
                                   'sub_total.etc',
                                   'sub_total.othersvc_price',
                                   'sub_total.service_price',
                                   'sub_total.subtotal_price',
                                   'sub_total.tax_price',
                                   'total.cashprice',
                                   'total.changeprice',
                                   'total.creditcardprice',
                                   'total.emoneyprice',
                                   'total.menuqty_cnt',
                                   'total.menutype_cnt',
                                   'total.total_etc',
                                   'total.total_price',
                                   'void_menu.nm',
                                   'void_menu.price']
                        )
                    ),
                    "image_path": datasets.Value("string"),
                }
            ),
            supervised_keys=None,
            homepage="https://github.com/clovaai/cord",
            citation=_CITATION,
        )

    def _split_generators(self, dl_manager: datasets.DownloadManager):
        """Returns SplitGenerators."""
        url_or_urls = ['https://drive.google.com/uc?id=1MqhTbcj-AHXOqYoeoh12aRUwIprzTJYI',
                       'https://drive.google.com/uc?id=1wYdp5nC9LnHQZ2FcmOoC0eClyWvcuARU']

        downloaded_file = dl_manager.extract(
            dl_manager.download_custom(url_or_urls, gdrive_downloader))

        return [
            datasets.SplitGenerator(
                name=datasets.Split.TRAIN, gen_kwargs={
                    "filepaths": downloaded_file, "mode": "/CORD/train"}
            ),
            datasets.SplitGenerator(
                name=datasets.Split.TEST, gen_kwargs={
                    "filepaths": downloaded_file, "mode": "/CORD/test"}
            ),
            datasets.SplitGenerator(
                name=datasets.Split.VALIDATION, gen_kwargs={
                    "filepaths": downloaded_file, "mode": "/CORD/dev"}
            ),
        ]

    def _generate_examples(self, filepaths, mode):
        guid = -1
        for filepath in filepaths:
            filepath_folder  = filepath + mode
            logger.info("⏳ Generating examples from = %s", filepath_folder)
            ann_dir = os.path.join(filepath_folder, "json")
            if not os.path.exists(ann_dir):
                continue
            img_dir = os.path.join(filepath_folder, "image")
            for file in sorted(os.listdir(ann_dir)):
                guid +=1
                tokens = []
                bboxes = []
                ner_tags = []

                file_path = os.path.join(ann_dir, file)
                with open(file_path, "r", encoding="utf8") as f:
                    data = json.load(f)

                image_path = os.path.join(img_dir, file)
                image_path = image_path.replace("json", "png")

                if not os.path.exists(image_path):
                    other_dir_idx = int(not (filepaths.index(filepath)+2)%2)
                    image_path = image_path.replace(
                        filepath, filepaths[other_dir_idx])

                roi = data["roi"]
                if roi:
                    top_left = [roi["x1"], roi["y1"]]
                    bottom_right = [roi["x3"], roi["y3"]]
                    bottom_left = [roi["x4"], roi["y4"]]
                    top_right = [roi["x2"], roi["y2"]]
                    roi = [top_left, top_right, bottom_right, bottom_left]
                else:
                    roi = []

                for item in data["valid_line"]:
                    for word in item['words']:
                        # get word
                        txt = word['text']

                        # get bounding box
                        x1 = word['quad']['x1']
                        y1 = word['quad']['y1']
                        x3 = word['quad']['x3']
                        y3 = word['quad']['y3']

                        box = [x1, y1, x3, y3]

                        # ADDED
                        # skip empty word
                        if len(txt) < 1:
                            continue

                        tokens.append(txt)
                        bboxes.append(box)
                        ner_tags.append(item['category'])

                yield guid, {"id": str(guid), "tokens": tokens, "bboxes": bboxes, "ner_tags": ner_tags, "image_path": image_path, "roi":roi}


class Hero:
    def __init__(self, name):
        self.name = name

    def __del__(self):
        # Hero 对象被销毁之前会自动调用一下 __del__
        print(f'{self.name} 即将被回收')

if __name__ == '__main__':
    hero = Hero('李白')
    hero_back = hero
    input('输入任意内容手动删除 hero:')
    del hero  # del object -->  减少 object 的一次引用, 一旦引用次数为 0 __del__
    try:
        print(hero)
    except Exception as e:
        print('e', e)
    input('输入任意内容,结束程序')


import streamlit as st
import csv

# Sidebar Logo
logo_image = "logo.jpg"  # Ruta de la imagen del logotipo
st.sidebar.image(logo_image, use_column_width=True)

def enter_data(firstname, lastname, title, age, nationality, education, registration_status, numcourses, numsemesters):
    """
    Ingresa los datos del usuario.

    Args:
        firstname (str): Nombre del usuario.
        lastname (str): Apellidos del usuario.
        title (str): Título del usuario.
        age (int): Edad del usuario.
        nationality (str): Continente de origen del usuario.
        education (str): Nivel educativo del usuario.
        registration_status (str): Estado de registro del usuario.
        numcourses (int): Número de cursos completados por el usuario.
        numsemesters (int): Número de semestres completados por el usuario.
    """
    accepted = st.checkbox("Acepto los términos y condiciones")

    if accepted:
        if firstname and lastname:
            if st.button("Ingresar Datos"):
                # Guardar los datos en un archivo CSV
                data = [title, firstname, lastname, age, nationality, education, registration_status, numcourses, numsemesters]
                save_data(data)

                st.write("Datos guardados correctamente.")
                st.write("---------------------------------------------")
        else:
            st.warning("Nombre y Apellidos son requeridos.")
    else:
        st.warning("No ha aceptado los términos y condiciones")

def save_data(data):
    """
    Guarda los datos en un archivo CSV.

    Args:
        data (list): Lista con los datos a guardar.
    """
    with open('registros.csv', mode='a', newline='') as file:
        writer = csv.writer(file)
        writer.writerow(data)

# Sidebar Menu
st.sidebar.title("Menú")
selected_menu = st.sidebar.radio("Seleccionar opción", ["Información de usuario", "Solicitud de inscripción"])

# User Info Form
if selected_menu == "Información de usuario":
    st.subheader("Informacion de usuario")
    firstname = st.text_input("Nombre")
    lastname = st.text_input("Apellidos")
    title = st.selectbox("Titulo Académico", ["", "Sr.", "Srita.", "Sra."])
    age = st.number_input("Edad", min_value=18, max_value=110)
    nationality = st.selectbox("Continente", ["Africa", "Antartida", "Europa", "Norteamerica", "Sudamerica", "Oceania"])
    education = st.selectbox("Nivel educativo", ["Educación Superior", "Universitario", "Ciclo Formativo", "Máster Profesional"])
    comentario = st.text_input("Comentario Breve")

    # Validar la longitud del comentario
    if len(comentario) > 120:
        st.warning("El comentario no puede exceder los 120 caracteres.")

    # Course Info
    st.subheader("Informacion de cursos")
    registration_status = st.radio("Estado de registro", ["Registrado", "No registrado"], index=1)
    numcourses = st.number_input("# Cursos completados", min_value=0)
    numsemesters = st.number_input("# Semestres completados", min_value=0)

    # Button

    enter_data(firstname, lastname, title, age, nationality, education, registration_status, numcourses, numsemesters)

# Inscription Request Form
elif selected_menu == "Solicitud de inscripción":
    st.subheader("Solicitud de inscripción")
    # Agrega los campos y la lógica correspondiente para la solicitud de inscripción


# search.py
# ---------
# Licensing Information:  You are free to use or extend these projects for
# educational purposes provided that (1) you do not distribute or publish
# solutions, (2) you retain this notice, and (3) you provide clear
# attribution to UC Berkeley, including a link to http://ai.berkeley.edu.
#
# Attribution Information: The Pacman AI projects were developed at UC Berkeley.
# The core projects and autograders were primarily created by John DeNero
# (denero@cs.berkeley.edu) and Dan Klein (klein@cs.berkeley.edu).
# Student side autograding was added by Brad Miller, Nick Hay, and
# Pieter Abbeel (pabbeel@cs.berkeley.edu).

"""
In search.py, you will implement generic search algorithms which are called by
Pacman agents (in searchAgents.py).
"""

import pdb
import util

class SearchProblem:
    """
    This class outlines the structure of a search problem, but doesn't implement
    any of the methods (in object-oriented terminology: an abstract class).

    You do not need to change anything in this class, ever.
    """

    def getStartState(self):
        """
        Returns the start state for the search problem.
        """
        util.raiseNotDefined()

    def isGoalState(self, state):
        """
          state: Search state

        Returns True if and only if the state is a valid goal state.
        """
        util.raiseNotDefined()

    def getSuccessors(self, state):
        """
          state: Search state

        For a given state, this should return a list of triples, (successor,
        action, stepCost), where 'successor' is a successor to the current
        state, 'action' is the action required to get there, and 'stepCost' is
        the incremental cost of expanding to that successor.
        """
        util.raiseNotDefined()

    def getCostOfActions(self, actions):
        """
         actions: A list of actions to take

        This method returns the total cost of a particular sequence of actions.
        The sequence must be composed of legal moves.
        """
        util.raiseNotDefined()

def tinyMazeSearch(problem):
    """
    Returns a sequence of moves that solves tinyMaze.  For any other maze, the
    sequence of moves will be incorrect, so only use this for tinyMaze.
    """
    from game import Directions
    s = Directions.SOUTH
    w = Directions.WEST
    return  [s, s, w, s, w, w, s, w]

def depthFirstSearch(problem):
    """
    Search the deepest nodes in the search tree first.
    """
    stk = util.Stack()
    start_state = problem.getStartState()
    stk.push(start_state)
    visited, parent, goal_state = {}, {}, ""
    parent[start_state] = (-1, "")
    while stk:
        temp_state = stk.pop()
        if problem.isGoalState(temp_state):
            goal_state = temp_state
            break
        if visited.get(temp_state, False) == True: continue
        visited[temp_state] = True
        successors = problem.getSuccessors(temp_state)
        for state, action, reward in successors:
            if visited.get(state, False) == False:
                stk.push(state)
                parent[state] = (temp_state, action)
    actions = []
    while True:
        temp, action = parent.get(goal_state)
        if action == '': break
        actions.append(action)
        goal_state = temp
    return actions[::-1]

def breadthFirstSearch(problem):
    """Search the shallowest nodes in the search tree first."""
    q = util.Queue()
    closed = set()
    start = (problem.getStartState(), 0, [])
    q.push(start)
    while not q.isEmpty():
        (node, cost, path) = q.pop()
        if problem.isGoalState(node):
            return path
        if not node in closed:
            closed.add(node)
            for child_node, child_action, child_cost in problem.getSuccessors(node):
                new_cost = cost + child_cost
                new_path = path + [child_action]
                new_state = (child_node, new_cost, new_path)
                q.push(new_state)
    util.raiseNotDefined()

def uniformCostSearch(problem):
    """Search the node of least total cost first."""
    q = util.PriorityQueue()
    distances, INF = {}, int(10e18)
    start_state = problem.getStartState()
    distances[start_state] = 0
    q.push(start_state, distances[start_state])
    visited, parent, goal_state = {}, {}, ""
    parent[start_state] = (-1, "")
    while not q.isEmpty():
        temp_state = q.pop()
        if problem.isGoalState(temp_state):
            goal_state = temp_state
            break
        if visited.get(temp_state, False) == True: continue
        visited[temp_state] = True
        successors = problem.getSuccessors(temp_state)
        for state, action, reward in successors:
            if visited.get(state, False) == True: continue
            if distances.get(temp_state) + reward < distances.get(state, INF):
                distances[state] = distances[temp_state] + reward
                q.push(state, distances[state])
                parent[state] = (temp_state, action)
    actions = []
    while True:
        temp, action = parent.get(goal_state)
        if action == '': break
        actions.append(action)
        goal_state = temp
    return actions[::-1]
    util.raiseNotDefined()

def nullHeuristic(state, problem=None):
    """
    A heuristic function estimates the cost from the current state to the nearest
    goal in the provided SearchProblem.  This heuristic is trivial.
    """
    return 0

def aStarSearch(problem, heuristic=nullHeuristic):
    """Search the node that has the lowest combined cost and heuristic first."""
    "*** YOUR CODE HERE ***"
    q = util.PriorityQueue()
    distances, INF = {}, int(10e18)
    start_state = problem.getStartState()
    g_scores, f_scores = {}, {}
    g_scores[start_state], f_scores[start_state] = 0, heuristic(start_state, problem)
    q.push(start_state, f_scores[start_state])
    visited, parent, goal_state = {}, {}, ""
    parent[start_state] = (-1, "")
    while not q.isEmpty():
        temp_state = q.pop()
        if problem.isGoalState(temp_state):
            goal_state = temp_state
            break
        if visited.get(temp_state, False) == True: continue
        visited[temp_state] = True
        successors = problem.getSuccessors(temp_state)
        for state, action, reward in successors:
            if visited.get(state, False) == True: continue
            temp_g_score = g_scores[temp_state] + reward
            if temp_g_score < g_scores.get(state, INF):
                g_scores[state] = temp_g_score
                f_scores[state] = g_scores[state] + heuristic(state, problem)
                q.push(state, f_scores[state])
                parent[state] = (temp_state, action)
    actions = []
    while True:
        temp, action = parent.get(goal_state)
        if action == '': break
        actions.append(action)
        goal_state = temp
    return actions[::-1]
    util.raiseNotDefined()

# Abbreviations
bfs = breadthFirstSearch
dfs = depthFirstSearch
astar = aStarSearch
ucs = uniformCostSearch


class Solution:
    def find_max(self, matrix, k):
        row = len(matrix)
        col = len(matrix[0])
        res = - (1 << 31)
        for i in range(1, row+1):
            for j in range(1, col+1):
                dp = [[0 for _ in range(col+1)] for _ in range(row+1)]
                dp[i][j] = matrix[i-1][j-1]
                for m in range(i, row+1):
                    for n in range(j, col+1):
                        dp[m][n] = dp[m-1][n] + dp[m][n-1] - dp[m-1][n-1] + matrix[m-1][n-1]
                        if dp[m][n] <= k:
                            res = max(res, dp[m][n])
        return res

if __name__ == "__main__":
    a = int(input())
    inp1 = [int(x) for x in input().split(",")]
    inp2 = [int(x) for x in input().split(",")]
    K = int(input())
    arr = [inp1, inp2]
    s = Solution()
    ans = s.find_max(arr, K)
    print(ans)


description = """
                    [ Search ]

    This Program searches for a file in Directories.
Displays filename containing the word/Search_Phrase if exists.

    Usage:
        python search.py SEARCH_PHRASE --METHODS
"""

from os import walk, getcwd, listdir, system
from fnmatch import fnmatch
from time import sleep
from sys import argv, exit

def disp_avail_args():
    print("=" * 55)
    print("""
            AVAILABLE ARGUMENTS
            ====================

        -s / --simple : SIMPLE SEARCH
        -----------------------------
    [Searches file in Current Working Directory]

        -d / --deep   : DEEP SEARCH
        ----------------------------
    [Searches file in all files and folders]

        -c / --crawl  : DIRECTORY CRAWLER
        ---------------------------------
  [Searches file by crawling through all directories]
    (root = current working directory) ["Default"]
""")

    print("=" * 55)

def matched(word,file):
    # If word / search phrase contains wilcard characters..
    # use fnmatch function else traditional substring in string

    if "?" in word or "*" in word:
        return fnmatch(file,word.lower())
    return word in file

def crawl_search(word):
    # Crawls through directories
    # Displays filename along with path if found
    # returns number of files found

    count = 0 # for counting number of files found
    for root,dirs,files in walk(getcwd()):
        count += simple_search(word, path = root,  display = True)
    return count

def simple_search(word, path = getcwd(), display = None):
    # Searches in Current Working Directory
    # Without going into subdirectories
    # Displays filename if found
    # returns number of files found

    count = 0 # for counting number of files found

    for file in listdir(path):

        if matched(word.lower(),file.lower()):
            count += 1
            if display == None:
                print("\n{}".format(file))
            else:
                print ("\n[FOUND] {}".format(file))
                print("[PATH]  {}".format(path))
    return count

def deep_search(word):
    # Searches in Files while crawling
    # Yet to be implemented!

    print("\nDeep Search Coming Soon!\nTry a different method for now..\n")
    exit()

def found(word = "", method = 1):
    # Heart of the script

    # Links search phrase with methods..

    method_names = {
    1: "Directory Crawler",
    2: "Simple Search",
    3: "Deep Search"
    }

    print("="*30)
    print ("\nUsing Method {} [{}]".format(method,method_names[method]))
    print("="*30)

    methods = {
    1: crawl_search,
    2: simple_search,
    3: deep_search
    }

    count = methods[method](word)

    print("\n\t--> {} files found!\n".format(count))
    print("\n**Search Finished!**\n".upper())
    return count

method = 1 #default (Crawler)
word = "" # Search Phrase

for arg in argv:
    # Custom Argument Parser

    if arg[0] == "-":

        if arg in ["-c", "--crawl"]:
            # default
            method = 1

        elif arg in ["-s", "--simple"]:
            method = 2

        elif arg in ["-d", "--deep"]:
            method = 3

        else:
            print('\n\nInvalid Argument!')
            disp_avail_args()
            exit()

    elif arg in ["/?", "-h", "--help"]:
            system("cls||clear") # Clears the screen
            print(description)
            disp_avail_args()
            exit()

    elif arg != __file__ and len(argv) != 1:
        word = arg

system("cls||clear") # Clears the screen

try:
    while True:

        print('-' * 30)

        while not word:

            word = input("\n\nEnter File to be Searched or Press Ctrl + C to Exit\n\nSearch Phrase: ")

            if not word :
                print("\n[ERROR] Searched Phrase can't be empty!")
                sleep(2)
            else:
                break

        system("cls||clear") # Clears the screen
        print('='*30)
        print("\n  Search Phrase : \"{}\"".format(word))
        print("\n**Search Started!**".upper())

        c = found(word,method)

        if c < 0:
            for t in range(3,0,-1):
                print("Enter again in {} seconds...".format(t) , end = "\r")
                sleep(1)
                system("cls||clear")

        word = ""

except KeyboardInterrupt:
    print("\nSearch Terminated!")


import os
import json
import tempfile
import hashlib
import cPickle
import copy

import boto
import Image
import numpy as np

import bson.json_util as json_util
from bson.objectid import ObjectId

import tornado.web
import tornado.httpserver
import tornado.ioloop
import tornado.options
import tornado.web
import tornado.autoreload
from tornado.options import define, options

import genthor.datasets as gd

define("port", default=27719, help="run on the given port", type=int)

class App(tornado.web.Application):
    """
        Tornado app which serves the API.
    """
    def __init__(self):
        handlers = [(r"/renderimage", RenderImageHandler)]
        settings = dict(debug=True)
        tornado.web.Application.__init__(self, handlers, **settings)

class BaseHandler(tornado.web.RequestHandler):
    def get(self):
        args = self.request.arguments
        for k in args.keys():
            args[k] = args[k][0]
        args = dict([(str(x),y) for (x,y) in args.items()])
        args['preproc'] = json.loads(args['preproc'])
        args['spec'] = json.loads(args['spec'])

        callback = args.pop('callback', None)

        resp = jsonize(self.get_response(args))

        if callback:
            self.write(callback + '(')
        self.write(json.dumps(resp, default=json_util.default))
        if callback:
            self.write(')')

        self.finish()

class RenderImageHandler(BaseHandler):
    def get_response(self, args):
        return image_response(args)

DATASET = None
BUCKET = None
def image_response(args):
    sha1 = hashlib.sha1(json.dumps(args)).hexdigest()
    filename = sha1 + '.png'
    global BUCKET
    if BUCKET is None:
        conn = boto.connect_s3()
        BUCKET = conn.get_bucket('dicarlo-renderedimages')
    k = BUCKET.get_key(filename)
    global DATASET
    if not k:
        if DATASET is None:
            DATASET = gd.GenerativeBase() #pull out into global?
        imarray = DATASET.get_image(args['preproc'], args['spec'])[::-1]
        im = Image.fromarray(imarray, mode=args['preproc']['mode'])
        tmp = tempfile.TemporaryFile()
        im.save(tmp, "png")
        del imarray
        del im
        tmp.seek(0)
        k = BUCKET.new_key(filename)
        k.set_contents_from_string(tmp.read(), headers={'Content-Type' : 'image/png'})
        k.make_public()
        tmp.close()
    url = "http://dicarlo-renderedimages.s3.amazonaws.com/" + filename
    resp = {"url": url}
    return resp

def jsonize(x):
    try:
        json.dumps(x)
    except:
        return SONify(x)
    else:
        return x

def SONify(arg, memo=None):
    if memo is None:
        memo = {}
    if id(arg) in memo:
        rval = memo[id(arg)]
    if isinstance(arg, ObjectId):
        rval = arg
    elif isinstance(arg, np.floating):
        rval = float(arg)
    elif isinstance(arg, np.integer):
        rval = int(arg)
    elif isinstance(arg, (list, tuple)):
        rval = type(arg)([SONify(ai, memo) for ai in arg])
    elif isinstance(arg, dict):
        rval = dict([(SONify(k, memo), SONify(v, memo))
            for k, v in arg.items()])
    elif isinstance(arg, (basestring, float, int, type(None))):
        rval = arg
    elif isinstance(arg, np.ndarray):
        if arg.ndim == 0:
            rval = SONify(arg.sum())
        else:
            rval = map(SONify, arg) # N.B. memo None
    # -- put this after ndarray because ndarray not hashable
    elif arg in (True, False):
        rval = int(arg)
    else:
        raise TypeError('SONify', arg)
    memo[id(rval)] = rval
    return rval

def main():
    """
        function which starts up the tornado IO loop and the app.
    """
    tornado.options.parse_command_line()
    ioloop = tornado.ioloop.IOLoop.instance()
    http_server = tornado.httpserver.HTTPServer(App())
    http_server.listen(options.port)
    tornado.autoreload.start()
    ioloop.start()

if __name__ == "__main__":
    main()


# SSGame0601Ship01Packs-
#for bundles (49, 67, 80, 162, 173, 180, 184, 198, 199, 212):

file = open("E:/tmp\SSGame0601Ship01Packs-v1.txt", 'r')
# file = open("E:/tmp/Games/SPE 0414_ACTIVECARTONCONTENT.txt", 'r')

f = open("E:/tmp\SSGame0601Ship01Packs-v1_CLEAN.txt", 'w')
#f = open("E:/tmp/Games/SPE 0414_ACTIVECARTONCONTENT_CLEAN_FIX.txt", 'w')

gamecode = "0601"
newline = ""
splitedline = [""]

num = 0
for line in file:
    splitedline = line.split("\t", -1)
    firstitem = splitedline[0].strip()
    if firstitem in '2':
        num = int(splitedline[3])
        if num in (49, 67, 80, 162, 173, 180, 184, 198, 199, 212):
            newline += splitedline[1]

    if len(newline.strip()) > 0:
        f.write(newline + '\r\n')
        newline = ""

f.close()
file.close()


__all__ = [
    "BoltMessageReader",
    "BoltMessageWriter",
    "Bolt",
    "Bolt1",
    "Bolt2",
    "Bolt3",
    "Bolt4x0",
    "Bolt4x1",
    "BoltTransaction",
    "BoltResult",
    "BoltResponse",
    "BoltProtocolError",
]

from collections import deque
from io import BytesIO
from itertools import islice
from logging import getLogger
from struct import pack as struct_pack, unpack as struct_unpack

from six import PY2, raise_from

from py2neo.client import Connection, Transaction, Result, Failure, Bookmark, \
    TransactionError, BrokenTransactionError, ConnectionUnavailable
from py2neo.client.config import bolt_user_agent, ConnectionProfile
from py2neo.client.packstream import pack_into, UnpackStream, PackStreamHydrant
from py2neo.wiring import Wire, WireError, BrokenWireError

log = getLogger(__name__)

class BoltMessageReader(object):

    def __init__(self, wire):
        self.wire = wire
        if PY2:
            self.read_message = self.read_message_py2

    def read_message(self):
        chunks = []
        while True:
            hi, lo = self.wire.read(2)
            if hi == lo == 0:
                break
            size = hi << 8 | lo
            chunks.append(self.wire.read(size))
        message = b"".join(chunks)
        _, n = divmod(message[0], 0x10)
        unpacker = UnpackStream(message, offset=2)
        fields = [unpacker.unpack() for _ in range(n)]
        return message[1], fields

    def read_message_py2(self):
        chunks = []
        while True:
            hi, lo = self.wire.read(2)
            if hi == lo == 0:
                break
            size = hi << 8 | lo
            chunks.append(self.wire.read(size))
        message = bytearray(b"".join(map(bytes, chunks)))
        _, n = divmod(message[0], 0x10)
        unpacker = UnpackStream(message, offset=2)
        fields = [unpacker.unpack() for _ in range(n)]
        return message[1], fields

    def peek_message(self):
        """ If another complete message exists, return the tag for
        that message, otherwise return `None`.
        """
        data = self.wire.peek()
        p = 0

        def peek_chunk():
            q = p + 2
            if q < len(data):
                size, = struct_unpack(">H", data[p:q])
                r = q + size
                if r < len(data):
                    return size
            return -1

        while True:
            chunk_size = peek_chunk()
            if chunk_size == -1:
                return None
            elif chunk_size == 0:
                return data[3]
            else:
                p += 2 + chunk_size

class BoltMessageWriter(object):

    def __init__(self, wire, protocol_version):
        self.wire = wire
        self.protocol_version = protocol_version
        self.buffer = BytesIO()

    def _write_chunk(self, data):
        size = len(data)
        self.wire.write(struct_pack(">H", size))
        self.wire.write(data)
        return size

    def write_message(self, tag, *fields):
        buffer = self.buffer
        buffer.seek(0)
        buffer.write(bytearray([0xB0 + len(fields), tag]))
        pack_into(buffer, *fields, version=self.protocol_version)
        buffer.seek(0)
        while self._write_chunk(buffer.read(0x7FFF)):
            pass

    def send(self, final=False):
        return self.wire.send(final=final)

class Bolt(Connection):
    """ This is the base class for Bolt client connections. This class
    is not intended to be instantiated directly, but contains an
    :meth:`~Bolt.open` factory method that returns an instance of the
    appropriate subclass, once a connection has been successfully
    established.
    """

    protocol_version = ()

    __local_port = 0

    @classmethod
    def _walk_subclasses(cls):
        for subclass in cls.__subclasses__():
            assert issubclass(subclass, cls)  # for the benefit of the IDE
            yield subclass
            for k in subclass._walk_subclasses():
                yield k

    @classmethod
    def _get_subclass(cls, protocol_version):
        for subclass in cls._walk_subclasses():
            if subclass.protocol_version == protocol_version:
                return subclass
        raise BoltProtocolError("Unsupported protocol version %d.%d" % protocol_version)

    @classmethod
    def default_hydrant(cls, profile, graph):
        return PackStreamHydrant(graph)

    @classmethod
    def protocol_catalogue(cls):
        return [bolt.protocol_version for bolt in Bolt._walk_subclasses()]

    @classmethod
    def open(cls, profile=None, user_agent=None, on_bind=None, on_unbind=None,
             on_release=None, on_broken=None):
        """ Open a Bolt connection to a server.

        :param profile: :class:`.ConnectionProfile` detailing how and
            where to connect
        :param user_agent:
        :param on_bind:
        :param on_unbind:
        :param on_release:
        :param on_broken:
        :returns: :class:`.Bolt` connection object
        :raises: :class:`.ConnectionUnavailable` if a connection cannot
            be opened, or a protocol version cannot be agreed
        """
        if profile is None:
            profile = ConnectionProfile(scheme="bolt")
        try:
            wire = cls._connect(profile, on_broken=on_broken)
            protocol_version = cls._handshake(wire)
            subclass = cls._get_subclass(protocol_version)
            if subclass is None:
                raise BoltProtocolError("Unable to agree supported protocol version")
            bolt = subclass(wire, profile, (user_agent or bolt_user_agent()),
                            on_bind=on_bind, on_unbind=on_unbind, on_release=on_release)
            bolt.__local_port = wire.local_address.port_number
            bolt._hello()
            return bolt
        except (BoltProtocolError, WireError) as error:
            raise_from(ConnectionUnavailable("Cannot open connection to %r" % profile), error)

    @classmethod
    def _connect(cls, profile, on_broken):
        log.debug("[#%04X] C: (Dialing <%s>)", 0, profile.address)
        wire = Wire.open(profile.address, keep_alive=True, on_broken=on_broken)
        local_port = wire.local_address.port_number
        log.debug("[#%04X] S: (Accepted)", local_port)
        if profile.secure:
            log.debug("[#%04X] C: (Securing connection)", local_port)
            wire.secure(verify=profile.verify, hostname=profile.host)
        return wire

    @classmethod
    def _handshake(cls, wire):
        local_port = wire.local_address.port_number
        log.debug("[#%04X] C: <BOLT>", local_port)
        wire.write(b"\x60\x60\xB0\x17")
        versions = list(reversed(cls.protocol_catalogue()))[:4]
        log.debug("[#%04X] C: <PROTOCOL> %s",
                  local_port, " | ".join("%d.%d" % v for v in versions))
        wire.write(b"".join(bytes(bytearray([0, 0, minor, major]))
                            for major, minor in versions).ljust(16, b"\x00"))
        wire.send()
        v = bytearray(wire.read(4))
        if v == bytearray([0, 0, 0, 0]):
            raise BoltProtocolError("Unable to negotiate compatible "
                                    "protocol version from {!r}".format(versions))
        log.debug("[#%04X] S: <PROTOCOL> %d.%d", local_port, v[-1], v[-2])
        return v[-1], v[-2]

    def __init__(self, wire, profile, user_agent, on_bind=None, on_unbind=None, on_release=None):
        super(Bolt, self).__init__(profile, user_agent,
                                   on_bind=on_bind, on_unbind=on_unbind, on_release=on_release)
        self._wire = wire

    def close(self):
        """ Close the connection.
        """
        if self.closed or self.broken:
            return
        try:
            self._goodbye()
            self._wire.close()
        except BrokenWireError:
            return
        log.debug("[#%04X] C: (Hanging up)", self.local_port)

    @property
    def closed(self):
        return self._wire.closed

    @property
    def broken(self):
        return self._wire.broken

    @property
    def local_port(self):
        return self.__local_port

    @property
    def bytes_sent(self):
        return self._wire.bytes_sent

    @property
    def bytes_received(self):
        return self._wire.bytes_received

    def _assert_open(self):
        # TODO: better errors and hooks back into the pool, for
        #  deactivating and/or removing addresses from the routing table
        if self.closed:
            raise RuntimeError("Connection has been closed")
        if self.broken:
            raise RuntimeError("Connection is broken")

    def supports_multi(self):
        return self.protocol_version >= (4, 0)

class Bolt1(Bolt):

    protocol_version = (1, 0)

    def __init__(self, wire, profile, user_agent, on_bind=None, on_unbind=None, on_release=None):
        super(Bolt1, self).__init__(wire, profile, user_agent,
                                    on_bind=on_bind, on_unbind=on_unbind, on_release=on_release)
        self._reader = BoltMessageReader(wire)
        self._writer = BoltMessageWriter(wire, self.protocol_version)
        self._responses = deque()
        self._transaction = None
        self._metadata = {}

    @property
    def transaction(self):
        return self._transaction

    def bookmark(self):
        return self._metadata.get("bookmark")

    def _hello(self):
        self._assert_open()
        extra = {"scheme": "basic",
                 "principal": self.profile.user,
                 "credentials": self.profile.password}
        clean_extra = dict(extra)
        clean_extra.update({"credentials": "*******"})
        log.debug("[#%04X] C: INIT %r %r", self.local_port, self.user_agent, clean_extra)
        response = self._write_request(0x01, self.user_agent, extra, vital=True)
        self._send()
        self._fetch()
        self._audit(response)
        self.connection_id = response.metadata.get("connection_id")
        self.server_agent = response.metadata.get("server")
        if not self.server_agent.startswith("Neo4j/"):
            raise RuntimeError("Unexpected server agent {!r}".format(self.server_agent))

    def reset(self, force=False):
        self._assert_open()
        if force or self._transaction:
            log.debug("[#%04X] C: RESET", self.local_port)
            response = self._write_request(0x0F, vital=True)
            self._sync(response)
            self._audit(response)

    def _set_transaction(self, graph_name=None, readonly=False, after=None, metadata=None, timeout=None):
        self._assert_open()
        self._assert_no_transaction()
        if graph_name and not self.supports_multi():
            raise TypeError("This Neo4j installation does not support "
                            "named graphs")
        if metadata:
            raise TypeError("Transaction metadata not supported until Bolt v3")
        if timeout:
            raise TypeError("Transaction timeout not supported until Bolt v3")
        self._transaction = BoltTransaction(graph_name, self.protocol_version,
                                            readonly, after)

    def auto_run(self, graph_name, cypher, parameters=None, readonly=False,
                 # after=None, metadata=None, timeout=None
                 ):
        self._set_transaction(graph_name, readonly=readonly,
                              # after, metadata, timeout
                              )
        result = self._run(graph_name, cypher, parameters or {}, final=True)
        try:
            # TODO: can we avoid this buffering before the first
            #   PULL/DISCARD is sent?
            result.buffer()
        except BrokenWireError as error:
            raise_from(TransactionError("Transaction could not run "
                                        "due to disconnection"), error)
        else:
            try:
                self._audit(self._transaction)
            except Failure as failure:
                raise_from(TransactionError("Failed to run transaction"), failure)
            else:
                return result

    def begin(self, graph_name, readonly=False,
              # after=None, metadata=None, timeout=None
              ):
        self._set_transaction(graph_name, readonly=readonly,
                              # after, metadata, timeout
                              )
        log.debug("[#%04X] C: RUN 'BEGIN' %r", self.local_port, self._transaction.extra)
        log.debug("[#%04X] C: DISCARD_ALL", self.local_port)
        responses = (self._write_request(0x10, "BEGIN", self._transaction.extra),
                     self._write_request(0x2F))
        try:
            self._sync(*responses)
        except BrokenWireError as error:
            raise_from(TransactionError("Transaction could not begin "
                                        "due to disconnection"), error)
        else:
            try:
                self._audit(self._transaction)
            except Failure as failure:
                raise_from(TransactionError("Failed to begin transaction"), failure)
            else:
                if callable(self._on_bind):
                    self._on_bind(self._transaction, self)
                return self._transaction

    def commit(self, tx):
        self._assert_open()
        self._assert_transaction_open(tx)
        self._transaction.set_complete()
        log.debug("[#%04X] C: RUN 'COMMIT' {}", self.local_port)
        log.debug("[#%04X] C: DISCARD_ALL", self.local_port)
        try:
            self._sync(self._write_request(0x10, "COMMIT", {}),
                       self._write_request(0x2F))
        except BrokenWireError as error:
            tx.mark_broken()
            raise_from(BrokenTransactionError("Transaction broken by disconnection "
                                              "during commit"), error)
        else:
            try:
                self._audit(self._transaction)
            except Failure as failure:
                tx.mark_broken()
                raise_from(BrokenTransactionError("Failed to commit transaction"), failure)
            else:
                return Bookmark()
        finally:
            if callable(self._on_unbind):
                self._on_unbind(self._transaction)

    def rollback(self, tx):
        self._assert_open()
        self._assert_transaction_open(tx)
        self._transaction.set_complete()
        log.debug("[#%04X] C: RUN 'ROLLBACK' {}", self.local_port)
        log.debug("[#%04X] C: DISCARD_ALL", self.local_port)
        try:
            self._sync(self._write_request(0x10, "ROLLBACK", {}),
                       self._write_request(0x2F))
        except BrokenWireError as error:
            tx.mark_broken()
            raise_from(BrokenTransactionError("Transaction broken by disconnection "
                                              "during rollback"), error)
        else:
            try:
                self._audit(self._transaction)
            except Failure as failure:
                tx.mark_broken()
                raise_from(BrokenTransactionError("Failed to rollback transaction"), failure)
            else:
                return Bookmark()
        finally:
            if callable(self._on_unbind):
                self._on_unbind(self._transaction)

    def run_in_tx(self, tx, cypher, parameters=None):
        self._assert_open()
        self._assert_transaction_open(tx)
        return self._run(tx.graph_name, cypher, parameters or {})

    def _run(self, graph_name, cypher, parameters, extra=None, final=False):
        # TODO: limit logging for big parameter sets (e.g. bulk import)
        log.debug("[#%04X] C: RUN %r %r", self.local_port, cypher, parameters)
        response = self._write_request(0x10, cypher, parameters)
        result = BoltResult(graph_name, self, response)
        self._transaction.append(result, final=final)
        return result

    def pull(self, result, n=-1, capacity=-1):
        self._assert_open()
        self._assert_result_consumable(result)
        if n != -1:
            raise TypeError("Flow control is not supported before Bolt 4.0")
        log.debug("[#%04X] C: PULL_ALL", self.local_port)
        response = self._write_request(0x3F, capacity=capacity)
        result.append(response, final=True)
        return response

    def discard(self, result, n=-1):
        self._assert_open()
        self._assert_result_consumable(result)
        if n != -1:
            raise TypeError("Flow control is not supported before Bolt 4.0")
        log.debug("[#%04X] C: DISCARD_ALL", self.local_port)
        response = self._write_request(0x2F)
        result.append(response, final=True)
        return response

    def _get_routing_info(self, graph_name, query, parameters):
        # This import is not ideal as it breaks abstraction layers,
        # but it's necessary until the ROUTE message is available.
        from py2neo import ClientError
        try:
            result = self.auto_run(graph_name, query, parameters)
            self.pull(result)
            for ttl, address_data in result.records():
                addresses = {}
                for a in address_data:
                    addresses[a["role"]] = [ConnectionProfile(self.profile, address=address)
                                            for address in a["addresses"]]
                return (addresses.get("ROUTE", []),
                        addresses.get("READ", []),
                        addresses.get("WRITE", []),
                        ttl)
        except ClientError as error:
            if error.title == "ProcedureNotFound":
                raise_from(TypeError("Neo4j service does not support routing"), error)
            else:
                raise

    def route(self, graph_name=None, context=None):
        #
        # Bolt 1 (< Neo4j 3.2) (Clusters only)
        #     cx.run("CALL dbms.cluster.routing.getServers"
        #
        #
        # Bolt 1 (>= Neo4j 3.2) / Bolt 2 / Bolt 3 (Clusters only)
        #     cx.run("CALL dbms.cluster.routing.getRoutingTable($context)",
        #    {"context": self.routing_context}
        #
        if graph_name is not None:
            raise TypeError("Multiple graph databases are not available prior to Bolt v4")
        query = "CALL dbms.cluster.routing.getRoutingTable($context)"
        parameters = {"context": context or {}}
        return self._get_routing_info(None, query, parameters)

    def sync(self, result):
        self._send()
        self._wait(result.last())
        self._audit(result)

    def fetch(self, result):
        if not result.has_records() and not result.done():
            while self._responses[0] is not result.last():
                self._wait(self._responses[0])
            self._wait(result.last())
            self._audit(result)
        record = result.take_record()
        return record

    def _assert_no_transaction(self):
        if self._transaction:
            raise BoltProtocolError("Cannot open multiple simultaneous transactions "
                                    "on a Bolt connection")

    def _assert_transaction_open(self, tx):
        if tx is not self._transaction:
            raise ValueError("Transaction %r is not open on this connection", tx)
        if tx.broken:
            raise ValueError("Transaction is broken")

    def _assert_result_consumable(self, result):
        if not self._transaction:
            raise BoltProtocolError("Cannot consume result without open transaction")
        if result is not self._transaction.last():
            raise TypeError("Random query access is not supported before Bolt 4.0")

    def _write_request(self, tag, *fields, **kwargs):
        # capacity denotes the preferred max number of records that a response can hold
        # vital responses close on failure and cannot be ignored
        response = BoltResponse(**kwargs)
        self._writer.write_message(tag, *fields)
        self._responses.append(response)
        return response

    def _send(self, final=False):
        sent = self._writer.send(final=final)
        if sent:
            log.debug("[#%04X] C: (Sent %r bytes)", self.local_port, sent)

    def _fetch(self):
        """ Fetch and process the next incoming message.

        This method does not raise an exception on receipt of a
        FAILURE message. Instead, it sets the response (and
        consequently the parent query and transaction) to a failed
        state. It is the responsibility of the caller to convert this
        failed state into an exception.
        """
        rs = self._responses[0]
        tag, fields = self._reader.read_message()
        if tag == 0x70:
            log.debug("[#%04X] S: SUCCESS %s", self.local_port, " ".join(map(repr, fields)))
            rs.set_success(**fields[0])
            self._responses.popleft()
            self._metadata.update(fields[0])
        elif tag == 0x71:
            # If a RECORD is received, check for more records
            # in the buffer immediately following, and log and
            # add them all at the same time
            while self._reader.peek_message() == 0x71:
                _, extra_fields = self._reader.read_message()
                fields.extend(extra_fields)
            more = len(fields) - 1
            log.debug("[#%04X] S: RECORD %r%s", self.local_port, fields[0],
                      " (...and %d more)" % more if more else "")
            rs.add_records(fields)
        elif tag == 0x7F:
            log.debug("[#%04X] S: FAILURE %s", self.local_port, " ".join(map(repr, fields)))
            rs.set_failure(**fields[0])
            self._responses.popleft()
            if rs.vital:
                self._wire.close()
            else:
                self.reset(force=True)
        elif tag == 0x7E and not rs.vital:
            log.debug("[#%04X] S: IGNORED", self.local_port)
            rs.set_ignored()
            self._responses.popleft()
        else:
            log.debug("[#%04X] S: (Unexpected protocol message #%02X)", self.local_port, tag)
            self._wire.close()
            raise RuntimeError("Unexpected protocol message #%02X", tag)

    def _wait(self, response):
        """ Read all incoming responses up to and including a
        particular response.

        This method calls fetch, but does not raise an exception on
        FAILURE.
        """
        while not response.full() and not response.done():
            self._fetch()
            if not self._transaction:
                self.release()

    def _sync(self, *responses):
        self._send()
        for response in responses:
            self._wait(response)

    def _audit(self, task):
        """ Checks a specific task (response, result or transaction)
        for failure, raising an exception if one is found.

        :raise BoltFailure:
        """
        try:
            task.audit()
        except Failure:
            self.reset(force=True)
            raise

class Bolt2(Bolt1):

    protocol_version = (2, 0)

class Bolt3(Bolt2):

    protocol_version = (3, 0)

    def _hello(self):
        self._assert_open()
        extra = {"user_agent": self.user_agent,
                 "scheme": "basic",
                 "principal": self.profile.user,
                 "credentials": self.profile.password}
        clean_extra = dict(extra)
        clean_extra.update({"credentials": "*******"})
        log.debug("[#%04X] C: HELLO %r", self.local_port, clean_extra)
        response = self._write_request(0x01, extra, vital=True)
        self._send()
        self._fetch()
        self._audit(response)
        self.server_agent = response.metadata.get("server")
        self.connection_id = response.metadata.get("connection_id")

    def _goodbye(self):
        log.debug("[#%04X] C: GOODBYE", self.local_port)
        self._write_request(0x02)
        self._send(final=True)

    def auto_run(self, graph_name, cypher, parameters=None, readonly=False,
                 # after=None, metadata=None, timeout=None
                 ):
        self._assert_open()
        self._assert_no_transaction()
        self._transaction = BoltTransaction(graph_name, self.protocol_version, readonly,
                                            # after, metadata, timeout
                                            )
        result = self._run(graph_name, cypher, parameters or {},
                           self._transaction.extra, final=True)
        try:
            result.buffer()
        except BrokenWireError as error:
            raise_from(TransactionError("Transaction could not run "
                                        "due to disconnection"), error)
        else:
            try:
                self._audit(self._transaction)
            except Failure as failure:
                raise_from(TransactionError("Failed to run transaction"), failure)
            else:
                return result

    def begin(self, graph_name, readonly=False,
              # after=None, metadata=None, timeout=None
              ):
        self._assert_open()
        self._assert_no_transaction()
        self._transaction = BoltTransaction(graph_name, self.protocol_version, readonly,
                                            # after, metadata, timeout
                                            )
        log.debug("[#%04X] C: BEGIN %r", self.local_port, self._transaction.extra)
        response = self._write_request(0x11, self._transaction.extra)
        try:
            self._sync(response)
        except BrokenWireError as error:
            raise_from(TransactionError("Transaction could not begin "
                                        "due to disconnection"), error)
        else:
            try:
                self._audit(self._transaction)
            except Failure as failure:
                raise_from(TransactionError("Failed to begin transaction"), failure)
            else:
                if callable(self._on_bind):
                    self._on_bind(self._transaction, self)
                return self._transaction

    def commit(self, tx):
        self._assert_open()
        self._assert_transaction_open(tx)
        self._transaction.set_complete()
        log.debug("[#%04X] C: COMMIT", self.local_port)
        response = self._write_request(0x12)
        try:
            self._sync(response)
        except BrokenWireError as error:
            tx.mark_broken()
            raise_from(BrokenTransactionError("Transaction broken by disconnection "
                                              "during commit"), error)
        else:
            try:
                self._audit(self._transaction)
            except Failure as failure:
                tx.mark_broken()
                raise_from(BrokenTransactionError("Failed to commit transaction"), failure)
            else:
                return Bookmark(response.metadata.get("bookmark"))
        finally:
            if callable(self._on_unbind):
                self._on_unbind(self._transaction)

    def rollback(self, tx):
        self._assert_open()
        self._assert_transaction_open(tx)
        self._transaction.set_complete()
        log.debug("[#%04X] C: ROLLBACK", self.local_port)
        response = self._write_request(0x13)
        try:
            self._sync(response)
        except BrokenWireError as error:
            tx.mark_broken()
            raise_from(BrokenTransactionError("Transaction broken by disconnection "
                                              "during rollback"), error)
        else:
            try:
                self._audit(self._transaction)
            except Failure as failure:
                tx.mark_broken()
                raise_from(BrokenTransactionError("Failed to rollback transaction"), failure)
            else:
                return Bookmark(response.metadata.get("bookmark"))
        finally:
            if callable(self._on_unbind):
                self._on_unbind(self._transaction)

    def _run(self, graph_name, cypher, parameters, extra=None, final=False):
        log.debug("[#%04X] C: RUN %r %r %r", self.local_port, cypher, parameters, extra or {})
        response = self._write_request(0x10, cypher, parameters, extra or {})
        result = BoltResult(graph_name, self, response)
        self._transaction.append(result, final=final)
        return result

class Bolt4x0(Bolt3):

    protocol_version = (4, 0)

    def pull(self, result, n=-1, capacity=-1):
        self._assert_open()
        self._assert_result_consumable(result)  # TODO: random query access (qid)
        args = {"n": n}
        log.debug("[#%04X] C: PULL %r", self.local_port, args)
        response = self._write_request(0x3F, args, capacity=capacity)
        result.append(response, final=(n == -1))
        return response

    def discard(self, result, n=-1):
        self._assert_open()
        self._assert_result_consumable(result)  # TODO: random query access (qid)
        args = {"n": n}
        log.debug("[#%04X] C: DISCARD %r", self.local_port, args)
        response = self._write_request(0x2F, args)
        result.append(response, final=(n == -1))
        return response

    def route(self, graph_name=None, context=None):
        # In Neo4j 4.0 and above, routing is available for all
        # topologies, standalone or clustered.
        if graph_name is None:
            # Default database
            query = "CALL dbms.routing.getRoutingTable($context)"
            parameters = {"context": context or {}}
        else:
            # Named database
            query = "CALL dbms.routing.getRoutingTable($context, $database)"
            parameters = {"context": context or {}, "database": graph_name}
        return self._get_routing_info("system", query, parameters)

class Bolt4x1(Bolt4x0):

    protocol_version = (4, 1)

# class Bolt4x2(Bolt4x1):
#
#     protocol_version = (4, 2)

class Task(object):

    def done(self):
        raise NotImplementedError

    def failed(self):
        raise NotImplementedError

    def audit(self):
        raise NotImplementedError

class ItemizedTask(Task):
    """ This class represents a form of dynamic checklist. Items may
    be added, up to a "final" item which marks the list as complete.
    Each item may then be marked as done.
    """

    def __init__(self):
        self._items = deque()
        self._items_len = 0
        self._complete = False

    def __bool__(self):
        return not self.done() and not self.failed()

    __nonzero__ = __bool__

    def items(self):
        return iter(self._items)

    def append(self, item, final=False):
        self._items.append(item)
        self._items_len += 1
        if final:
            self.set_complete()

    def set_complete(self):
        self._complete = True

    def complete(self):
        """ Flag to indicate whether all items have been appended to
        this task, whether or not they are done.
        """
        return self._complete

    def first(self):
        try:
            return self._items[0]
        except IndexError:
            return None

    def last(self):
        try:
            return self._items[-1]
        except IndexError:
            return None

    def done(self):
        """ Flag to indicate whether the list of items is complete and
        all items are done.
        """
        if self._complete:
            try:
                last = self._items[-1]
            except IndexError:
                return True
            else:
                return last.done()
        else:
            return False

    def failed(self):
        return any(item.failed() for item in self._items)

    def audit(self):
        for item in self._items:
            item.audit()

class BoltTransaction(ItemizedTask, Transaction):

    def __init__(self, graph_name, protocol_version,
                 readonly=False, after=None, metadata=None, timeout=None
                 ):
        if graph_name and protocol_version < (4, 0):
            raise TypeError("Database selection is not supported "
                            "prior to Neo4j 4.0")
        ItemizedTask.__init__(self)
        Transaction.__init__(self, graph_name, readonly=readonly)
        self.after = after
        self.metadata = metadata
        self.timeout = timeout

    @property
    def extra(self):
        extra = {}
        if self.graph_name:
            extra["db"] = self.graph_name
        if self.readonly:
            extra["mode"] = "r"
        if self.after:
            extra["bookmarks"] = list(Bookmark(self.after))
        if self.metadata:
            extra["metadata"] = self.metadata
        if self.timeout:
            extra["timeout"] = self.timeout
        return extra

# TODO: use 'has_more' metadata from PULL success response
class BoltResult(ItemizedTask, Result):
    """ The result of a query carried out over a Bolt connection.

    Implementation-wise, this form of query is comprised of a number of
    individual message exchanges. Each of these exchanges may succeed
    or fail in its own right, but contribute to the overall success or
    failure of the query.
    """

    def __init__(self, graph_name, cx, response):
        ItemizedTask.__init__(self)
        Result.__init__(self, graph_name)
        self.__record_type = None
        self.__cx = cx
        self.append(response)
        self._last_taken = 0

    @property
    def graph_name(self):
        return self._items[-1].metadata.get("db", super(BoltResult, self).graph_name)

    @property
    def query_id(self):
        return self.header().metadata.get("qid")

    @property
    def protocol_version(self):
        return self.__cx.protocol_version

    def buffer(self):
        if self.done():
            return
        try:
            self.__cx.sync(self)
        except BrokenWireError as error:
            self.__cx.transaction.mark_broken()
            raise_from(BrokenTransactionError("Transaction broken by disconnection "
                                              "while buffering"), error)

    def header(self):
        if not self.done():
            self.buffer()
        return self._items[0]

    def fields(self):
        return self.header().metadata.get("fields")

    def summary(self):
        return dict(self._items[-1].metadata,
                    connection=dict(self.__cx.profile))

    def fetch(self):
        return self.__cx.fetch(self)

    def has_records(self):
        i = self._last_taken
        while i < self._items_len:
            response = self._items[i]
            if response.records:
                return True
            i += 1
        return False

    def take_record(self):
        i = self._last_taken
        while i < self._items_len:
            response = self._items[i]
            try:
                record = response.records.popleft()
            except IndexError:
                i += 1
            else:
                self._last_taken = i
                return record
        return None

    def peek_records(self, limit):
        records = []
        i = self._last_taken
        while i < self._items_len:
            response = self._items[i]
            records.extend(response.peek_records(limit - len(records)))
            if len(records) == limit:
                break
            i += 1
        return records

class BoltResponse(Task):

    # status values:
    #   0 = not done
    #   1 = success
    #   2 = failure
    #   3 = ignored

    def __init__(self, capacity=-1, vital=False):
        super(BoltResponse, self).__init__()
        self.capacity = capacity
        self.vital = vital
        self.records = deque()
        self._status = 0
        self._metadata = {}
        self._failure = None

    def __repr__(self):
        if self._status == 1:
            return "<BoltResponse SUCCESS %r>" % self._metadata
        elif self._status == 2:
            return "<BoltResponse FAILURE %r>" % self._metadata
        elif self._status == 3:
            return "<BoltResponse IGNORED>"
        else:
            return "<BoltResponse ?>"

    def add_records(self, value_lists):
        self.records.extend(value_lists)

    def peek_records(self, n):
        return islice(self.records, 0, n)

    def set_success(self, **metadata):
        self._status = 1
        self._metadata.update(metadata)

    def set_failure(self, **metadata):
        self._status = 2
        # FIXME: This currently clumsily breaks through abstraction
        #   layers. This should create a Failure instead of a
        #   Neo4jError. See also HTTPResponse.audit
        from py2neo.database.work import Neo4jError
        self._failure = Neo4jError.hydrate(metadata)

    def set_ignored(self):
        self._status = 3

    def full(self):
        if self.capacity >= 0:
            return len(self.records) >= self.capacity
        else:
            return False

    def done(self):
        return self._status != 0

    def failed(self):
        return self._status >= 2

    def audit(self):
        if self._failure:
            self.set_ignored()
            raise self._failure

    @property
    def metadata(self):
        return self._metadata

class BoltProtocolError(Exception):

    pass


from __future__ import annotations
import sys
from pathlib import Path
from airflow import DAG
from airflow.operators.bash import BashOperator
from airflow.operators.python import ShortCircuitOperator
from airflow.decorators import task
from datetime import datetime, timedelta
import os

with DAG(dag_id="tune_dag",
         default_args={
             'owner': 'DuongPD7',
             'email': 'phandaiduonghcb@gmail.com',
             'email_on_retry': False,
             'email_on_failure': True,
             'retries': 1,
             'retry_delay': 5,
             'retry_exponential_backoff': False,
             'depends_on_past': True,
         },
         description="A DAG for hyperparameter tuning",
         start_date=datetime(2023,5,15),
         schedule_interval=None,
         catchup=True,
         tags=["mlops"],
         max_active_runs=1,
         ) as dag:

    @task.branch(task_id="branch_task")
    def branch_func(**kwargs):
        ti = kwargs['ti']
        xcom_value = int(ti.xcom_pull(task_ids="check_git_task"))
        if xcom_value == 0:
            return "create_env_task"
        else:
            return None

    check_git_task = BashOperator(
        task_id="check_git_task",
        bash_command=" bash -i /opt/airflow/dags/scripts/check_git.sh ",
    )
    check_git_task.do_xcom_push = True

    branch_op = branch_func()

    create_env_task = BashOperator(
        task_id="create_env_task",
        bash_command=" bash -i /opt/airflow/dags/scripts/create_env.sh ",
        retries=1,
    )

    tune_task = BashOperator(
        task_id="tune_task",
        bash_command=" bash -i /opt/airflow/dags/scripts/train.sh ",
        retries=1,
    )

    deploy_task = BashOperator(
        task_id="deploy_task",
        bash_command=" bash -i /opt/airflow/dags/scripts/deploy.sh ",
        retries=1,
    )
    check_git_task >> branch_op >> create_env_task >> tune_task >> deploy_task


import random
print ("kita akan membuat program guessing game")
a = random.randint(-100,100)

is_correct = True

while is_correct:

    angka_tebakan = int(input("masukkan angka tebakan anda "))

    if angka_tebakan < a:
        print("anda anda terlalu kecil, coba lagi")
    elif angka_tebakan > a:
        print("angka anda terlalu besar, coba lagi")
    else:
        print ("selamat anda benar ")
        print("angka anda adalah ", a)
        is_correct = False

print("game pun selesai")

Learn Python/Projects/Game simple guessing gam.py


from datetime import datetime as dt
import sys
import os
import numpy as np
import matplotlib.pyplot as plt
import argparse

def load_data(path):
    return np.loadtxt(path, delimiter='\t', skiprows=1)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    parser.add_argument('--file', type=str, required=True)

    # Graph options
    parser.add_argument('--xlabel', type=str, default='million steps')
    parser.add_argument('--ylabel', type=str, default='score')

    parser.add_argument('--filename', type=str, default='result.png')

    parser.add_argument('--xplim', type=int, default=10)
    parser.add_argument('--yplim', type=int, default=500)
    parser.add_argument('--xnlim', type=int, default=0)
    parser.add_argument('--ynlim', type=int, default=0)

    parser.add_argument('--legend-pos', type=str, default='upper left')

    args = parser.parse_args()

    results = load_data(args.file)

    avg_y = results[:, 1]
    med_y = results[:, 2]
    x = np.asarray(range(len(med_y))) * 0.1

    plt.figure(figsize=(5, 4), dpi=80)

    plt.plot(x, med_y, label='median', linewidth=1)
    plt.plot(x, avg_y, label='average', linewidth=1)

    plt.legend(loc=args.legend_pos, fontsize=8)
    plt.xlim(args.xnlim, args.xplim)
    plt.ylim(args.ynlim, args.yplim)

    plt.xlabel(args.xlabel)
    plt.ylabel(args.ylabel)

    plt.savefig(args.filename)
    plt.show()


# 2 4
# # ABCD
# # ABCE
# # 1 2 3 4
sizes=list(map(int,input().split(' ')))
size=sizes[0]
title=sizes[1]
strs=[]
for i in range(size):
    strs.append(input())
grade=list(map(int,input().split(' ')))
res=0
maxindex=0
maxnum=0
for i in range(title):
    tmp = [0 for i in range(5)]
    for j in range(size):
        tmp[ord(strs[j][i])-65]=tmp[ord(strs[j][i])-65]+1
    maxnum=max(tmp)
    res=res+grade[i]*maxnum
print(res)


class Solution(object):
    def rob(self, nums):
        """
        :type nums: List[int]
        :rtype: int
        """
        if not nums:
            return 0

        n=len(nums)

        dp=[0 for _ in xrange(n+1)]

        for i in xrange(1,n+1):
            dp[i]=max(dp[i-1],nums[i-1]+dp[i-2])

        return max(dp)


from typing import List

class Solution:
    def longestCommonPrefix(self, strs: List[str]) -> str:
        result = ""

        if len(strs) == 0:
            return result

        for i in range(len(strs[0])):
            for j in range(1, len(strs)):
                if i >= len(strs[j]) or strs[0][i] != strs[j][i]:
                    return result
            result += strs[0][i]

        return result

if __name__ == "__main__":
    strs = ["flower", "flow", "flight"]
    print(Solution().longestCommonPrefix(strs))


"""
Author: Daniel Mohr.

Date: 2017-02-07 (last change).

License: GNU GENERAL PUBLIC LICENSE, Version 3, 29 June 2007.
"""

import logging

from .read_data_from_file import read_data_from_file

def add_logging_level_name(lvl, levelname):
    """
    :Author: Daniel Mohr
    :Email: daniel.mohr@dlr.de
    :Date: 2016-12-04 (last change).
    :License: GNU GENERAL PUBLIC LICENSE, Version 3, 29 June 2007.
    """
    methodname = levelname.lower()
    if hasattr(logging, levelname):
        print("debug: levelname already exists")
    if hasattr(logging, methodname):
        print("debug: methodname already exists in logging")
    if hasattr(logging.getLoggerClass(), methodname):
        print("debug: methodname already exists in logging.getLoggerClass")

    def logforlevel(self, message, *args, **kwargs):
        # pylint: disable=missing-docstring,protected-access
        if self.isEnabledFor(lvl):
            self._log(lvl, message, args, **kwargs)

    def logtoroot(message, *args, **kwargs):
        # pylint: disable=missing-docstring
        logging.log(lvl, message, *args, **kwargs)
    logging.addLevelName(lvl, levelname)
    setattr(logging, levelname, lvl)
    setattr(logging.getLoggerClass(), methodname, logforlevel)
    setattr(logging, methodname, logtoroot)

add_logging_level_name(15, "VERBOSEINFO")

__all__ = ['read_data_from_file']