# -*- coding: utf-8 -*-
"""
Created on Mon Feb 25 16:31:17 2019

@author: engelen
"""

import xarray as xr
from scipy import interpolate
from scipy.ndimage.morphology import binary_opening
import numpy as np
import os
if os.name == "posix":
    import matplotlib
    matplotlib.use('agg')
    import matplotlib.pyplot as plt
    plt.ioff()
else:
    import matplotlib.pyplot as plt

#%%Helper functions
def _griddata(ip, val, targgrid):
    if val.ndim == 1:
        ip.values = np.expand_dims(val, 1)
    else:
        ip.values = val
    return(np.reshape(ip(tuple([i.flatten() for i in targgrid])), targgrid[0].shape))

def _clip_f(arr, sel, a_min=None, a_max=None):
    """Wrapper for clipping with fancy indexing"""
    def check(a_m, sel):
        if a_m is None:
            return(a_m)
        else:
            return(a_m[:, sel])

    arr[:, sel] = np.clip(arr[:, sel], check(a_min, sel), check(a_max, sel))
    return(arr)

def _ellipse(phi_f, H, phis):
    return(np.multiply.outer(H/phi_f,np.sqrt(phi_f**2-phis**2)))

def _cart2pol(x, y):
    rho = np.sqrt(x**2 + y**2)
    phi = np.arctan2(y, x)
    return(rho, phi)

def _pol2cart(rho, phi):
    x = rho * np.cos(phi)
    y = rho * np.sin(phi)
    return(x, y)

def _calc_z_onshoreshelf(l_a, alpha, beta, apex, L_ab, x):
    z = np.zeros(x.shape)
    a_cells = x <= (l_a*L_ab)
    b_cells = x  > (l_a*L_ab)
    z[a_cells] = apex - (np.tan(alpha)) * x[a_cells]
    z[b_cells] = 0. - (np.tan(beta)) * x[:np.count_nonzero(b_cells)]
    return(z)

def _calc_z_slope(gamma, z_slope_end, x):
    return(z_slope_end + np.tan(gamma) * x[::-1])

def _calc_apex(l_a, alpha, L):
    return(L * l_a * np.tan(alpha))

def _calc_shelf_edge(l_b, beta, L):
    return(-L * l_b * np.tan(beta))

def _calc_1D_top_bot(l_a, alpha, beta, gamma, L, H_b, f_H, x):
    """Onshore is determined based on where the coastal shelf and coastal slope
    intersect. Everywhere where the coastal slope is higher than the coastal shelf is
    shelf. This leads to quite some non-intuitive code unfortunately for very
    thick systems, as here the coastal slope never exceeds the shelf.
    """
    top = np.zeros(x.shape)

    L_ab_old = 0
    L_ab = L

    count=0

    #Iteratively create crosssection as we do not know where the slope and
    #the coastal shelf will intersect
    while np.isclose(L_ab, L_ab_old) == False:
        L_ab_old = L_ab
        apex = _calc_apex(l_a, alpha, L_ab)
        top_ab = _calc_z_onshoreshelf(l_a, alpha, beta, apex, L_ab, x)

        bot = _calc_bot(apex, l_a, H_b, f_H, L, x)
        top_slope = _calc_z_slope(gamma, bot[-1], x)

        ab_cells = top_ab < top_slope

        if np.all(ab_cells==False):
            L_ab = L
        else:
            L_ab = x[ab_cells][-1]

        count+=1
        if count>999:
            raise ValueError("Stuck in infinite loop, geometry did not converge."
                             +"\nL_ab_old=%d \nL_ab=%d" % (L_ab_old, L_ab))

    c_cells = ~ab_cells

    if np.all(c_cells): #For deep systems : We don't want a coastal slope everywhere, should be all shelf
        top = top_ab
    else:
        top[ab_cells] = top_ab[ab_cells]
        top[c_cells] = top_slope[c_cells]

    L_a = L_ab * l_a

    return(top, bot, c_cells, L_a)

def _calc_bot(apex, l_a, H_b, f_H, L, x):
    H_apex = apex - H_b*f_H
    slope = (H_apex + H_b)/(L*l_a)
    return(H_apex - x*slope)

def _get_pizza_slice(d1, length, phis):
    idx_slice = d1["rho"] <= length
    return(_pol2cart(*np.meshgrid(d1["rho"][idx_slice], phis)))

def _cake_cuts(N, phi, rhos, f_offset = 0.):
    """Create N equally spaced lines through the delta.
    Similar to how you would subdivide a slice of cake.

    Parameters
    ----------
    N : int
        number of cuts

    phi : float
        phi

    rhos : array
        array with all the values of rho used in the polar discretization

    f_offset : float
        factor from zero to unity that is multiplied with the spacing to get an
        offset

    """

    assert(0. <= f_offset <= 1.0)

    phi_cuts = np.linspace(-phi/2, phi/2, num=N+2)[1:-1]
    offset = phi/(N+1)/2 * f_offset
    phi_cuts += offset

    #CHECK: If this leads to wide enough diagonal lines.
    x_cuts, y_cuts = _pol2cart(*np.meshgrid(rhos, phi_cuts))
    return(x_cuts, y_cuts)

def _rho_min_max(interface, rho):
    rho_ct = np.where(~np.isnan(interface), rho, 0)
    rho_max = np.max(rho_ct)
    cutoff = rho_max - 0.1 * rho_max
    rho_min = np.nanmin(np.where(np.max(rho_ct, axis=1) > cutoff, np.max(rho_ct, axis=1), np.nan))
    return(rho_min, rho_max)

def _round2cell(v, dcell):
    return(np.round(v/dcell, 0)*dcell)

#%%Main functions: Geometry
def create_crosssection(l_a, alpha, beta, gamma, H_b, f_H, L, n_inp):
    d1 = {}

    d1["rho"] = np.linspace(0, L, num=n_inp)

    d1["top"], d1["bot"], d1["slope"], L_a = _calc_1D_top_bot(l_a, alpha, beta, gamma,
                                          L, H_b, f_H, d1["rho"])

    return(d1, L_a)

def create_2D_top_bot(phi, d1, n_inp):
    d2 = {}

    phis = np.linspace(-phi/2, phi/2, num=n_inp-20)
    d2["rho"], d2["phi"] = np.meshgrid(d1["rho"], phis)
    d2["phi_nan"] = np.copy(d2["phi"])
    d2["x"], d2["y"] = _pol2cart(d2["rho"], d2["phi"])

    ##Create tops
    d2["tops"], _ = np.meshgrid(d1["top"], phis)

    ##Fit ellipse for bottom
    d2["bots"]= _ellipse(phi/2, d1["bot"], phis).T

    ##Clip off weird part where tops and bot intersect
    d2["tops"] = _clip_f(d2["tops"], d1["slope"], a_min = d2["bots"])
    d2["bots"] = _clip_f(d2["bots"], ~d1["slope"], a_max = d2["tops"])
    nan_idx = np.isclose(d2["tops"], d2["bots"])
    nan_idx[:, ~d1["slope"]] = False
    for key, arr in d2.items():
        if key == "phi_nan":
            arr[~nan_idx] = np.nan
        else:
            arr[nan_idx] = np.nan
    return(d2, nan_idx, phis)

def get_targgrid(dx, dy, L, phi_max):
    x_edge, y_edge = _pol2cart(L, phi_max)

    x_max = _round2cell(L, dx) + 0.5*dx
    x_out = np.arange(0.5 * dx, x_max+1, dx)
    y_max = _round2cell(y_edge, dy) + 0.5*dy
    y_out = np.arange(-y_max, y_max+1, dy)
    return(np.meshgrid(x_out, y_out))

def pol2griddata(poldata, nan_idx, griddata, key_prep = ""):
    #Remove nans,which broadcasts to 1 dimensional array.
    poldata_no_nan=dict([(key, arr[~nan_idx]) for key, arr in poldata.items()])

    vrbls = [i for i in poldata.keys() if not i in ["x", "y", "phi", "phi_nan", "rho"]]
    #Explicitly create NDINterpolator once so QHULL has to be called only once, _griddata then just overrides the values.
    ip = interpolate.LinearNDInterpolator(np.vstack((poldata_no_nan["x"],poldata_no_nan["y"])).T, poldata_no_nan[vrbls[0]])
    for vrbl in vrbls:
        key = key_prep+vrbl
        griddata[key] = _griddata(ip, poldata_no_nan[vrbl], (griddata["x"], griddata["y"]))
    return(griddata)

def get_edges(ibound, top, z_shelf):
    dz = ibound.z[1] - ibound.z[0]
    is_shelf = (top>=z_shelf)
    edges = (
            (
            #Only select top layer of the shelf
            is_shelf & (ibound.z>(top-dz))
            ) | (
            #Select the coastal slope + the part just underneath the shelf break (roll(x=-1))
            ~is_shelf.roll(x=-1, roll_coords=False) | ~is_shelf
                        #Cells need to be active
                    )
            ) & ibound

    edges_z = edges.where(edges==1) * edges.z
    edges_x = edges.where(edges==1) * edges.x

    edges = xr.where(
        (
                edges.x == edges_x.max(dim="x")
        ) | (
                edges.z == edges_z.max(dim="z")
                ), edges, 0)

    edges = edges.transpose("z", "y", "x")
    return(edges)

def pol_to_d2_grid(dx, dy, phi_f, d2, nan_idx, d2_conf, nan_conf):
    d2_grid = {}

    d2_grid["x"], d2_grid["y"] = get_targgrid(dx, dy,
           np.max(d2["x"][~nan_idx]), phi_f/2)
    d2_grid["rho"], d2_grid["phi"] = _cart2pol(d2_grid["x"], d2_grid["y"])
    d2_grid = pol2griddata(d2, nan_idx, d2_grid)
    if d2_conf is not None:
        d2_grid = pol2griddata(d2_conf, nan_conf, d2_grid, "conf_")
    return(d2_grid)

def dict_to_ds(dic, dims, coords):
    ds = xr.Dataset(
            dict([(key, (dims, value)) for key, value in dic.items()\
                  if key not in dims]),
            coords = coords)
    return(ds)

def ds_d2_grid(d2_grid):
    coords = {"x" : d2_grid["x"][0, :],
          "y" : d2_grid["y"][:, 0]}
    ds = dict_to_ds(d2_grid, ["y", "x"], coords)
    ds = ds.dropna("y", "all")
    #For some reason this also clips off the first row where y["all"]!=np.nan??
    return(ds)

def ds_d1(d1):
    return(dict_to_ds(d1, ["rho"], {"rho" : d1["rho"]}))

def create_3d_grid(d2_grid, d1, nz):

    #TODO: This selection should be actually done when determining d1,
    #but currently results in
    to_keep = d1["top"] > d1["bot"]

    z = np.linspace(np.min(d1["bot"][to_keep]),
                    np.max(d1["top"][to_keep]), num=nz)

    d3 = xr.Dataset(coords = dict(d2_grid.coords)).assign_coords(z=z)
    d3["IBOUND"] = xr.where((d3.z<d2_grid["tops"])&(d3.z>d2_grid["bots"]), 1, 0)
    #Brush away small isolated cells.
    d3["IBOUND"] = xr.full_like(d3["IBOUND"],
      #Maybe first allocate larger array to imitate mode="mirror" of convolvend
      binary_opening(d3["IBOUND"], structure=np.ones((2,2,2))).astype(np.int)
      )
    return(d3)

#%%Main functions: Geology
def get_relative_clay_depths(f_aqt, N_aqt):
    frac_sand = (1-f_aqt) / (N_aqt+1) #+1 because there also is a confining layer
    frac_clay = f_aqt/(N_aqt+1)

    rel_clay_depths = np.cumsum([frac_clay, frac_sand]*(N_aqt+1))
    return(rel_clay_depths)

def create_confining_layer(l_conf, d2, d1, phis, L_a, frac, n_inp):
    d2_clay = {}
    x_ons, y_ons = _get_pizza_slice(d1, L_a, phis)

    x_offset=L_a * (1-l_conf)
    rho_clay, phi_clay = _cart2pol(x_ons[0][-1] - x_offset, y_ons[0][-1])
    rhos_clay = np.linspace(0, rho_clay, num=n_inp)
    phis_clay = np.linspace(-phi_clay, phi_clay, num=n_inp-20)

    topf = interpolate.interp1d(d1["rho"], d1["top"], bounds_error=False, fill_value = np.nan)
    botf = interpolate.interp1d(d1["rho"], d1["bot"], bounds_error=False, fill_value = "extrapolate")
    d_clay_coast = (botf(rhos_clay[-1]+x_offset) * frac)

    depth_conf =  np.linspace(0, d_clay_coast, num=n_inp)

    d2_clay["tops"] = np.zeros((n_inp-20, n_inp)) + topf(rhos_clay+x_offset) #Hack to get 2D tops of clay layer
    d2_clay["bots"] = d2_clay["tops"] - _ellipse(phi_clay, depth_conf, phis_clay).T

    d2_clay["x"], d2_clay["y"] = _pol2cart(*np.meshgrid(rhos_clay, phis_clay))
    d2_clay["x"] += x_offset

    #deactivate clay_layer past coastline
    nan_clay = np.greater(d2_clay["x"].T, np.nanmax(x_ons, axis = 1)).T
    for key, arr in d2_clay.items():
        arr[nan_clay] = np.nan
    return(d2_clay, nan_clay)

def create_clayer(frac, d1, d2, phis, phi_f, a_real, bot_offset=0):
    idx = int(d1["top"].shape[0] * a_real)
    base_slope = interpolate.interp1d(np.arange(idx), d1["top"][:idx], fill_value="extrapolate")
    base_slope = base_slope(np.arange(d1["top"].shape[0]))

    depth_clay = base_slope  - (base_slope-d1["bot"])*(frac)

    clayer = (_ellipse(phi_f/2, depth_clay, phis).T + depth_clay)/2.
    in_prism=(clayer<(d2["tops"]+bot_offset)) & (clayer>d2["bots"])
    clayer[~in_prism] = np.nan
    return(clayer)

def create_clayers(fracs, d1, d2, phis, phi_f, a_real, N_aqt):
    rho_min, rho_max = {}, {}

    for i in range(N_aqt):
        d2["ct%d"%i] = create_clayer(fracs[1::2][i], d1, d2, phis, phi_f, a_real)
        d2["cb%d"%i] = create_clayer(fracs[2::2][i], d1, d2, phis, phi_f, a_real)

        rho_min["ct%d"%i], rho_max["ct%d"%i]  = _rho_min_max(d2["ct%d"%i], d2["rho"])
        rho_min["cb%d"%i], rho_max["cb%d"%i]  = _rho_min_max(d2["cb%d"%i], d2["rho"])

        #Create edges at shoreline, so they extent fully to the sea
        d2["ct%d"%i] = np.where(np.isnan(d2["ct%d"%i]) & (d2["cb%d"%i] < d2["tops"]), d2["tops"], d2["ct%d"%i])
        #Find maximum the clay layer is allowed to be. Used to fix interpolation artifacts in finish_clayer_grid
        d2["ctmax%d"%i] =  np.full_like(d2["ct0"], 1) * np.nanmax(d2["ct0"], axis=0)
        #Create bottom at the side edges
        d2["cb%d"%i] = np.where(np.isnan(d2["cb%d"%i]) & (d2["ct%d"%i] > d2["bots"]), d2["bots"], d2["cb%d"%i])
        #Find minimum the clay layer is allowed to be. Used to fix interpolation artifacts in finish_clayer_grid
        d2["cbmin%d"%i] =  (np.full_like(d2["cb%d"%i], 1).T * np.nanmin(d2["cb%d"%i], axis=1)).T

    return(rho_min, rho_max, d2)

def finish_clayer_grid(d2_grid, N_aqt, rho_min, rho_max):
    """Final touch to erase errors caused by the interpolation to a grid
    """

    for i in range(N_aqt):
        #Make sure empty corners of clay layers are filled
        corner_top = ((d2_grid["rho"] < rho_max['ct%d'%i]) & (d2_grid["rho"] > rho_min['ct%d'%i]) & np.isnan(d2_grid["ct%d"%i]))
        corner_bot = ((d2_grid["rho"] < rho_max['cb%d'%i]) & (d2_grid["rho"] > rho_min['cb%d'%i]) & np.isnan(d2_grid["cb%d"%i]))

        d2_grid["ct%d"%i] = np.where(corner_top, np.minimum(d2_grid["tops"], d2_grid["ctmax%d"%i]), d2_grid["ct%d"%i])
        d2_grid["cb%d"%i] = np.where(corner_bot, d2_grid["cbmin%d"%i], d2_grid["cb%d"%i])
    return(d2_grid)

def create_paleo_channels(d2_ds, N_aqt, N_pal, s_pal, phi_f, rhos): #Perhaps also include w_pal
    pal_masks = {}

    offset = np.zeros(N_aqt)
    offset[::2] = s_pal

    for i in range(N_aqt):
        pal = _cake_cuts(N_pal, phi_f, rhos, f_offset = offset[i])
        pal = [p.flatten() for p in pal]

        da = d2_ds.sel(x=xr.DataArray(pal[0], dims="foo"),
                       y=xr.DataArray(pal[1], dims="foo"),
                       method="nearest")

        pal_masks[i] = ((d2_ds.x == da.x) & (d2_ds.y == da.y)).max(dim="foo")

    return(pal_masks)

def create_lith(d3, d2_grid, N_aqt, pal_mask):
    d3["lith"] = d3["IBOUND"]

    #Confining clayer gets nr 2 as nr
    conf_nr = 2
    if "conf_tops" in d2_grid.keys():
        d3["lith"] = xr.where((d3.z<d2_grid["conf_tops"])&(d3.z>d2_grid["conf_bots"]), conf_nr, d3["lith"])

    pal_nr = 3
    for i in range(N_aqt):
        #The other clay layer are assigned a number exceeding pal_nr
        clay_nr = 1 + pal_nr + i
        in_clayer = (d3.z<d2_grid["ct%d"%i])&(d3.z>d2_grid["cb%d"%i])
        d3["lith"] = xr.where(in_clayer, clay_nr, d3["lith"])

    d3["aqt"] = d3["lith"].copy() #Save backup of aquitards numbers without paleochannels

    for i in range(N_aqt):
        in_clayer = (d3.z<d2_grid["ct%d"%i])&(d3.z>d2_grid["cb%d"%i])
        d3["lith"] = xr.where(in_clayer & (pal_mask[i]),  pal_nr , d3["lith"])
    return(d3)

def calc_clay_thicknesses(d2_grid, N_aqt):
    if "conf_tops" in d2_grid.keys():
        d2_grid["conf_d"] = d2_grid["conf_tops"] - d2_grid["conf_bots"]
    for i in range(N_aqt):
        d2_grid["cd%d"%i] = d2_grid["ct%d"%i] - d2_grid["cb%d"%i]
    return(d2_grid)

def create_Kh(d3, Kh_aqf=0., Kv_aqt=0., f_Kh_pal=0., Kh_Kv=0., **kwargs):
    Kh_aqt = Kv_aqt*Kh_Kv
    b = np.log10(Kh_aqt)
    a = np.log10(Kh_aqf) - np.log10(Kh_aqt)

    Kh_pal = 10**(a*f_Kh_pal + b)

    d3["Kh"] = xr.zeros_like(d3["lith"])
    d3["Kh"] = xr.where(d3["lith"]==1, Kh_aqf,  d3["Kh"])
    d3["Kh"] = xr.where(d3["lith"]==2, Kh_aqt,  d3["Kh"])
    d3["Kh"] = xr.where(d3["lith"]==3, Kh_pal,  d3["Kh"])
    d3["Kh"] = xr.where(d3["lith"]>3,  Kh_aqt,  d3["Kh"])
    return(d3)

def calculate_aqfrs(geo, Kh_aqf=None, **pars):
    pal_nr = 3
    lith = geo["aqt"] - (pal_nr-1) #everything below this value is not an aquitard

    #This step not necessary if this issue gets fixed https://github.com/pydata/xarray/issues/2017
    aqfrs = xr.full_like(lith, 0, dtype=np.int64)

    axis = dict(axis = aqfrs.get_axis_num("z"))

    #Flipping to do a reverse accumulate
    aqfrs.values = np.flip(np.maximum.accumulate(np.flip(lith.values, **axis), **axis), **axis)

    aqfrs = aqfrs.clip(min=1)

    #Select aquifers, not aquitards
    aqfrs = xr.where(geo["Kh"].isel(time=-1)==(Kh_aqf), aqfrs, np.nan)
    return(aqfrs)

#%%Sedimentation/Erosion
def dynamic_confining_layer(d3, sea, t_tra):
    sea = sea.max(dim="z")
    d3["lith"] = xr.where((d3["lith"] == 2) & (sea == 1)          , 1, d3["lith"]).astype(np.int64)
    d3["lith"] = xr.where((d3["lith"] == 2) & (d3["time"] > t_tra), 1, d3["lith"]).astype(np.int64)
    d3 = d3.transpose("time", "z", "y", "x")
    return(d3)

def erosion_aquitards(d3, is_bc, bcs):
    #Choose maximum over z to keep aquitards active near the coastal slope.
    #Coastal shelf should only have one layer of bc per timestep.
    z_max_z = xr.where(is_bc, bcs.z, np.nan).max(dim="z")
    #Coordinates have to be monotonically increasing for ufunc
    z_max_z = z_max_z.assign_coords(time=z_max_z.time[::-1])

    #ignore nans seems not to be supported yet so have to use sentinel values
    #also erode clay at edges that are just left untouched by our bcs
    sentinel=9999
    ffill = z_max_z.min(dim="y").fillna(sentinel)
    z_max_z = z_max_z.fillna(ffill)

    ##Take minimum over time
    #This seems to work, I do not fully understand the docs though.
    z_min_t = xr.apply_ufunc(np.minimum.accumulate, z_max_z,
                           input_core_dims  = [["x", "y"]],
                           output_core_dims = [["x", "y"]])
    z_min_t = z_min_t.where(z_min_t<sentinel).assign_coords(time=z_min_t.time[::-1])

    d3["lith"] = xr.where((d3["lith"] >= 3) & (d3["z"]>z_min_t), 1, d3["lith"]).astype(np.int64)
    d3 = d3.transpose("time", "z", "y", "x")
    return(d3)

#%%Extra information
def add_topsystem(d2_grid):
    d2_grid["topsys"] = (np.nan_to_num(d2_grid["tops"]/d2_grid["tops"]) +
               (d2_grid["tops"] >= 0)).astype(np.int8)

    if "conf_tops" in d2_grid.keys():
        d2_grid["topsys"] += np.nan_to_num(d2_grid["conf_tops"]/d2_grid["conf_tops"]).astype(np.int8)

    return(d2_grid)

#%%Master function
def get_geometry(l_a=None,   alpha=None,  beta=None,   gamma=None,   L=None,
                 H_b=None,   f_H=None,    phi_f=None,
                 f_aqt=None, N_aqt=None,  l_conf=None, N_pal=None, s_pal=None,
                 dx=None,    dy=None,     nz=None,    figfol=None, ncfol=None,
                 **kwargs):

    n_inp = 200 #Discretization polar coordinates, not in actual model

    #Process central crosssection
    d1, L_a = create_crosssection(l_a, alpha, beta, gamma, H_b, f_H, L, n_inp)

    #Create 3D top & bottom
    d2, nan_idx, phis = create_2D_top_bot(phi_f, d1, n_inp)

    #Create confining layer
    frac_clay = f_aqt/(N_aqt+1)

    if l_conf > 0.:
        d2_conf,nan_conf = create_confining_layer(l_conf, d2, d1,
                                                  phis, L_a, frac_clay, n_inp)
    else:
        d2_conf,nan_conf = None, None

    #Create clay layers
    rel_clay_depths = get_relative_clay_depths(f_aqt, N_aqt)
    rho_min, rho_max, d2 = create_clayers(rel_clay_depths,
                                          d1, d2, phis, phi_f, L_a/L, N_aqt)

    if figfol is not None:
        clayer_plot(d2, d2_conf, N_aqt, L_a, L, figfol)

    #Convert to Cartesian regular grid
    d2_grid = pol_to_d2_grid(dx, dy, phi_f, d2, nan_idx, d2_conf, nan_conf)

    #Fill up last gaps in clayers
    d2_grid = finish_clayer_grid(d2_grid, N_aqt, rho_min, rho_max)

    if figfol is not None:
        top_bot_grid_plot(d2_grid,figfol)

    #Convert to xarray
    d2_grid = ds_d2_grid(d2_grid)
    d1 = ds_d1(d1)

    #Mask paleo channels
    pal_mask = create_paleo_channels(d2_grid, N_aqt,
                                     N_pal, s_pal, phi_f, d1["rho"])

    #Add topsystem
    d2_grid = add_topsystem(d2_grid)
    if figfol is not None:
        _plot_imshow(d2_grid["topsys"], os.path.join(figfol, "topsystem_grid.png"))

    #Calculate clay thicknesses
    d2_grid = calc_clay_thicknesses(d2_grid, N_aqt)

    #Create 3D
    d3 = create_3d_grid(d2_grid, d1, nz)
    d3 = create_lith(d3, d2_grid, N_aqt, pal_mask)

    if np.all(d1["slope"]==True):
        z_shelf_edge = d1["top"][-1]
    else:
        z_shelf_edge = d1["top"][~d1["slope"]][-1]
    d3["edges"] = get_edges(d3["IBOUND"], d2_grid["tops"], z_shelf_edge)
    d3["topsys"], d3["tops"], d3["bots"] = d2_grid["topsys"], d2_grid["tops"], d2_grid["bots"]

    #Save as netcdf
    if ncfol is not None:
        d3.to_netcdf(os.path.join(ncfol, "geo.nc"))

    #Assign layers after writing nc, as Paraview otherwise struggles with double coordinates
    layers = xr.DataArray(np.arange(len(d3.z))[::-1]+1, coords={"z":d3.z}, dims=["z"])
    d3 = d3.assign_coords(layer = layers)

    return(d3, d1, L_a)

#%%Plot functions
def clayer_plot(d2, d2_conf, N_aqt, L_a, L, figfol, ext=".png"):
    id_x = int(d2["bots"].shape[1] * L_a /L / 2)

    slcs = [np.s_[:, id_x],np.s_[90, :]]
    dims = ["y", "x"]
    fig, axes = plt.subplots(ncols=2, figsize=(8, 4))

    for i, slc in enumerate(slcs):
        xtop = xbot = d2[dims[i]][slc]
        axes[i].plot(xbot, d2["bots"][slc], label = "bot")
        axes[i].plot(xtop, d2["tops"][slc], label = "top")

        if dims[i] == "y":
            xval = d2["x"][slc][90]
            idx = np.searchsorted(d2["x"][90, :], xval)
            idx = np.s_[:, idx]
        else:
            axes[i].axvline(x=L_a, ls=":", color=".20")
            axes[i].axvline(x=L, color="k")
            idx = slc

        if d2_conf is not None:
            xconf = d2_conf[dims[i]][idx]
            axes[i].fill(np.append(xconf[::-1], xconf), np.append(d2_conf["tops"][idx][::-1], d2_conf["bots"][idx]), label="conf")

        for j in range(N_aqt):
            xtop = xbot = d2[dims[i]][slc]
            top = d2["ct%d"%j][slc]
            bot = d2["cb%d"%j][slc]

            xtop = xtop[~np.isnan(top)]
            xbot = xbot[~np.isnan(bot)]

            axes[i].fill(np.append(xtop[::-1], xbot), np.append(top[~np.isnan(top)][::-1], bot[~np.isnan(bot)]), label = "c%d"%j)

    plt.legend()
    plt.tight_layout()
    plt.savefig(os.path.join(figfol, "clayers%s" % ext))
    plt.close()

def _plot_imshow(da, fp):
    plt.imshow(da)
    plt.savefig(fp)
    plt.close()

def top_bot_grid_plot(d2_grid,figfol):
    _plot_imshow(d2_grid["tops"], os.path.join(figfol, "top_grid.png"))
    _plot_imshow(d2_grid["bots"], os.path.join(figfol, "bot_grid.png"))


import numpy as np
import time
from Solutions import ex_ruido, ex_grasp
from neighborhood import two_opt, two_opt_ruido, insertion, exchange, perturbation
np.warnings.filterwarnings('ignore')

# Process the initial solution to obtain a list of numpy arrays of each route of the multitrip problem
def process_route(routes, distance):
  vectores = []
  dist_total = []
  for route in routes:
    vector = [0]
    for i in range(len(route)):
      if route[i] == 0:
        if i != 0 and i != len(route)-1:
          vector.append(0)
          vectores.append(np.array(vector))
          vector = [0]
      else:
        vector.append(route[i])
    if vector:
      vector.append(0)
      vectores.append(np.array(vector))

  for i in range(len(vectores)):
    dist_route = []
    for j in range(len(vectores[i])-1):
      dist_route.append(distance[vectores[i][j]][vectores[i][j+1]])
    dist_total.append(sum(dist_route))

  return vectores, dist_total

# Assign which car is going to do each route
def assign_routes(duplas_ordenadas, R):
  # Divide the list in R equal parts
  partes = np.array_split(duplas_ordenadas, R)
  rutas = []
  distancias = []
  for parte in partes:
    # Create a route for each car, that consists of the concatenation of several routes
    ruta = np.concatenate([d[1][:-1] for d in parte] + [np.array([0])])
    distancia = sum([d[0] for d in parte])

    rutas.append(ruta)
    distancias.append(distancia)
  return rutas, distancias

# This function will return the route and the total distance of each car
def neighborhood(route, distances, request, R, Q, neighborhood_type): #x debe ser un numpy array
  [route, obj] = process_route(route,distances)
  optimal_pair = []
  # If the neighborhood type is 1, then we will apply 2-opt
  if neighborhood_type == 1:
    for i in range(len(route)):
      [best_sub_route, best_sub_fo] = two_opt(route[i], obj[i], distances)
      element = (best_sub_fo, best_sub_route)
      optimal_pair.append(element)
  # If the neighborhood type is 2, then we will apply insertion
  elif neighborhood_type == 2:
    [best_route, best_fo] = insertion(route, obj, request, distances, Q)
    optimal_pair = list(zip(best_fo, best_route))
  # If the neighborhood type is 3, then we will apply exchange
  else:
    [best_route, best_fo] = exchange(route, obj, request, distances, Q)
    optimal_pair = list(zip(best_fo, best_route))

  [car_routes, car_distances] = assign_routes(optimal_pair,R)
  return car_routes, car_distances

# This function will return the route and the total distance of each car
def neighborhood_mixed(route, distances, request, R, Q, neighborhood_type, param_num, param_ruido): #x debe ser un numpy array
  [route, obj] = process_route(route,distances)
  optimal_pair = []
  # If the neighborhood type is 1, then we will apply 2-opt
  if neighborhood_type == 1:
    for i in range(len(route)):
      [best_sub_route, best_sub_fo] = two_opt_ruido(route[i], obj[i], distances, param_ruido)
      element = (best_sub_fo, best_sub_route)
      optimal_pair.append(element)
  # If the neighborhood type is 2, then we will apply insertion
  elif neighborhood_type == 2:
    [best_route, best_fo] = perturbation(route, obj, request, distances, Q, param_num)
    optimal_pair = list(zip(best_fo, best_route))
  # If the neighborhood type is 3, then we will apply exchange
  else:
    [best_route, best_fo] = insertion(route, obj, request, distances, Q)
    optimal_pair = list(zip(best_fo, best_route))

  [car_routes, car_distances] = assign_routes(optimal_pair,R)
  return car_routes, car_distances

def func_VND(file_path, param, start_time, time_limit):   #max_iter es el numero de vecindarios
  [initial_route, initial_dist, dist, request, R, Th, Q, timer] = ex_ruido(file_path, param)
  solution = initial_route
  best_Th = initial_dist
  aux_init = [x > Th for x in initial_dist]
  cons_len_init = [int(x) if x else 0 for x in aux_init]
  rest_Th = sum(cons_len_init)
  best_fo = sum(initial_dist)
  neigh_type = 1
  n = 3

  while((time.time()-start_time)<time_limit and neigh_type <= n):
    [neighbor, neigh_Th_dist] = neighborhood(solution, dist, request, R, Q, neigh_type)
    neigh_aux = [x > Th for x in neigh_Th_dist]
    cons_len_neigh = [int(x) if x else 0 for x in neigh_aux]

    if(sum(neigh_Th_dist) < best_fo and sum(cons_len_neigh)<=rest_Th):
      solution = neighbor
      best_Th = neigh_Th_dist
      rest_Th = sum(cons_len_neigh)
      best_fo = sum(neigh_Th_dist)

      neigh_type = 1
    else:
        neigh_type += 1

  return solution, best_Th, R, Th

def func_VND_mixed(file_path, param_grasp, param_num, param_ruido, start_time, time_limit):
  sol_final = []
  while((time.time()-start_time)<time_limit):
    [initial_route, initial_dist, dist, request, R, Th, Q, timer] = ex_grasp(file_path, param_grasp)
    solution = initial_route
    best_Th = initial_dist
    aux_init = [x > Th for x in best_Th]
    cons_len_init = [int(x) if x else 0 for x in aux_init]
    rest_Th = sum(cons_len_init)
    best_fo = sum(best_Th)
    neigh_type = 1
    n = 3
    while((time.time()-start_time)<time_limit and neigh_type <= n):
      [neighbor, neigh_Th_dist] = neighborhood_mixed(solution, dist, request, R, Q, neigh_type, param_num, param_ruido)
      neigh_aux = [x > Th for x in neigh_Th_dist]
      cons_len_neigh = [int(x) if x else 0 for x in neigh_aux]

      if(sum(neigh_Th_dist) < best_fo and sum(cons_len_neigh)<=rest_Th):
        solution = neighbor
        best_Th = neigh_Th_dist
        rest_Th = sum(cons_len_neigh)
        best_fo = sum(neigh_Th_dist)

        neigh_type = 1
      else:
          neigh_type += 1
    sol_final.append((sum(best_Th), best_Th, solution))
  duplas_ordenadas = sorted(sol_final, key=lambda x: x[0])
  dist_Th = duplas_ordenadas[0][1]
  final_solution = duplas_ordenadas[0][2]

  return final_solution, dist_Th, R, Th


from PIL import Image
import numpy as np
from torchvision import datasets, transforms
import matplotlib.pyplot as plt
plt.rcParams['figure.figsize'] = [10, 5]

def crop_image(img, d=32):
    '''Make dimensions divisible by `d`'''

    new_size = (img.size[0] - img.size[0] % d,
                img.size[1] - img.size[1] % d)

    bbox = [
            int((img.size[0] - new_size[0])/2),
            int((img.size[1] - new_size[1])/2),
            int((img.size[0] + new_size[0])/2),
            int((img.size[1] + new_size[1])/2),
    ]

    img_cropped = img.crop(bbox)
    return img_cropped

def load_image(fname, img_width=None, img_height=None):
    if img_width is None or img_height is None:
        transform = transforms.Compose([transforms.ToTensor()])
    else:
        transform = transforms.Compose([transforms.Resize((img_height, img_width)),
                                        transforms.ToTensor()])
    img = Image.open(fname)
    img = crop_image(img, 64)
    img = transform(img)
    img = img.numpy()

    mask = np.random.randint(0, 2, size=img.shape)[0]

    new_mask = np.zeros(img.shape)
    for i in range(len(new_mask)):
        new_mask[i, :,: ] = mask
    mask = new_mask

    corrupted_img = mask * img

    return img, corrupted_img, mask

def visualize_corrupted_image(clean_img, corrupted_img):

    fig, ax = plt.subplots(1,2)
    if clean_img.shape[0] > 1:
        vis_corrupted = np.rollaxis(corrupted_img, 0, 3)
        clean_image = np.rollaxis(clean_img, 0, 3)
        ax[0].imshow(clean_image)
        ax[0].axis("off")
        ax[1].imshow(vis_corrupted)
        ax[1].axis("off")

    else:
        vis_corrupted = corrupted_img[0]
        clean_image = clean_img[0]
        cmap='Greys'
        ax[0].imshow(1-clean_image, cmap=cmap)
        ax[0].axis("off")
        ax[1].imshow(1-vis_corrupted, cmap=cmap)
        ax[1].axis("off")
    plt.show()


class Solution(object):
    def hasCycle(self, head):
        """
        :type head: ListNode
        :rtype: bool
        """

        slow = head
        fast = head

        while fast and fast.next:
            slow = slow.next
            fast = fast.next.next

            if slow == fast:
                return True

        return False


#베르트랑 공준 - 기초 수학 2
#에라토스테네스의 체 사용
'''
에라토스테네스의 체 알고리즘
2부터 소수를 구하고자 하는 구간의 모든 수를 나열한다.
2부터 시작해서 나열된 수에서 지워지지 않은 수 중 가장 작은 2를 소수로 선택하고 2의 배수를 지운다.
3도 지워지지 않았기 때문에 소수로 선택하고 3의 배수를 지운다.
4는 지워졌기 때문에 넘어가고 5를 소수로 선택하고 5의 배수를 지운다.
2,3,4와 같은 과정을 반복한다.
반복이 끝나면 지워지지 않은 수들을 소수로 출력한다.
'''
import math
from sys import stdin

ans = []

#에라토스테네스의 체
def eratos(n):
    if n == 2:
        return True
    if n % 2 == 0:
        return False

    for i in range(2,int(math.sqrt(n))+1):
        if n%i == 0 :
            return False

    return True

while True:
    n = int(stdin.readline())

    if n == 0:
        break

    cnt = 0

    for i in range(n+1,(2*n)+1):
        if eratos(i):
            cnt += 1

    print(cnt)


t = int(input())
for i in range(0, t):
    n = int(input())
    lstA = list(map(int, input().split(" ")))
    setA = set(lstA)
    if len(setA) == 1:
        print(0)
    else:
        miN = min(lstA)
        count = 0
        for item in lstA:
            if item == miN:
                count += 1
        print(n - count)


import pygame
import random
import sys

# Defining blocks number, blocks size, margin and etc:
masLength = masHeight = BLOCKS = 4
SIZE_BLOCK = 93
MARGIN = 7
EDGE_MARGIN = 4
UP_BLOCK = 140
DOWN_BLOCK = 90

# Colors in RGB:
WHITE = (255, 255, 255)
BLACK = (0, 0, 0)
GREY = (195, 195, 195)

class Block:
    size = SIZE_BLOCK

    def __init__(self, number):
        if number == 16:
            self.number = ''
            self.color = GREY
        else:
            self.number = number
            self.color = WHITE

class Puzzle:
    def __init__(self):
        self.container = {}
        for i in range(1, 17):
            self.container[i] = Block(i)

    @property
    def empty_position(self):
        for key in self.container:
            if self.container[key].number == '':
                return key

    def move(self, empty_position, key_change):
        self.container[empty_position], self.container[empty_position + key_change] = \
            self.container[empty_position + key_change], self.container[empty_position]

    def move_left(self):
        self.move(self.empty_position, 1)

    def move_right(self):
        self.move(self.empty_position, -1)

    def move_up(self):
        self.move(self.empty_position, 4)

    def move_down(self):
        self.move(self.empty_position, -4)

    def shuffle(self):
        a = list(range(1, masLength * masHeight + 1))
        random.shuffle(a)
        n = 0
        for i in range(len(a) - 1):
            if a[i] == 16:
                continue
            for j in range(i + 1, len(a)):
                if a[j] == 16:
                    continue
                if a[i] > a[j]:
                    n += 1
        if n % 2 == 0:
            if a[-1] == 16:
                a[-2], a[-3] = a[-3], a[-2]
            elif a[-2] == 16:
                a[-1], a[-3] = a[-3], a[-1]
            else:
                a[-1], a[-2] = a[-2], a[-1]
        b = []
        self.container = {}
        for i in range(masLength):
            if i % 2 == 0:
                b.append(a[i * masLength:i * masLength + masLength])
            else:
                b.append(a[i * masLength + masLength - 1:i * masLength - 1:-1])
        for i in range(len(b)):
            for j in range(len(b[i])):
                self.container[i * masLength + j + 1] = Block(b[i][j])

# Game window size:
WIDTH = SIZE_BLOCK * BLOCKS + MARGIN * (BLOCKS - 1) + EDGE_MARGIN * 2
HEIGH = UP_BLOCK + SIZE_BLOCK * BLOCKS + MARGIN * (BLOCKS - 1) + DOWN_BLOCK

# Active game area:
GAME_AREA = pygame.Rect(EDGE_MARGIN, UP_BLOCK, SIZE_BLOCK * BLOCKS + MARGIN * (BLOCKS - 1),
                        SIZE_BLOCK * BLOCKS + MARGIN * (BLOCKS - 1))

screen = pygame.display.set_mode((WIDTH, HEIGH))
pygame.display.set_caption('15 PUZZLE')

if __name__ == '__main__':
    pygame.mixer.pre_init(44100, -16, 1, 512)
    pygame.init()

    movement_sound = pygame.mixer.Sound('1.wav')  # Blocks moving sound
    shuffle_sound = pygame.mixer.Sound('2.wav')  # Shuffle sound

    pygame.mixer.Sound.set_volume(movement_sound, 0.07)
    pygame.mixer.Sound.set_volume(shuffle_sound, 0.6)

    # Fonts:
    font1 = pygame.font.SysFont('arial', 46, bold=True)
    font2 = pygame.font.SysFont('timesnewroman', 22, italic=True)
    font3 = pygame.font.SysFont('timesnewroman', 28)

    # Text to the passive game area design:
    text_rules_1 = font3.render('Shuffle puzzle and assemble', 0, WHITE)
    text_rules_2 = font3.render('numbers in the right order', 0, WHITE)
    text_press = font2.render('press', 0, WHITE)
    text_space = font3.render('Space', 0, WHITE)
    text_to = font2.render('to', 0, WHITE)
    text_shuffle = font3.render('SHUFFLE', 0, WHITE)

    puzzle = Puzzle()

    while True:
        for event in pygame.event.get():
            if event.type == pygame.QUIT or (event.type == pygame.KEYDOWN and event.key == pygame.K_ESCAPE):
                pygame.quit()
                sys.exit(0)
            elif event.type == pygame.KEYDOWN:
                if event.key == pygame.K_LEFT:
                    if puzzle.empty_position % 4 != 0:
                        puzzle.move_left()
                        pygame.mixer.Sound.play(movement_sound)
                elif event.key == pygame.K_RIGHT:
                    if puzzle.empty_position % 4 != 1:
                        puzzle.move_right()
                        pygame.mixer.Sound.play(movement_sound)
                elif event.key == pygame.K_UP:
                    if (puzzle.empty_position - 1) // 4 != 3:
                        puzzle.move_up()
                        pygame.mixer.Sound.play(movement_sound)
                elif event.key == pygame.K_DOWN:
                    if (puzzle.empty_position - 1) // 4 != 0:
                        puzzle.move_down()
                        pygame.mixer.Sound.play(movement_sound)
                elif event.key == pygame.K_SPACE:
                    puzzle.shuffle()
                    pygame.mixer.Sound.play(shuffle_sound)
        pygame.draw.rect(screen, GREY, GAME_AREA) # Drawing active game area with blocks on the screen:

        for key in puzzle.container:
            block_AREA = pygame.Rect(EDGE_MARGIN + (Block.size + MARGIN) * ((key - 1) % 4), UP_BLOCK + (Block.size
                        + MARGIN) * ((key - 1) // 4), Block.size, Block.size)
            pygame.draw.rect(screen, puzzle.container[key].color, block_AREA)
            text_rect = font1.render(str(puzzle.container[key].number), 1, BLACK).get_rect(center=(EDGE_MARGIN +
                        (Block.size + MARGIN) * ((key - 1) % 4) + int(Block.size / 2), UP_BLOCK + (Block.size +
                        MARGIN) * ((key - 1) // 4) + int(Block.size / 2)))
            screen.blit(font1.render(str(puzzle.container[key].number), 1, BLACK), text_rect)

        screen.blit(text_rules_1, (40, 30)) # Designing passive game area
        screen.blit(text_rules_2, (53, 70))
        screen.blit(text_press, (39, 564))
        screen.blit(text_space, (105, 560))
        screen.blit(text_to, (196, 564))
        screen.blit(text_shuffle, (236, 561))
        pygame.display.update()


n = int(input().strip())
rectanges = []
circles = []
for i in range(0, n):
	line = input().strip().split()
	if line[0] == "circle":
		circles.append(list(map(int, line[1:])))
	else:
		rectanges.append(list(map(int, line[1:])))
n = int(input())
for i in range(0, n):
	count = 0
	point = list(map(int, input().strip().split()))
	for r in rectanges:
		if r[0] <= point[0] <= r[2] and r[1] <= point[1] <= r[3]:
			count += 1
	for c in circles:
		if ((c[0]-point[0])**2 + (c[1]-point[1])**2)**0.5 <= c[2]:
			count += 1
	print(count)


numero0 = "32"
numero1 = 100
numero2 = 0

try:
    if(type(numero0) is not int):
        raise Exception("La variable no es numérica")  # lanzar excepciones
    print(numero1/numero2)
except ZeroDivisionError:
    print('Error: No se puede dividir por cero.')
except Exception as e:
    print(e)
except:
    print('Error.')
else:
    print('Todo ok.')
finally:
    print('Fin del programa.')


# -*- coding: utf-8 -*-

""" Utility helpers """
from __future__ import absolute_import

import json
import logging
import os
import time
from enum import Enum

import boto3
import click
import yaml
from future.moves import subprocess
from future.utils import iteritems
from halo import Halo
from kubernetes.client import Configuration
from kubernetes.config.kube_config import KubeConfigLoader

import ekscli
from ekscli import EKSCliException

LOG = logging.getLogger(ekscli.__app_name__)

# py2 compatibility
def which(executable):
    path = os.getenv('PATH')

    for p in path.split(os.path.pathsep):
        p = os.path.join(p, executable)
        if os.path.exists(p) and os.access(p, os.X_OK):
            return p

def get_stack(stack_name, expected_states):
    stacks = [s for s in boto3.resource('cloudformation').stacks.all() if s.stack_name == stack_name]
    return {
        'expected': [s for s in stacks if s.stack_status in expected_states],
        'unexpected': [s for s in stacks if s.stack_status not in expected_states]
    }

def delete_stack(stack_name, resources, stack=None):
    if not stack:
        stacks = get_stack(stack_name, ['DELETE_COMPLETE'])
        if len(stacks.get('unexpected')) == 0:
            LOG.warning('the stack of eks cluster [name={}] does not exists. Nothing to do.'.format(stack_name))
            return False
        elif len(stacks.get('unexpected')) > 1:
            raise EKSCliException('More than one EKS stack exists!')
        stack = stacks.get('unexpected')[0]

    cf = boto3.session.Session().resource('cloudformation')
    stack = cf.Stack(stack.stack_id)
    states = {rs.logical_resource_id: rs for rs in stack.resource_summaries.all()}
    for r in resources:
        if r.name in states:
            r.status = Status.created
            r.resource_id = states.get(r.name).physical_resource_id
        else:
            r.status = Status.provided

    if stack.stack_status not in ['DELETE_IN_PROGRESS']:
        stack.delete()

    reporter = ResourceReporter()
    reporter.report_stack_deletion(stack_name, resources, stack.stack_id)

    return True

# Todo: https://github.com/kubernetes-client/python/issues/514
def load_kubeconf(config_file):
    with open(config_file, 'r') as f:
        config = yaml.load(f)
        user = [user for user in config['users'] if user['name'] == 'aws'][0]['user']
        command = [user['exec']['command']]
        command.extend(user['exec']['args'])
        output = subprocess.check_output(command)
        c = json.loads(output.decode('utf-8'))
        user['token'] = c['status']['token']
        del user['exec']

        loader = KubeConfigLoader(config)
        config = type.__call__(Configuration)
        loader.load_and_set(config)
        Configuration.set_default(config)

class Status(Enum):
    provided = 0
    not_exist = 1
    creating = 2
    created = 3
    failed = 4
    deleting = 5
    deleted = 6

    def __str__(self):
        return ' '.join(self.name.split('_'))

class ResourceReporter:
    def __init__(self):
        self.spinner = Halo(text='', spinner='dots')

    def progress(self, resource):
        self.spinner.start(text='[ {0: <10}] {1}'.format(resource.status, resource.description))

    def succeed(self, resource):
        self.spinner.succeed(
            text='[ {0: <10}] {1} [{2}].'.format(resource.status, resource.description, resource.resource_id))

    def fail(self, resource):
        self.spinner.fail(text='[ {0: <10}] {1}'.format(resource.status, resource.description))

    def warn(self, text):
        self.spinner.warn(text=text)

    def info(self, resource):
        self.spinner.info(text='[ {} ] {} [{}].'.format(resource.status, resource.description, resource.resource_id))

    def report_stack_creation(self, name, resources, stack_id):
        cf = boto3.session.Session().resource('cloudformation')
        stack = cf.Stack(stack_id)
        rmap = {r.name: r for r in resources}
        completed = set()
        for r in resources:
            if r.status == Status.provided or r.status == Status.created:
                self.succeed(r)
                completed.add(r.name)

        current = None
        while stack.stack_status in ['CREATE_IN_PROGRESS', 'ROLLBACK_IN_PROGRESS']:
            states = {rs.logical_resource_id: rs for rs in stack.resource_summaries.all()}
            if current:
                rs = states.get(current)
                r = rmap.get(current)
                if rs.resource_status in ['CREATE_IN_PROGRESS']:
                    time.sleep(2)
                    continue
                else:
                    self.report_completed_resource(completed, r, rs, ['CREATE_COMPLETE'], Status.created)
                    current = None

            for name, rs in iteritems(states):
                r = rmap.get(name)
                if name not in completed and r:
                    if rs.resource_status in ['CREATE_IN_PROGRESS']:
                        current = name
                        r.status = Status.creating
                        self.progress(r)
                        break

                    self.report_completed_resource(completed, r, rs, ['CREATE_COMPLETE'], Status.created)
            time.sleep(2)
            stack = cf.Stack(stack_id)

        states = {rs.logical_resource_id: rs for rs in stack.resource_summaries.all()}
        if current:
            r = rmap.get(current)
            rs = states.get(current)
            self.report_completed_resource(completed, r, rs, ['CREATE_COMPLETE'], Status.created)

        for name, rs in iteritems(states):
            r = rmap.get(name)
            if name not in completed and r:
                self.report_completed_resource(completed, r, rs, ['CREATE_COMPLETE'], Status.created)

        if stack.stack_status in ['CREATE_FAILED', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_FAILED', 'ROLLBACK_COMPLETE']:
            raise EKSCliException('Failed to create EKS cluster {}: {}'.format(name, stack.stack_status))

        return stack

    def report_completed_resource(self, completed, resource, resource_summary, success_states, success):
        if resource_summary.resource_status in success_states:
            resource.status = success
            resource.resource_id = resource_summary.physical_resource_id
            self.succeed(resource)
        else:
            resource.status = Status.failed
            self.fail(resource)
        completed.add(resource.name)

    def report_stack_deletion(self, name, resources, stack_id):
        cf = boto3.session.Session().resource('cloudformation')
        stack = cf.Stack(stack_id)
        rmap = {r.name: r for r in resources}
        completed = set()
        for r in resources:
            if r.status == Status.not_exist or r.status == Status.deleted or r.status == Status.provided:
                self.succeed(r)
                completed.add(r.name)

        current = None
        while stack.stack_status not in ['DELETE_COMPLETE', 'DELETE_FAILED']:
            states = {rs.logical_resource_id: rs for rs in stack.resource_summaries.all()}
            if current:
                rs = states.get(current)
                r = rmap.get(current)
                if rs.resource_status in ['DELETE_IN_PROGRESS', 'CREATE_COMPLETE', 'UPDATE_COMPLETE']:
                    time.sleep(2)
                    continue
                else:
                    self.report_completed_resource(completed, r, rs, ['DELETE_COMPLETE'], Status.deleted)
                    current = None

            for name, rs in iteritems(states):
                r = rmap.get(name)
                if name not in completed and r:
                    if rs.resource_status in ['DELETE_IN_PROGRESS', 'CREATE_COMPLETE', 'UPDATE_COMPLETE']:
                        current = name
                        r.status = Status.deleting
                        self.progress(r)
                        break

                    self.report_completed_resource(completed, r, rs, ['DELETE_COMPLETE'], Status.deleted)
            time.sleep(2)
            stack = cf.Stack(stack_id)

        states = {rs.logical_resource_id: rs for rs in stack.resource_summaries.all()}
        if current:
            r = rmap.get(current)
            rs = states.get(current)
            self.report_completed_resource(completed, r, rs, ['DELETE_COMPLETE'], Status.deleted)

        for name, rs in iteritems(states):
            r = rmap.get(name)
            if name not in completed and r:
                self.report_completed_resource(completed, r, rs, ['DELETE_COMPLETE'], Status.deleted)

        if stack.stack_status in ['DELETE_FAILED']:
            raise EKSCliException('Failed to create EKS cluster {}: {}'.format(name, stack.stack_status))

        return

# https://stackoverflow.com/questions/37310718/mutually-exclusive-option-groups-in-python-click/
class MutuallyExclusiveOption(click.Option):
    mutex_groups = {}

    def __init__(self, *args, **kwargs):
        opts_list = kwargs.pop('mutex_group', '')
        self.mutex_group_key = ','.join(opts_list)
        self.mutex_groups[self.mutex_group_key] = 0
        help = kwargs.get('help', '')
        kwargs['help'] = help + (' NOTE: This argument may be one of: [{}].'.format(self.mutex_group_key))
        super(MutuallyExclusiveOption, self).__init__(*args, **kwargs)

    def handle_parse_result(self, ctx, opts, args):
        if self.name in self.mutex_group_key and self.name in opts:
            self.mutex_groups[self.mutex_group_key] += 1
        if self.mutex_groups[self.mutex_group_key] > 1:
            exclusive_against = self.mutex_group_key.split(',')
            exclusive_against.remove(self.name)
            raise click.UsageError("Illegal usage: `{}` is mutually exclusive against options {}.".format(
                self.name, exclusive_against))

        return super(MutuallyExclusiveOption, self).handle_parse_result(ctx, opts, args)


import itertools
import ctypes
import math

class BadAddress(Exception):
    pass

class BadInstruction(Exception):
    pass

class Memory:

    def __init__(self) -> None:
        self._storage = {}

    def get_byte(self, addr):
        if addr < 0:
            raise BadAddress
        return self._storage.get(addr, 0)

    def get_quad(self, addr):
        if addr < 0:
            raise BadAddress
        word = 0
        for pos in reversed(range(addr, addr + 8)):
            word <<= 8
            word |= self.get_byte(pos)
        return ctypes.c_long(word).value

    def set_byte(self, addr, byte):
        if addr < 0:
            raise BadAddress
        self._storage[addr] = byte & 255

    def set_quad(self, addr, word):
        if addr < 0:
            raise BadAddress
        for pos in range(addr, addr + 8):
            self.set_byte(pos, word)
            word >>= 8

    def to_json(self):
        data = {}
        for addr, _ in itertools.groupby(self._storage.keys(), lambda x: math.floor(x / 8) * 8):
            word = self.get_quad(addr)
            if word:
                data[addr] = word
        return data

    @classmethod
    def from_json(cls, data):
        mem = cls()
        for addr, word in data.items():
            mem.set_quad(int(addr), word)
        return mem

    def load_yo_buffer(self, buffer):
        for text in buffer.strip().split('\n'):
            addr_bincode, _ = text.split('|')
            addr_bincode = addr_bincode.split(':')
            if len(addr_bincode) == 2:
                addr, bincode = addr_bincode

                addr = int(addr, 16)
                bincode = bincode.strip()

                for byte_offset, byte_start_pos in enumerate(range(0, len(bincode), 2)):
                    byte = int(bincode[byte_start_pos: byte_start_pos + 2], 16)
                    self.set_byte(addr + byte_offset, byte)

class Registers:
    RAX = 0x0
    RCX = 0x1
    RDX = 0x2
    RBX = 0x3
    RSP = 0x4
    RBP = 0x5
    RSI = 0x6
    RDI = 0x7
    R8  = 0x8
    R9  = 0x9
    R10 = 0xa
    R11 = 0xb
    R12 = 0xc
    R13 = 0xd
    R14 = 0xe

    def __init__(self) -> None:
        self._reg = [
            0,  # RAX
            0,  # RCX
            0,  # RDX
            0,  # RBX
            0,  # RSP
            0,  # RBP
            0,  # RSI
            0,  # RDI
            0,  # R8
            0,  # R9
            0,  # R10
            0,  # R11
            0,  # R12
            0,  # R13
            0,  # R14
        ]

    def __getitem__(self, index):
        return ctypes.c_long(self._reg[index]).value

    def __setitem__(self, index, value):
        self._reg[index] = ctypes.c_long(value).value

    def to_json(self):
        data = {}
        for i, reg in enumerate(['rax', 'rcx', 'rdx', 'rbx', 'rsp', 'rbp', 'rsi', 'rdi', 'r8', 'r9', 'r10', 'r11', 'r12', 'r13', 'r14']):
            data[reg] = self[i]
        return data

    @classmethod
    def from_json(cls, data):
        reg = cls()
        reg._reg = [
            data.pop(r)
            for r in ['rax', 'rcx', 'rdx', 'rbx', 'rsp', 'rbp', 'rsi', 'rdi', 'r8', 'r9', 'r10', 'r11', 'r12', 'r13', 'r14']
        ]
        return reg

class OPCode:
    HALT   = 0x00
    NOP    = 0x10
    RRMOVQ = 0x20
    CMOVLE = 0x21
    CMOVL  = 0x22
    CMOVE  = 0x23
    CMOVNE = 0x24
    CMOVGE = 0x25
    CMOVG  = 0x26
    IRMOVQ = 0x30
    RMMOVQ = 0x40
    MRMOVQ = 0x50
    ADDQ   = 0x60
    SUBQ   = 0x61
    ANDQ   = 0x62
    XORQ   = 0x63
    JMP    = 0x70
    JLE    = 0x71
    JL     = 0x72
    JE     = 0x73
    JNE    = 0x74
    JGE    = 0x75
    JG     = 0x76
    CALL   = 0x80
    RET    = 0x90
    PUSHQ  = 0xa0
    POPQ   = 0xb0
    IADDQ  = 0xc0
    ISUBQ  = 0xc1
    IANDQ  = 0xc2
    IXORQ  = 0xc3

class CONDCode:
    NC = 0x0
    LE   = 0x1
    L    = 0x2
    E    = 0x3
    NE   = 0x4
    GE   = 0x5
    G    = 0x6

class OPQCode:
    ADD = 0x0
    SUB = 0x1
    AND = 0x2
    XOR = 0x3

class STATCode:
    AOK = 0x1
    HLT = 0x2
    ADR = 0x3
    INS = 0x4

def bisect_byte(byte):
    a = (byte >> 4) & 15
    b = byte & 15
    return a, b

class CPU:
    def __init__(self):
        self.PC = 0
        self.STAT = STATCode.AOK
        self.CC = {
            'ZF': 1,
            'SF': 0,
            'OF': 0,
        }
        self.REG = Registers()
        self.MEM = Memory()

    @property
    def state(self):
        state = {
            "PC": self.PC,
            "REG": self.REG.to_json(),
            "MEM": self.MEM.to_json(),
            "CC": self.CC,
            "STAT": self.STAT,
        }
        return state

    @state.setter
    def state(self, value):
        self.PC = value.pop('PC')
        self.CC = value.pop('CC')
        self.STAT = value.pop('STAT')
        self.MEM = Memory.from_json(value.pop('MEM'))
        self.REG = Registers.from_json(value.pop('REG'))

    def load_program(self, buffer):
        self.MEM.load_yo_buffer(buffer)

    def cond(self, ifun):

        ZF = bool(self.CC['ZF'])
        SF = bool(self.CC['SF'])
        OF = bool(self.CC['OF'])

        if ifun == CONDCode.NC:
            return True
        elif ifun == CONDCode.LE:
            return ZF ^ OF | SF
        elif ifun == CONDCode.L:
            return SF ^ OF
        elif ifun == CONDCode.E:
            return ZF
        elif ifun == CONDCode.NE:
            return not ZF
        elif ifun == CONDCode.GE:
            return not (SF ^ OF)
        elif ifun == CONDCode.G:
            return not (ZF ^ OF | SF)

    def update_CC(self, ifun, a, b, t):
        self.CC["ZF"] = int(t == 0)
        self.CC["SF"] = int(bool(t >> 63))

        if ifun == OPQCode.ADD:
            self.CC["OF"] = int(bool(a < 0 == b < 0 and t < 0 != a < 0))
        elif ifun == OPQCode.SUB:
            self.CC["OF"] = int(bool(a < 0 == b < 0 and t < 0 != b < 0))
        else:
            self.CC["OF"] = 0

    def op(self, ifun):
        if ifun == OPQCode.ADD:
            return lambda a, b: b + a
        elif ifun == OPQCode.SUB:
            return lambda a, b: b - a
        elif ifun == OPQCode.AND:
            return lambda a, b: b & a
        elif ifun == OPQCode.XOR:
            return lambda a, b: b ^ a

    def fetch_decode_execute(self):
        opcode = self.MEM.get_byte(self.PC)
        icode, ifun = bisect_byte(opcode)
        try:
            if opcode == OPCode.HALT:
                self.STAT = STATCode.HLT
            elif opcode == OPCode.NOP:
                val_p = self.PC + 1
                self.PC = val_p
            elif opcode in (OPCode.RRMOVQ,
                            OPCode.CMOVLE,
                            OPCode.CMOVL,
                            OPCode.CMOVE,
                            OPCode.CMOVNE,
                            OPCode.CMOVGE,
                            OPCode.CMOVG):
                val_p = self.PC + 2
                if self.cond(ifun):
                    ra, rb = bisect_byte(self.MEM.get_byte(self.PC + 1))
                    val_a = self.REG[ra]
                    self.REG[rb] = val_a
                self.PC = val_p
            elif opcode == OPCode.IRMOVQ:
                _, ra = bisect_byte(self.MEM.get_byte(self.PC + 1))
                val_p = self.PC + 10
                val_c = self.MEM.get_quad(self.PC + 2)
                self.REG[ra] = val_c
                self.PC = val_p
            elif opcode == OPCode.RMMOVQ:
                ra, rb = bisect_byte(self.MEM.get_byte(self.PC + 1))
                val_p = self.PC + 10
                val_c = self.MEM.get_quad(self.PC + 2)
                val_a = self.REG[ra]
                val_b = self.REG[rb]
                val_e = val_a + val_b
                self.MEM.set_quad(val_e, val_a)
                self.PC = val_p
            elif opcode == OPCode.MRMOVQ:
                ra, rb = bisect_byte(self.MEM.get_byte(self.PC + 1))
                val_c = self.MEM.get_quad(self.PC + 2)
                val_p = self.PC + 10
                val_b = self.REG[rb]
                val_e = val_b + val_c
                val_m = self.MEM.get_quad(val_e)
                self.REG[ra] = val_m
                self.PC = val_p
            elif opcode in (OPCode.ADDQ,
                            OPCode.SUBQ,
                            OPCode.ANDQ,
                            OPCode.XORQ):
                ra, rb = bisect_byte(self.MEM.get_byte(self.PC + 1))
                val_p = self.PC + 2
                val_a = self.REG[ra]
                val_b = self.REG[rb]
                val_e = self.op(ifun)(val_a, val_b)
                self.update_CC(ifun, val_a, val_b, val_e)
                self.REG[rb] = val_e
                self.PC = val_p
            elif opcode in (OPCode.JMP,
                            OPCode.JLE,
                            OPCode.JL,
                            OPCode.JE,
                            OPCode.JNE,
                            OPCode.JGE,
                            OPCode.JG):
                val_p = self.PC + 9
                if self.cond(ifun):
                    val_c = self.MEM.get_quad(self.PC + 1)
                    self.PC = val_c
                else:
                    self.PC = val_p
            elif opcode == OPCode.CALL:
                val_c = self.MEM.get_quad(self.PC + 1)
                val_p = self.PC + 9
                val_b = self.REG[Registers.RSP]
                val_e = val_b - 8

                self.MEM.set_quad(val_e, val_p)
                self.REG[Registers.RSP] = val_e
                self.PC = val_c
            elif opcode == OPCode.RET:
                val_p = self.PC + 1
                val_a = self.REG[Registers.RSP]
                val_b = self.REG[Registers.RSP]
                val_e = val_b + 8
                val_m = self.MEM.get_quad(val_a)
                self.REG[Registers.RSP] = val_e
                self.PC = val_m
            elif opcode == OPCode.PUSHQ:
                ra, _ = bisect_byte(self.MEM.get_byte(self.PC + 1))
                val_p = self.PC + 2
                val_a = self.REG[ra]
                val_b = self.REG[Registers.RSP]
                val_e = val_b - 8
                self.REG[Registers.RSP] = val_e
                self.MEM.set_quad(val_e, val_a)
                self.PC = val_p
            elif opcode == OPCode.POPQ:
                ra, _ = bisect_byte(self.MEM.get_byte(self.PC + 1))
                val_p = self.PC + 2
                val_a = self.REG[Registers.RSP]
                val_b = self.REG[Registers.RSP]
                val_e = val_b + 8
                val_m = self.MEM.get_quad(val_a)
                self.REG[Registers.RSP] = val_e
                self.REG[ra] = val_m
                self.PC = val_p
            elif opcode in (OPCode.IADDQ,
                            OPCode.ISUBQ,
                            OPCode.IANDQ,
                            OPCode.IXORQ):
                _, ra = bisect_byte(self.MEM.get_byte(self.PC + 1))
                val_p = self.PC + 10
                val_c = self.MEM.get_quad(self.PC + 2)
                val_a = self.REG[ra]
                val_e = self.op(ifun)(val_c, val_a)
                self.update_CC(ifun, val_c, val_a, val_e)
                self.REG[ra] = val_e
                self.PC = val_p
            else:
                raise BadInstruction
        except BadAddress:
            self.STAT = STATCode.ADR
        except BadInstruction:
            self.STAT = STATCode.INS

    def run(self):
        while self.STAT == STATCode.AOK:
            self.fetch_decode_execute()
            yield self.state


#  数据读取-------------------------------------------------------------
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np

def print_sum(x,y):
  z = x + y
  return(z)

print_sum(10,10)   # !!! 哇，也是蛮好用的吗！

obj1 = np.arange(16).reshape(2,8)

obj1.dtype

obj1.shape = (4,4)

obj1

obj1.shape = (8,2)

obj1
df = pd.read_csv("D:/RBookLearning/Causal_Inference/Python/5data/ex1.csv")
df

pd.read_table("D:/RBookLearning/Causal_Inference/Python/5data/ex1.csv",
              sep = ",")

pd.read_csv("D:/RBookLearning/Causal_Inference/Python/5data/ex2.csv",header = None)

pd.read_csv("D:/RBookLearning/Causal_Inference/Python/5data/ex2.csv",
            header = None,
            names = ["a","b","c","d","message"])

names = ["a","b","c","d","message"]
names

pd.read_csv("Python/5data/ex2.csv",names = names,index_col = "message")

pd.read_csv("Python/5data/csv_mindex.csv")

parsed = pd.read_csv("Python/5data/csv_mindex.csv",index_col = ["key1","key2"])
parsed

list(open("Python/5data/ex3.txt"))

pd.read_table("Python/5data/ex3.txt",sep = "\s+")

pd.read_csv("Python/5data/ex4.csv")

pd.read_csv("Python/5data/ex4.csv",skiprows = [0,2,3])

result = pd.read_csv("Python/5data/ex5.csv")
result

pd.isnull(result)

# 逐行读取文本文件

pd.read_csv("Python/5data/ex6.csv")

pd.read_csv("Python/5data/ex6.csv",nrows = 5)

result = pd.read_csv("Python/5data/ex6.csv",chunksize = 1000)
result

# 文件写出

# 读取Excel数据

data = pd.read_excel("Python/5data/ex7.xlsx")
data
data.to_excel("Python/5data/data.xlsx")

# 读取JSON数据


import keyboard

# Press PAGE UP then PAGE DOWN to type "foobar".
keyboard.add_hotkey('page up, page down', lambda: keyboard.write('foobar'))

keyboard.press_and_release('shift+s, space')

# Blocks until you press esc.
keyboard.wait('esc')

# Record events until 'esc' is pressed.
recorded = keyboard.record(until='esc')
# Then replay back at three times the speed.
keyboard.play(recorded, speed_factor=3)


# -*- coding: utf-8 -*-
"""
data preparation for modeling: cafe shops and renting shops information

"""

import pandas as pd
import numpy as np
import geopandas as gpd

# 讀取檔案
rent = pd.read_csv('./data/renting_shop.csv',encoding='utf-8')
point = pd.read_csv('./data/feat_points.csv',encoding='utf-8')
cafe = pd.read_csv('./data/cafe.csv',encoding='utf-8')

# 經緯度轉成 geometry資料型態 (原座標系統:  EPSG:4326 == WGS84)
g_rent = gpd.GeoDataFrame(rent, crs='epsg:4326', geometry=gpd.points_from_xy(rent.long, rent.lat))
g_total = gpd.GeoDataFrame(point, crs='epsg:4326', geometry=gpd.points_from_xy(point['long'], point['lat']))
g_cafe = gpd.GeoDataFrame(cafe, crs='epsg:4326', geometry=gpd.points_from_xy(cafe['long'], cafe['lat']))

# 轉換成台灣座標系統，位置會比較準確 (EPSG:3826 == TWD97)
g_rent = g_rent.to_crs('EPSG:3826')
g_total = g_total.to_crs('EPSG:3826')
g_cafe = g_cafe.to_crs('EPSG:3826')

#### 咖啡店資料
# 計算距離 (EPSG:3826 算出的距離為公尺)
distance = g_total.geometry.apply(lambda g: round(g_cafe.distance(g),0))

# 彙整距離表
cafe_distance = pd.DataFrame(distance[0])
cafe_distance.columns = ['distance']
cafe_distance['cafe_id'] = 0
cafe_distance['point_id'] = distance.index
cafe_distance['point_type'] = point['type2']

for i in range(1, distance.shape[1]):
    tmp = pd.DataFrame(distance[i])
    tmp.columns = ['distance']
    tmp['cafe_id'] = i
    tmp['point_id'] = distance.index
    tmp['point_type'] = point['type2']
    cafe_distance = pd.concat([cafe_distance, tmp], ignore_index=True)

cafe_distance = cafe_distance [['cafe_id', 'point_id', 'point_type', 'distance']]
cafe_distance = pd.read_csv('./data/cafe_distance.csv',encoding='utf-8') # 匯出檔案

# 建立特徵表
# 增加人口與收入資訊
cafe = cafe.loc[:,['area','rating','comment', 'brand', 'name', 'addr', 'filter', 'long', 'lat', 'village_rent','density_2021', 'ave_pop_growth', '2019_income']]

# 建立特徵表: 計算特定距離內的點數量
d_list = [500, 400, 300, 200, 100]
for d in d_list:
    types = ['cafe','small_cafe', 'breakfast', 'beverage','fastfood', 'supermarket','MRT', 'bus_stop', 'parking_space', 'CVS', 'school', 'bank', 'train']
    for i in range(len(types)):
        cafe_tmp = cafe_distance[cafe_distance['point_type'] == types[i]]
        for j in range(cafe.shape[0]):
            tmp = cafe_tmp[cafe_tmp['cafe_id']==j]
            cafe.loc[j,types[i]] = tmp[tmp['distance']<d].shape[0]
    cafe.to_csv(f'./data/cafe_features_{d}m.csv', index=False)

# 檢視特徵表
distance = [100,200,300,400,500]
for d in distance:
    features = pd.read_csv(f'./data/cafe_features_{d}m.csv',encoding='utf-8')
    print(features.shape)
    print(features.isnull().sum())

# 刪除空值
for d in distance:
    features = pd.read_csv(f'./data/cafe_features_{d}m.csv',encoding='utf-8')
    features.insert(0,'id',features.index)
    features.isnull().sum()
    features[features['density_2021'].isnull()]
    features = features.drop(features[features['density_2021'].isnull()].index, axis=0)
    features = features.drop(features[features['rating']==0].index, axis=0)
    features = features.reset_index(drop=True)
    features.to_csv(f'./data/cafe_features_{d}m.csv', index=False)

#### 出租店資料
# 距離計算
distance_rent = g_total.geometry.apply(lambda g: round(g_rent.distance(g),0))

# 彙整距離表
idx = distance_rent.index
types = point['type2']
rent_distance = pd.DataFrame(distance_rent.iloc[:,0])
rent_distance.columns = ['distance']
rent_distance['rent_id'] = 0
rent_distance['point_id'] = idx
rent_distance['point_type'] = types

for i in range(1, distance_rent.shape[1]):#
    tmp = pd.DataFrame(distance_rent.iloc[:,i])
    tmp.columns = ['distance']
    tmp['rent_id'] = i
    tmp['point_id'] = idx
    tmp['point_type'] = types
    rent_distance = pd.concat([rent_distance, tmp], ignore_index=True)

rent_distance = rent_distance[['rent_id', 'point_id', 'point_type', 'distance']]
rent_distance.to_csv('./data/rent_distance.csv', index=False)

# 建立特徵表
# 增加人口與收入資訊
rent = rent.loc[:,['id','city','address','filter','price','area','lat','long', 'density_2021','ave_pop_growth','2019_income']]
rent['avg_price'] = rent['price'] / rent['area']
rent['avg_price'] = rent['avg_price'].astype(int)
rent = rent[['id','city','address','filter','price','area','lat','long','avg_price','density_2021','ave_pop_growth','2019_income']]

# 建立特徵表: 計算特定距離內的點數量
d_list = [500, 400, 300, 200, 100]
for d in d_list:
    types = ['cafe','small_cafe', 'breakfast', 'beverage','fastfood', 'supermarket','MRT', 'bus_stop', 'parking_space', 'CVS', 'school', 'bank', 'train']
    for i in range(len(types)):
        rent_tmp = rent_distance[rent_distance['point_type'] == types[i]]
        for j in range(rent.shape[0]):
            tmp = rent_tmp[rent_tmp['rent_id']==j]
            rent.loc[j,types[i]] = tmp[tmp['distance']<d].shape[0]
    rent.to_csv(f'./data/rent_features_{d}m_new.csv', index=False)

# 檢視特徵表
distance = [100,200,300,400,500]
for d in distance:
    features = pd.read_csv(f'./data/rent_features_{d}m.csv',encoding='utf-8')
    print(features.shape)
    print(features.isnull().sum())

# 刪除空值
for d in distance:
    features = pd.read_csv(f'./data/rent_features_{d}m.csv',encoding='utf-8')
    features.insert(0,'id',features.index)
    features.isnull().sum()
    features[features['density_2021'].isnull()]
    features = features.drop(features[features['density_2021'].isnull()].index, axis=0)
    features = features.reset_index(drop=True)
    features.to_csv(f'./data/rent_features_{d}m.csv', index=False)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import json

from alipay.aop.api.constant.ParamConstants import *

class PosDiscountDetail(object):

    def __init__(self):
        self._discount_name = None
        self._discount_type = None
        self._ext_info = None
        self._mrt_discount = None
        self._rt_discount = None

    @property
    def discount_name(self):
        return self._discount_name

    @discount_name.setter
    def discount_name(self, value):
        self._discount_name = value
    @property
    def discount_type(self):
        return self._discount_type

    @discount_type.setter
    def discount_type(self, value):
        self._discount_type = value
    @property
    def ext_info(self):
        return self._ext_info

    @ext_info.setter
    def ext_info(self, value):
        self._ext_info = value
    @property
    def mrt_discount(self):
        return self._mrt_discount

    @mrt_discount.setter
    def mrt_discount(self, value):
        self._mrt_discount = value
    @property
    def rt_discount(self):
        return self._rt_discount

    @rt_discount.setter
    def rt_discount(self, value):
        self._rt_discount = value

    def to_alipay_dict(self):
        params = dict()
        if self.discount_name:
            if hasattr(self.discount_name, 'to_alipay_dict'):
                params['discount_name'] = self.discount_name.to_alipay_dict()
            else:
                params['discount_name'] = self.discount_name
        if self.discount_type:
            if hasattr(self.discount_type, 'to_alipay_dict'):
                params['discount_type'] = self.discount_type.to_alipay_dict()
            else:
                params['discount_type'] = self.discount_type
        if self.ext_info:
            if hasattr(self.ext_info, 'to_alipay_dict'):
                params['ext_info'] = self.ext_info.to_alipay_dict()
            else:
                params['ext_info'] = self.ext_info
        if self.mrt_discount:
            if hasattr(self.mrt_discount, 'to_alipay_dict'):
                params['mrt_discount'] = self.mrt_discount.to_alipay_dict()
            else:
                params['mrt_discount'] = self.mrt_discount
        if self.rt_discount:
            if hasattr(self.rt_discount, 'to_alipay_dict'):
                params['rt_discount'] = self.rt_discount.to_alipay_dict()
            else:
                params['rt_discount'] = self.rt_discount
        return params

    @staticmethod
    def from_alipay_dict(d):
        if not d:
            return None
        o = PosDiscountDetail()
        if 'discount_name' in d:
            o.discount_name = d['discount_name']
        if 'discount_type' in d:
            o.discount_type = d['discount_type']
        if 'ext_info' in d:
            o.ext_info = d['ext_info']
        if 'mrt_discount' in d:
            o.mrt_discount = d['mrt_discount']
        if 'rt_discount' in d:
            o.rt_discount = d['rt_discount']
        return o


import random
import sys
from pyspark import SparkContext
import binascii
from blackbox import BlackBox

sc = SparkContext.getOrCreate()

bx = BlackBox()

"""
class BlackBox:

    def ask(self, file, num):
        lines = open(file,'r').readlines()
        users = [0 for i in range(num)]
        for i in range(num):
            users[i] = lines[random.randint(0, len(lines) - 1)].rstrip("\n")
        return users

if __name__ == '__main__':
    bx = BlackBox()
"""

input_data = sys.argv[1]

stream_size = int(sys.argv[2])

num_of_asks = int(sys.argv[3])

output_file = sys.argv[4]

def check_prime(val):
	for num in range(2, int(val ** 1/2) + 1):
		if not val % num:
			return False
	return True

def compute(val):
	for num in range(val + 1, val + 10000):
		if check_prime(num): return num

def calculate_hash_values(no_of_hashf, length):

	a = random.sample(range(1, length), no_of_hashf)
	b = random.sample(range(1, length), no_of_hashf)

	p = [compute(i) for i in random.sample(range(length, length + 10000), no_of_hashf)]

	return {'a' : a, 'b' : b, 'p' : p}

no_of_hashf = 50
params = calculate_hash_values(no_of_hashf, 69997)

def myhashs(user_id):

	hashed_user = []
	user_int = int(binascii.hexlify(user_id.encode('utf8')),16)

	for i in range(no_of_hashf):
		hashed = (((params['a'][i]*user_int + params['b'][i])%params['p'][i])%69997)
		hashed_user.append(hashed)

	return hashed_user

def set_parameters():
    global filter_bit_Array
    global traversed
    global false_positives
    global true_negatives
    global next

    return filter_bit_Array, traversed, false_positives, true_negatives, nextptr

def calculate_fpr(false_positives, true_negatives):

    return (false_positives / (false_positives + true_negatives))

filter_bit_Array = [0] * 69997
traversed =set()
false_positives = 0.0
true_negatives = 0.0
nextptr = 0

def BloomFiltering(user_id):

    global filter_bit_Array
    global traversed
    global false_positives
    global true_negatives
    global nextptr

    for u in user_id:

        now = myhashs(u)
        ctr = 0
        for i in range(no_of_hashf):
            if (filter_bit_Array[now[i]] == 1):
                ctr += 1

        if u not in traversed:
            if (ctr == no_of_hashf):
                false_positives += 1.0
            else:
                true_negatives += 1.0

        for i in range(no_of_hashf):
            filter_bit_Array[now[i]] = 1

        traversed.add(u)

    FPR = calculate_fpr(false_positives, true_negatives)

    with open(output_file, "a+") as out:
        out.write(str(nextptr) + ',' + str(FPR) + '\n')

    nextptr += 1

def run_algorithm(num_of_asks, input_data, stream_size, bx):

    for i in range(0, num_of_asks):
        input_par = bx.ask(input_data, stream_size)
        BloomFiltering(input_par)

    return

with open(output_file, "w") as f:
	f.write('Time,FPR' + '\n')

for i in range(0, num_of_asks):
	item = bx.ask(input_data, stream_size)
	BloomFiltering(item)


import os
import glob
import regex as re
import sys
import argparse
from configparser import ConfigParser

def read_params(args):
    parser = argparse.ArgumentParser(description='''整理结果文件，1.0版本''')
    parser.add_argument('-c', '--config', dest='config', metavar='file', type=str, required=True,
                        help='标准复制文件的配置文档')
    parser.add_argument('-p', '--project_config', dest='project_config', metavar='file', type=str, required=True,
                        help='项目的配置文件，为了获取项目路径和分组名称')
    parser.add_argument('-g', '--group_path', dest='group_path', metavar='dir', type=str, required=True,
                        help='项目的配置文件，为了获取项目路径和分组名称')
    parser.add_argument('-o', '--output_dir', dest='output_dir', metavar='output_dir', type=str, required=True,
                        help='结果文件的输出路径（因为项目路径可能有多个，因此不能根据项目路径确定）')
    parser.add_argument('-os', '--out_suffix', dest='out_suffix', metavar='out_suffix', type=str, required=True,
                        help='设置新配置文件的后缀，输出文件为：分组名_后缀')
    args = parser.parse_args()
    params = vars(args)
    return params

def mkdir(*path):
    for sub_path in path:
        if not os.path.isdir(sub_path):
            os.system('mkdir -p %s' % sub_path)

def get_name(path):
    '''
    默认以.作为分隔符，最后一个点之前的为名称，最后一个点之后的为后缀
    '''
    basename = os.path.basename(path)
    dirname  = os.path.split(path)[0]
    filename = os.path.splitext(basename)[0]
    suffix   = os.path.splitext(basename)[1]
    return dirname,filename,suffix

def check_groups_exists(path, groups):
    '''
    检查文件夹下是否存在这几个分组文件
    '''
    unexist_groups = []
    for group in groups:
        if not glob.glob(path+'/'+group+'_group.list'):
            unexist_groups.append(group)

    if unexist_groups:
        raise Exception('%s 中缺少分组文件： %s' % (path, ', '.join(unexist_groups)))

def check_same_dir(dirs, save_suffix):
    '''
    判断多个文件夹是否一致，文件夹内对应后缀的所有文件名和大小都一致
    若都空，则返回None；若非空但文件内容和大小都一致则返回其一；若不一致，则返回不一致的所有文件夹（不包含空文件夹）
    '''
    # 统计每个文件夹中的符合条件的文件
    dir_dict = {}
    for dir in dirs:
        for suf in save_suffix:
            suf_files = [f for f in glob.glob(dir+'/*.'+suf) if os.path.isfile(f)]
            if suf_files:
                if dir not in dir_dict.keys():
                    dir_dict[dir] = suf_files
                else:
                    dir_dict[dir] += suf_files

    # 若只有一个文件夹，直接返回；若一个都没有，返回与配置路径最相似的文件夹；多个文件夹，判断文件个数和文件大小是否都一致，一致返回其中之一，不一致，返回所有路径
    if len(dir_dict.keys()) == 1:
        return list(dir_dict.keys()), []
    elif len(dir_dict.keys()) == 0:
        # 为了避免路径中存在//还有最后没有/的情况，重新对路径格式进行调整
        dirs = ['/'+'/'.join([d for d in dir.split('/') if d])+'/' for dir in dirs]
        for i in range(-1, -min([len(dir.split('/'))-2 for dir in dirs]), -1):
            similar_dirs = [dir for dir in dirs if os.path.exists('/'.join(dir.split('/')[:i]))]
            if similar_dirs:
                return [], similar_dirs

        return [], similar_dirs
    else:
        files = []
        for dir in dir_dict.keys():
            files += [os.path.basename(f) for f in dir_dict[dir]]
        files = list(set(files))
        if set([len(files) for files in dir_dict.values()]) == 1 and len(files) == len(list(dir_dict.values())[0]):
            for file in files:
                if set([os.path.getsize(d+'/'+file) for d in dir_dict.keys()]) != 1:
                    return list(dir_dict.keys()), []
            return [list(dir_dict.keys())[0]], []
        else:
            return list(dir_dict.keys()), []

def check_if_sample_enough(dirs):
    '''
    通过这层或者上一层是否存在样品不足的日志文件来判断此分析是否是样品不足
    足返回True，不足返回False
    '''
    for dir in dirs:
        if glob.glob(dir+'/Sample_not_enough.log') or glob.glob('/'+'/'.join([d for d in dir.split('/') if d][:-1])+'/Sample_not_enough.log') or glob.glob('/'+'/'.join([d for d in dir.split('/') if d][:-2])+'/Sample_not_enough.log'):
            return False
    return True

def check_if_group_enough(dirs):
    '''
    通过这层或者上一层是否存在分组不足的日志文件来判断此分析是否是分组不足
    足返回True，不足返回False
    '''
    for dir in dirs:
        if glob.glob(dir+'/group_num.log') or glob.glob('/'+'/'.join([d for d in dir.split('/') if d][:-1])+'/group_num.log') or glob.glob('/'+'/'.join([d for d in dir.split('/') if d][:-2])+'/group_num.log'):
            return False
    return True

def is_file_exists(dirs, file):
    '''
    判断多个文件中是否含有该文件（文件为含通配符的名称或者固定文件名）
    若不存在，则返回与配置路径最相近的文件夹名称
    '''
    if '*' in file:
        file_dirs = []
        for dir in dirs:
            for f in glob.glob(dir+'/'+file):
                if os.path.isfile(f):
                    file_dirs.append(dir+'/'+file)
                    break
    else:
        file_dirs = [dir+'/'+file for dir in dirs if glob.glob(dir+'/'+file) if os.path.isfile(dir+'/'+file)]
    if file_dirs:
        return file_dirs, []
    else:
        # 为了避免路径中存在//还有最后没有/的情况，重新对路径格式进行调整
        dirs = ['/'+'/'.join([d for d in (dir+'/'+file).split('/') if d][:-1])+'/' for dir in dirs]
        for i in range(-1, -min([len(dir.split('/'))-2 for dir in dirs]), -1):
            similar_dirs = [dir for dir in dirs if os.path.exists('/'.join(dir.split('/')[:i]))]
            if similar_dirs:
                return [], similar_dirs

def check_same_file(dirs, file):
    '''
    检查多个文件夹中是否含有一样名称的文件，若有，检查大小是否一致，若一致，则认为一样名称的两个文件是同一个文件
    例如：file为*.txt时找到aaa/b.txt bbb/b.txt ccc/b.txt aaa/a.txt，对于b.txt，若大小一致，则认为是一个文件，若不一致则认为存在多个文件
    return 多个一致文件的全路径
    '''
    files = []
    for dir in dirs:
        files += [f for f in glob.glob(dir + '/' + file) if os.path.isfile(f)]

    files_dict = {}
    for file in files:
        basename = os.path.basename(file)
        if basename not in files_dict.keys():
            files_dict[basename] = [file]
        else:
            files_dict[basename].append(file)

    same_files = {}
    for file in files_dict.keys():
        if len(files_dict[file]) > 1 and len(set([os.path.getsize(f) for f in files_dict[file]])) != 1:
            same_files[file] = files_dict[file]
    return same_files

def list_all_files(dir, standard_dirs, suffixes, files=[]):
    '''
    列举文件夹中所有需要的文件
    '''
    all_files = os.listdir(dir)
    # standard_dirs为了去除第一层的文件夹，第二层以后的迭代就不需要判断了
    all_files = [dir+'/'+file for file in all_files if file not in standard_dirs]
    for file in all_files:
        if os.path.isfile(file) and any([file.endswith(suf) for suf in suffixes]):
            files.append(file)
        elif os.path.isdir(file):
            files = list_all_files(file, [], suffixes, files)
    return files

if __name__ == '__main__':
    params = read_params(sys.argv)

    config_file  = params['config']
    group_path   = params['group_path']
    output_dir   = params['output_dir']
    out_suffix = params['out_suffix']

    if os.path.isdir(output_dir):
        os.system('rm -rf %s/*' % output_dir)
    else:
        mkdir(output_dir)

    project_config = ConfigParser()
    project_config.read(params['project_config'])
    # 获取一个或多个工作路径的名称
    work_dirs = re.split('\s+|\t+|,', project_config.get('html','work_dir'))
    # 对dir格式进行规范，统一改成/aaa/bbb/ccc格式
    work_dirs = ['/'+'/'.join([d for d in dir.split('/') if d]) for dir in work_dirs if dir]

    # 获取分组名称
    groups = re.split('\s+|\t+|,', project_config.get('html','group'))
    groups = [os.path.splitext(g)[0] for g in groups if g]  # 需要确定是写分组名称还是文件
    check_groups_exists(group_path, groups)

    # 获取组装方法
    assembly_method = project_config.get('param','assembly_method')

    # 获取project信息
    project_num = project_config.get('project','project_num')
    project_name = project_config.get('project','project_name')

    # 获取batch最后一次的名称，用来复制最后一次raw data和qc的统计结果
    batch_list = re.split('\s+|\t+|,', project_config.get('html','batch_list'))
    batch_list = [b for b in batch_list if b]
    if len(batch_list) > 1:
        sys.stderr.write('发现多个batch_list文件：%s，请按照时间顺序合并成一个文件并重新配置%s文件！\n' % ('、'.join(batch_list), params['project_config']))
        sys.exit()

    with open(batch_list[0], 'r') as batch:
        for line in batch:
            tabs = line.strip().split('\t')
            tabs = [t.strip() for t in tabs if t]
            if len(tabs) > 1:
                batch_name = tabs[0]
    print('     final batch name is ： %s' % batch_name)

    for group in groups:
        with open(group_path+'/'+group+'_group.list', 'r') as gf:
            example_sample = gf.readline().split('\t')[0]

        with open(config_file, 'r') as conf, open(output_dir+'/'+group+'_'+out_suffix, 'w') as new_conf:
            # 在新的配置文件前先输出项目名称和项目编号，可以根据这个得到创建的文件夹的名称。
            new_conf.write('# 结果文件的输出路径\n')
            new_conf.write('%s\t%s\n' % ('output_dir', output_dir))
            new_conf.write('# 项目编号\n')
            new_conf.write('%s\t%s\n' % ('project_num', project_num))
            new_conf.write('# 项目名称\n')
            new_conf.write('%s\t%s\n' % ('project_name', project_name))
            new_conf.write('# 参考的分组文件\n')
            new_conf.write('%s\t%s\n' % ('group', group))

            for line in conf:
                if line.strip().startswith('#') or not line.strip():
                    new_conf.write(line)
                    continue

                tabs = line.strip().split('\t')

                if '#@' in tabs[1]:
                    for i, j in zip(['#@batch', '#@sample', '#@assembly', '#@group'], [batch_name, example_sample, assembly_method, group]):
                        tabs[1] = re.sub(i, j, tabs[1])

                # 判断样品够不够，其实只有涉及group的时候才会存在样品够不够，但是其他基本上也不会存在sample_not_enough.log文件，因此我们可以简单的对所有的情况统一判断
                if check_if_sample_enough([dir+'/'+os.path.dirname(tabs[1]) for dir in work_dirs]):
                    # 样品足：
                    #    1、文件是否存在 （1）不存在 （2）存在一个 （3）存在多个
                    if check_if_group_enough([dir+'/'+os.path.dirname(tabs[1]) for dir in work_dirs]):
                        # 分组够
                        file_dirs, similar_dirs = is_file_exists(work_dirs, tabs[1])
                        if file_dirs:
                            # 文件存在
                            if len(file_dirs) == 1 or len(set([os.path.getsize(f) for f in file_dirs])) == 1:
                                # 文件只有一个，或多个文件大小一致，选取其中之一
                                new_conf.write('%s\t%s\t%s\n' % (tabs[0], file_dirs[0], tabs[2]))
                            else:
                                new_conf.write('\n#################################################################################################################################################\n')
                                new_conf.write('# warning : %s文件有%d个，且大小不一，请检查原因并修改下面配置！（默认选择第一个）\n' % (re.sub('#@batch', batch_name, tabs[1]), len(file_dirs)))
                                for f in file_dirs:
                                    new_conf.write('%s\t%s\t%s\n' % (tabs[0], f, tabs[2]))
                                new_conf.write('#################################################################################################################################################\n\n')
                        else:
                            print(tabs)
                            # 文件不存在
                            new_conf.write('\n#################################################################################################################################################\n')
                            new_conf.write('# warning : %s文件不存在，请检查原因并修改下面配置！\n' % tabs[1])
                            for f in similar_dirs:
                                new_conf.write('%s\t%s\t%s\n' % (tabs[0], f+'/'+os.path.basename(tabs[1]), tabs[2]))
                            new_conf.write('#################################################################################################################################################\n\n')
                    else:
                        # 分组不够
                        new_conf.write('%s\t%s\tgroup_not_enough.log\n' % (tabs[0], tabs[1]))

                else:
                    # 样品不足
                    new_conf.write('%s\t%s\tsample_not_enough.log\n' % (tabs[0], tabs[1]))


import torch, time, gc

# Timing utilities
timer_dict = {}

def start(name='default'):
    global timer_dict
    gc.collect()
    torch.cuda.empty_cache()
    torch.cuda.reset_max_memory_allocated()
    torch.cuda.synchronize()
    start_time = time.time()
    start_mem = torch.cuda.max_memory_allocated()
    timer_dict[name] = {
        'time': start_time,
        'mem':start_mem,
    }

def end(name='default'):
    global timer_dict
    torch.cuda.synchronize()
    end_time = time.time()
    end_mem = torch.cuda.max_memory_allocated()

    elapsed = end_time - timer_dict[name]['time']
    memory_allocated = end_mem - timer_dict[name]['mem']

    return elapsed, memory_allocated