#! /usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Feb 26 17:12:48 2019

@author: ben
"""
import os
os.environ["MKL_NUM_THREADS"]="1"
os.environ["OPENBLAS_NUM_THREADS"]="1"

import numpy as np

from LSsurf.smooth_xytb_fit import smooth_xytb_fit
#from LSsurf.assign_firn_correction import assign_firn_correction
from surfaceChange.reread_data_from_fits import reread_data_from_fits
import pointCollection as pc
from CS2_fit.read_CS2_data import read_CS2_data
from pyTMD import compute_tide_corrections

import h5py
import sys
import glob
import re

def get_SRS_proj4(hemisphere):
    if hemisphere==1:
        return '+proj=stere +lat_0=90 +lat_ts=70 +lon_0=-45 +k=1 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs'
    else:
        return '+proj=stere +lat_0=-90 +lat_ts=-71 +lon_0=0 +k=1 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs'

def save_fit_to_file(S,  filename, sensor_dict=None, dzdt_lags=None):
    if os.path.isfile(filename):
        os.remove(filename)
    with h5py.File(filename,'w') as h5f:
        h5f.create_group('/data')
        for key in S['data'].fields:
            h5f.create_dataset('/data/'+key, data=getattr(S['data'], key))
        h5f.create_group('/meta')
        h5f.create_group('/meta/timing')
        for key in S['timing']:
            h5f['/meta/timing/'].attrs[key]=S['timing'][key]
        h5f.create_group('/RMS')
        for key in S['RMS']:
           h5f.create_dataset('/RMS/'+key, data=S['RMS'][key])
        h5f.create_group('E_RMS')
        for key in S['E_RMS']:
            h5f.create_dataset('E_RMS/'+key, data=S['E_RMS'][key])
        for key in S['m']['bias']:
            h5f.create_dataset('/bias/'+key, data=S['m']['bias'][key])
        if 'slope_bias' in S['m']:
            sensors=np.array(list(S['m']['slope_bias'].keys()))
            h5f.create_dataset('/slope_bias/sensors', data=sensors)
            x_slope=[S['m']['slope_bias'][key]['slope_x'] for key in sensors]
            y_slope=[S['m']['slope_bias'][key]['slope_y'] for key in sensors]
            h5f.create_dataset('/slope_bias/x_slope', data=np.array(x_slope))
            h5f.create_dataset('/slope_bias/y_slope', data=np.array(y_slope))
        if sensor_dict is not None:
            h5f.create_group('meta/sensors')
            for key in sensor_dict:
                h5f['/meta/sensors'].attrs['sensor_%d' % key]=sensor_dict[key]

    for key , ds in S['m'].items():
        if isinstance(ds, pc.grid.data):
            ds.to_h5(filename, group=key)
    return

def save_errors_to_file( S, filename, sensor_dict=None):
    with h5py.File(filename,'r+') as h5f:
        h5f.create_group('/dz/sigma/')
        for ii, name in enumerate(['y','x','t']):
            h5f.create_dataset('/dz/sigma/'+name, data=S['grids']['dz'].ctrs[ii])
        h5f.create_dataset('/dz/sigma/dz', data=S['E']['dz'])
        h5f.create_group('/z0/sigma')
        for ii, name in enumerate(['y','x']):
            h5f.create_dataset('/z0/sigma/'+name, data=S['grids']['z0'].ctrs[ii])
        h5f.create_dataset('/z0/sigma/z0', data=S['E']['z0'])
        for key in S['E']['bias']:
            h5f.create_dataset('/bias/sigma/'+key, data=S['E']['bias'][key])
        if 'slope_bias' in S['E']:
            sensors=np.array(list(S['E']['slope_bias'].keys()))
            h5f.create_dataset('/slope_bias/sigma/sensors', data=sensors)
            x_slope=[S['E']['slope_bias'][key]['slope_x'] for key in sensors]
            y_slope=[S['E']['slope_bias'][key]['slope_y'] for key in sensors]
            h5f.create_dataset('/slope_bias/sigma/x_slope', data=np.array(x_slope))
            h5f.create_dataset('/slope_bias//sigma/y_slope', data=np.array(y_slope))
    return

def fit_CS2(xy0, Wxy=4e4, E_RMS={}, t_span=[2003, 2020], spacing={'z0':2.5e2, 'dz':5.e2, 'dt':0.5},  \
            hemisphere=1, reference_epoch=None, reread_dirs=None, max_iterations=5, N_subset=None, Edit_only=False, \
            avg_scales=None, dzdt_lags=None,\
            sensor_dict={}, out_name=None, replace=False, DOPLOT=False, spring_only=False, \
            geoid_file=None, mask_file=None, calc_tide=False,\
            bias_nsigma_edit=None, bias_nsigma_iteration=3,\
            calc_error_file=None, data_file=None, restart_TSE=False,\
            DEM_file=None, DEM_tol=None):
    """
        Wrapper for smooth_xytb_fit that can find data and set the appropriate parameters
    """
    print("fit_CS2: working on %s" % out_name, flush=True)
    baseline_code={'C':2, 'D':3}
    # temporary:
    if hemisphere==-1:
        srs_proj4= '+proj=stere +lat_0=-90 +lat_ts=-71 +lon_0=0 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs'
        #index_files={ 'D':{'POCA': glob.glob('/Volumes/ice3/ben/CS2_data/AA/retrack/2*/*/index_POCA.h5'),
        #                  'swath':glob.glob('/Volumes/ice3/ben/CS2_data/AA/retrack/2*/*/index_SW.h5')}}
        index_files={ 'D':{'POCA': glob.glob('/Volumes/ice3/ben/CS2_data/AA/tiles/2*/*/POCA/GeoIndex.h5'),
                          'swath':glob.glob('/Volumes/ice3/ben/CS2_data/AA/tiles/2*/*/sw/GeoIndex.h5')}}

    if hemisphere==1:
        srs_proj4= '+proj=stere +lat_0=90 +lat_ts=70 +lon_0=-45 +x_0=0 +y_0=0 +datum=WGS84 +units=m +no_defs'
        index_files={'C':{'POCA':glob.glob('/Volumes/ice2/ben/CS2_data/GL/retrack_BC/2*/index_POCA.h5'),
                          'swath':glob.glob('/Volumes/ice2/ben/CS2_data/GL/retrack_BC/2*/index_SW.h5')},
                    'D':{'POCA':glob.glob('/Volumes/ice2/ben/CS2_data/GL/retrack/2*/index_POCA.h5'),
                          'swath':glob.glob('/Volumes/ice2/ben/CS2_data/GL/retrack/2*/index_SW.h5')}
                    }

    compute_E=False
    # set defaults for E_RMS, then update with input parameters
    E_RMS0={'d2z0_dx2':200000./3000/3000, 'd3z_dx2dt':1000./3000/3000, 'd2z_dxdt':6000/3000, 'd2z_dt2':5000}
    E_RMS0.update(E_RMS)

    W={'x':Wxy, 'y':Wxy,'t':np.diff(t_span)}
    ctr={'x':xy0[0], 'y':xy0[1], 't':np.mean(t_span)}
    if out_name is not None:
        try:
            out_name=out_name %(xy0[0]/1000, xy0[1]/1000)
        except:
            pass

    if calc_error_file is not None:
        # get xy0 from the filename
        re_match=re.compile('E(.*)_N(.*).h5').search(calc_error_file)
        xy0=[float(re_match.group(ii))*1000 for ii in [1, 2]]
        data, sensor_dict=reread_data_from_fits(xy0, Wxy, [os.path.dirname(calc_error_file)])
        compute_E=True
        max_iterations=0
        N_subset=None
    elif data_file is not None:
        data=pc.data().from_h5(data_file, group='data')
        ###testing
        data.sigma_corr = 0.05 + 2*(data.swath>0.5)
    elif reread_dirs is None:
        data=[]
        for baseline in index_files.keys():
            data += [read_CS2_data(xy0, Wxy, index_files[baseline],
                                  apply_filters=True, DEM_file=DEM_file,
                                  dem_tol=50)]
            #'sigma':np.sqrt(0.2**2+0.5*(data.swath>0.5))
            ###testing
            data[-1].assign({'z':data[-1].h, \
                  'sigma_corr': 0.05 + 2*(data[-1].swath>0.5),
                  'baseline': baseline_code[baseline]+np.zeros_like(data[-1].swath)})
        if len(data) > 1:
            data[0].index(~np.in1d(data[0].abs_orbit, np.unique(data[1].abs_orbit)))
            data = pc.data().from_list(data)
        else:
            data=data[0]
        D_PC=data[data.swath==0]
        D_sw=data[data.swath==1]

        orb_dict=pc.bin_rows(np.c_[D_sw.abs_orbit])
        D_sw_sub=pc.data().from_list([D_sw[orb_dict[key]].blockmedian(400) for key in orb_dict])
        data=pc.data().from_list([D_PC, D_sw_sub])

        data.index((data.time>t_span[0]) & (data.time < t_span[1]))
        if calc_tide:
            if hemisphere==-1:
                # note: compute_tide_correction returns a masked array.
                # for now, we're going to convert it to an array by stripping the mask
                temp=compute_tide_corrections(\
                    data.x,data.y,(data.time-2000)*365.25*24*3600,
                    DIRECTORY='/Volumes/ice3/tyler/tide_models',MODEL='CATS2008',
                    EPOCH=(2000,1,1,0,0,0), TYPE='drift', TIME='utc')
                temp[temp.mask]=np.NaN
                data.assign({'tide':np.array(temp)})
                data.z[np.isfinite(data.tide)] -= data.tide[np.isfinite(data.tide)]
    else:
        data, sensor_dict = reread_data_from_fits(xy0, Wxy, reread_dirs, template='E%d_N%d.h5')
        N_subset=None

    if restart_TSE:
        if 'three_sigma_edit' in data.fields:
            data.three_sigma_edit=np.ones_like(data.x, dtype=bool)
        else:
            data.assign({'three_sigma_edit':np.ones_like(data.x, dtype=bool)})

    if reference_epoch is None:
        reference_epoch=np.ceil(len(np.arange(t_span[0], t_span[1], spacing['dt']))/2).astype(int)

    for field in data.fields:
        setattr(data, field, getattr(data, field).astype(np.float64))

    if DEM_file is not None:
        DEM=pc.grid.data().from_geotif(DEM_file, \
              bounds=[[xy0[0]-W['x']/1.8, xy0[0]+W['x']/1.8],\
                      [xy0[1]-W['y']/1.8, xy0[1]+W['y']/1.8]])
        data.assign({'DEM':DEM.interp(data.x, data.y)})

    # want less than 900K points
    if data.size > 9e5:
        D_sw=data[data.swath==1]
        D_poca=data[data.swath==0]
        D_sw.index(np.arange(0, D_sw.size, D_sw.size/(9e5-D_poca.size)).astype(int))
        data=pc.data().from_list([D_sw, D_poca])

    # run the fit
    KWargs={'data':data,
     'W': W,
     'ctr':ctr,
     'spacing':spacing,
     'srs_proj4':srs_proj4,
     'mask_file':mask_file,
     'E_RMS':E_RMS0,
     'E_RMS_d2x_PS_bias':1.e-7,
     'E_RMS_PS_bias':1,
     'compute_E':compute_E,
     'verbose': True,
     'mask_scale':{0:10, 1:1},
     'max_iterations':max_iterations,
     'N_subset':N_subset,
     'bias_params':['abs_orbit','swath'],
     'DEM_tol':DEM_tol,
     'avg_scales':avg_scales,
     'bias_filter': lambda D:D[D.swath>0.5],
     'bias_nsigma_edit':bias_nsigma_edit,
     'bias_nsigma_iteration':bias_nsigma_iteration
     }
    ### testing
    KWargs['bias_filter']=None
    S=smooth_xytb_fit(**KWargs)

    if out_name is not None:
        if calc_error_file is None:
            save_fit_to_file(S, out_name, sensor_dict, dzdt_lags=S['dzdt_lags'])
        else:
            save_errors_to_file(S, out_name, sensor_dict)
    print("fit_CS2: done with %s" % out_name, flush=True)
    return S, data, sensor_dict

def main(argv):
    # account for a bug in argparse that misinterprets negative agruents
    for i, arg in enumerate(argv):
        if (arg[0] == '-') and arg[1].isdigit(): argv[i] = ' ' + arg

    import argparse
    parser=argparse.ArgumentParser(description="function to fit icebridge data with a smooth elevation-change model", \
                                   fromfile_prefix_chars="@")
    parser.add_argument('xy0', type=float, nargs=2, help="fit center location")
    parser.add_argument('--Width','-W',  type=float, help="fit width")
    parser.add_argument('--time_span','-t', type=str, help="time span, first year,last year AD (comma separated, no spaces)")
    parser.add_argument('--grid_spacing','-g', type=str, help='grid spacing:DEM (meters),dh maps xy (meters),dh_maps time (years): comma-separated, no spaces', default='250.,4000.,1.')
    parser.add_argument('--Hemisphere','-H', type=int, default=1, help='hemisphere: -1=Antarctica, 1=Greenland')
    parser.add_argument('--base_directory','-b', type=str, help='base directory')
    parser.add_argument('--out_name', '-o', type=str, help="output file name")
    parser.add_argument('--centers', action="store_true")
    parser.add_argument('--edges', action="store_true")
    parser.add_argument('--corners', action="store_true")
    parser.add_argument('--E_d2zdt2', type=float, default=5000)
    parser.add_argument('--E_d2z0dx2', type=float, default=0.02)
    parser.add_argument('--E_d3zdx2dt', type=float, default=0.0003)
    parser.add_argument('--data_gap_scale', type=float,  default=2500)
    parser.add_argument('--max_iterations', type=int, default=8)
    parser.add_argument('--mask_file', type=str)
    parser.add_argument('--geoid_file', type=str)
    parser.add_argument('--DEM_file', type=str)
    parser.add_argument('--DEM_tol', type=float, default=10)
    parser.add_argument('--calc_error_file','-c', type=str)
    parser.add_argument('--data_file', type=str)
    parser.add_argument('--restart_TSE', action='store_true', help='if reloading data, restart the three-sigma editing')
    parser.add_argument('--dzdt_lags', type=str, default='1,4', help='lags for which to calculate dz/dt, comma-separated list, no spaces')
    parser.add_argument('--avg_scales', type=str, help='scales at which to report average errors, comma-separated list, no spaces')
    parser.add_argument('--bias_nsigma_edit', type=int, default=6, help='edit points whose estimated bias is more than this value times the expected')
    parser.add_argument('--bias_nsigma_iteration', type=int, default=6, help='apply bias_nsigma_edit after this iteration')
    parser.add_argument('--calc_tide', action='store_true')

    args=parser.parse_args()

    if args.avg_scales is not None:
        args.avg_scales = [np.int(temp) for temp in args.avg_scales.split(',')]
    args.dzdt_lags = [np.int(temp) for temp in args.dzdt_lags.split(',')]

    args.grid_spacing = [np.float(temp) for temp in args.grid_spacing.split(',')]
    args.time_span = [np.float(temp) for temp in args.time_span.split(',')]

    spacing={'z0':args.grid_spacing[0], 'dz':args.grid_spacing[1], 'dt':args.grid_spacing[2]}
    E_RMS={'d2z0_dx2':args.E_d2z0dx2, 'd3z_dx2dt':args.E_d3zdx2dt, 'd2z_dxdt':args.E_d3zdx2dt*args.data_gap_scale,  'd2z_dt2':args.E_d2zdt2}

    reread_dirs=None
    if args.centers:
        dest_dir = args.base_directory+'/centers'
    if args.edges or args.corners:
        reread_dirs=[args.base_directory+'/centers']
        dest_dir = args.base_directory+'/edges'
    if args.corners:
        reread_dirs += [args.base_directory+'/edges']
        dest_dir = args.base_directory+'/corners'

    if args.calc_error_file is not None:
        dest_dir=os.path.dirname(args.calc_error_file)
        # get xy0 from the filename
        re_match=re.compile('E(.*)_N(.*).h5').search(args.calc_error_file)
        args.xy0=[float(re_match.group(ii))*1000 for ii in [1, 2]]

    if not os.path.isdir(args.base_directory):
        os.mkdir(args.base_directory)
    if not os.path.isdir(dest_dir):
        os.mkdir(dest_dir)

    if args.out_name is None:
        args.out_name=dest_dir + '/E%d_N%d.h5' % (args.xy0[0]/1e3, args.xy0[1]/1e3)

    if args.DEM_tol is not None and args.DEM_file is None:
        print("Must specify a DEM if DEM_tol is specified")
        sys.exit(1)

    fit_CS2(args.xy0, Wxy=args.Width, E_RMS=E_RMS, t_span=args.time_span, spacing=spacing, \
            hemisphere=args.Hemisphere, reread_dirs=reread_dirs, \
            out_name=args.out_name, DOPLOT=False, \
            mask_file=args.mask_file, geoid_file=args.geoid_file, \
                DEM_file=args.DEM_file, DEM_tol=args.DEM_tol, \
                calc_error_file=args.calc_error_file,
                data_file=args.data_file,
                restart_TSE=args.restart_TSE,
                max_iterations=args.max_iterations,
                avg_scales=args.avg_scales,
                dzdt_lags=args.dzdt_lags,
                bias_nsigma_edit=args.bias_nsigma_edit,
                bias_nsigma_iteration=args.bias_nsigma_iteration,
                calc_tide=args.calc_tide)

if __name__=='__main__':
    main(sys.argv)

# -440000. -560000. -W 40000 -t 2010,2019 -g 500,2000,1 -H -1 -b /Volumes/ice2/ben/CS2_fit/AA -o test1_halfdata.h5 --centers

# for Antarctica
#  -350000 -460000 --centers @/Volumes/ice2/ben/CS2_fit/AA/default_args_AA.txt

# for Greenland:
#  600000 -1240000 --centers  @/Volumes/ice2/ben/CS2_fit/GL/default_args_GL.txt


# -*- coding: utf-8 -*-
"""
Created on Mon Mar 25 23:15:02 2019

@author: ljp
"""

import numpy as np
from scipy.stats import multivariate_normal
import random
from scipy.ndimage import convolve

##### Function that creates plans from parameters #####

def unet_create_plans(number_of_plans, plan_size, num_mean, num_std, info_neuron, coord_threshold):

    """Function that creates stack of plans from parameters.

    %%%%% INPUTS %%%%%
    - number_of_plans: final number of stacked plans in output (integer)
    - plan_size: size of each plan in pixels (vector of 2 integers)
    - num_mean: mean number of neurons per plan (integer)
    - num_std: standard deviation of number of neurons per plan (integer)
    - info_neuron: series of 8 floats describing each neuron, comprising:
            - neu_mean: mean intensity
            - neu_std: standard deviation of intensity
            - cor_mean: mean covariance (inclinaison)
            - cor_std: standard deviation of covariance
            - xvar_mean: mean variance in the x-axis direction
            - xvar_std: standard deviation of the variance in the x-axis direction
            - yvar_mean: mean variance in the y-axis direction
            - yvar_std: standard deviation of the variance in the y-axis direction

    %%%%% OUTPUTS %%%%%
    - plans_temp: plans with neurons stacked along first dimension ((number_of_plans, plan_size[0], plan_size[1]) matrix)
    - coord_temp: coordinates of all neurons, with associated plan ((number_of_plans, plan_size[0], plan_size[1]) matrix)
    """

    # Retriving information on neurons
    neu_mean, neu_std, cor_mean, cor_std, xvar_mean, xvar_std, yvar_mean, yvar_std = info_neuron

    # Creating initial set of plans
    plans_temp = np.zeros((number_of_plans, plan_size[0], plan_size[1]))
    # Defining coordinates vector
    coord_temp = np.zeros((number_of_plans, plan_size[0], plan_size[1]))

    # Creating Xfield and Yfield vectors
    Xfield, Yfield = np.mgrid[-6:7, -6:7]
    Xfield = Xfield.flatten()[:, np.newaxis]
    Yfield = Yfield.flatten()[:, np.newaxis]

    # Defining number of neurons per plan
    numfin = np.round_(num_mean + num_std*np.random.randn(number_of_plans))
    # Loop on each plan
    for i in range(number_of_plans):
        # Loop on each potential neuron in plan i
        for j in range(int(numfin[i])):
            # Finding available space for neuron
            finding = -1
            numround = 0
            numlim = 50
            while finding == -1 and numround < numlim:
                randx = int(np.round_(np.random.rand() * (plan_size[0] - 15)) + 7)
                randy = int(np.round_(np.random.rand() * (plan_size[1] - 15)) + 7)
                anafield = plans_temp[i, (randx-7):(randx+6), (randy-7):(randy+6)]
                if np.count_nonzero(anafield) == 0:
                    break
                numround += 1
            if numround == numlim:
                continue
            # Defining neuron's information
            matcorval = cor_mean + cor_std * np.random.randn()
            matrixstd = np.array([[xvar_mean + xvar_std*np.random.randn(), matcorval],
                                 [matcorval, yvar_mean + yvar_std*np.random.randn()]])
            if np.linalg.det(matrixstd) < 0:
                continue
            # Making matrix with neuron's information
            numval = (neu_mean + neu_std*np.random.randn()) * np.sqrt((2*np.pi)**2 * np.linalg.det(matrixstd))
            mvn = multivariate_normal(mean = np.array([0, 0]), cov = matrixstd)
            valexp = numval * mvn.pdf(np.hstack((Xfield, Yfield)))
            if numval <= 0:
                continue
            # Replacing in final plan
            plan_temp = valexp.reshape(13, 13)
            plans_temp[i, (randx-7):(randx+6), (randy-7):(randy+6)] = plan_temp
            # Adding neuron detection in coord_temp
            coord_temp[i, (randx-7):(randx+6), (randy-7):(randy+6)] = 1 * (plan_temp > neu_mean/coord_threshold)

    return plans_temp, coord_temp

##### Function that stacks plans from create_plans function #####

def unet_stack_plans(nstacks, plans_dataset, coord_dataset, cropped_size, stack_mean, stack_std, noise_mean, noise_std):

    """Function that randomly stack plans given as inputs.

    %%%%% INPUTS %%%%%
    - nstacks: final number of stacked plans (integer)
    - plans_dataset: plans dataset obtained from unet_create_plans (3d matrix)
    - coord_dataset: coordinates dataset obtained from unet_create_plans (3d matrix)
    - cropped_size: final size of plans after cropped (integer)
    - stack_mean: mean number of plans stacked for one final plan (integer)
    - stack_std: standard deviation of number of plans stacked for one final plan (integer)
    - noise_mean: mean of noise plan added to stacked plans (float)
    - noise_std: standard deviation of noise plan added to stacked plans (float)

    %%%%% OUTPUTS %%%%%
    - plans_stack: final plans stacked along first dimension ((nstacks, cropped_size, cropped_size) matrix)
    - coord_stack: coordinates of all neurons, with associated final plan ((number of neurons, 3) matrix)
    """

    # First defining final matrices
    plans_stack = np.zeros((nstacks, cropped_size, cropped_size))
    coord_stack = np.zeros((nstacks, cropped_size, cropped_size))

    # Number of stacks in plans_dataset
    nstacks_old = plans_dataset.shape[0]

    # Parameters for cropping
    xcrop = plans_dataset.shape[1] - cropped_size
    ycrop = plans_dataset.shape[2] - cropped_size

    # Filter for noise, will be convolved to have continuous noise
    filter_shape = 5
    filter_noise = np.zeros((filter_shape, filter_shape))
    for i in range(filter_shape):
        for j in range(filter_shape):
            radius_temp = np.sqrt((i-(filter_shape-1)/2)**2 + (j-(filter_shape-1)/2)**2)
            if radius_temp <= 1:
                filter_noise[i, j] = 1
            elif radius_temp <= 2:
                filter_noise[i, j] = 0.5

    # Launching loop
    for i in range(nstacks):
        # Selecting number of stacks
        stack_f = int(np.round_(stack_mean + stack_std*np.random.randn()))
        if stack_f <= 0:
            stack_f = 1
        # Creating temporary plan
        plans_temp = np.zeros((stack_f+1, cropped_size, cropped_size))
        # Creating a temporaty coordinates matrix
        coords_temp = np.zeros((stack_f, cropped_size, cropped_size))
        # Chosing plans to use
        plan_use = np.floor(nstacks_old*np.random.rand(stack_f)).astype(int)
        # Second for loop for each plan
        for j in range(stack_f):
            # Loading plan and coordinates
            plan_temp = plans_dataset[plan_use[j], :, :]
            coord_temp = coord_dataset[plan_use[j], :, :]
            # Cropping
            xcrop_init = random.randint(0, xcrop) #np.random.random_integers(0, xcrop)
            ycrop_init = random.randint(0, ycrop) #np.random.random_integers(0, ycrop)
            plan_temp = plan_temp[xcrop_init:(xcrop_init+cropped_size), ycrop_init:(ycrop_init+cropped_size)]
            coord_temp = coord_temp[xcrop_init:(xcrop_init+cropped_size), ycrop_init:(ycrop_init+cropped_size)]
            # Rotating
            rotnum = random.randint(0, 3) #np.random.random_integers(0, 3)
            for k in range(rotnum):
                plan_temp = plan_temp.T[::-1, :]
                coord_temp = coord_temp.T[::-1, :]
            # Adding to temporary stack
            plans_temp[j, :, :] = plan_temp
            coords_temp[j, :, :] = coord_temp
        # Adding noise plan
        plan_noise = noise_mean + np.abs(noise_std*np.random.randn(cropped_size, cropped_size))
        plans_temp[-1, :, :] = convolve(plan_noise, filter_noise, mode="constant") # convolution to filter
        # Merging stacks into stack_plan
        stack_plan_temp = np.max(plans_temp, axis=0)
        stack_plan_temp[stack_plan_temp < 0] = 0
        plans_stack[i, :, :] = stack_plan_temp
        # Deleting doublons in stack_coord
        argmax_plan_temp = np.argmax(plans_temp, axis=0) + 1
        coord_stack[i, :, :] = argmax_plan_temp * np.max(coords_temp, axis=0)

    # Returning values
    return plans_stack, coord_stack


from flash import *
from werkzeug.utils import * #vThis will help us to import uploading files concept

app=Flask(__name__)
@app.route('/')
def upload():
    return render_template('file_upload.html')
@app.route('/uploader',methods=['POST','GET'])
def uploader():
    if request.method=='post':
        f=request.files['file']
        f.save(secure_filename(f.filename)) #will save to py uploaded files in our folder
    return 'file uploaded sucessfully'
if __name__=='__main__':
    app.run(debug=True)


# -*- coding: utf-8 -*-

'''
Main operations in EMBL2checklists
'''

#####################
# IMPORT OPERATIONS #
#####################
import ChecklistOps as ClOps
import PrerequisiteOps as PreOps
import globalVariables as GlobVars

import sys, os
from Bio import SeqIO

# Import package modules of EMBL2checklists irrespective of install status
try:
    import EMBL2checklists
except ImportError:
    try:
        package_topLevel = __file__.split(__info__)[0] + __info__
        if os.path.isdir(package_topLevel):
            sys.path.append(package_topLevel)
        if not os.path.isdir(package_topLevel):
            raise ValueError('Top level of package not set.')
    except ValueError:
        try:
            package_topLevel = os.path.dirname(os.path.split(inspect.getfile(EMBL2checklists))[0])  # Homes in on `__init__.py`
            if not os.path.isdir(package_topLevel):
                raise ValueError('Top level of package not set.')
        except ValueError:
            package_topLevel = os.path.dirname(os.path.dirname(__file__))
            if not os.path.isdir(package_topLevel):
                raise ValueError('Top level of package not set.')

###############
# AUTHOR INFO #
###############
__author__ = 'Michael Gruenstaeudl <m.gruenstaeudl@fu-berlin.de>'
__copyright__ = 'Copyright (C) 2016-2018 Michael Gruenstaeudl'
__info__ = 'EMBL2checklists'
__version__ = '2018.09.18.1700'

#############
# DEBUGGING #
#############
import pdb
# pdb.set_trace()

#############
# FUNCTIONS #
#############

def EMBL2checklists(path_to_embl, path_to_outfile, file_format="embl", checklist_type=None, env_sample="no"):
    ''' Primary code function; it executes all subfunctions in the correct order
    Args:
        path_to_embl (string)
        path_to_outfile (string)
        checklist_type (string)
        file_format (string)
        env_sample (string)
    Returns:
        bool: True if nothing went wrong
    Raises:
        Pass exceptions to GUI or CMD interface
    '''

# 1. OPEN OUTHANDLE
    outp_handle = []

# 2. PARSE DATA FROM EMBL-FILE
    try:
        seq_records = SeqIO.parse(open(path_to_embl, "r"), file_format)
    except:
        raise Exception('ERROR: The file ´%s´ could not be parsed.' % (path_to_embl))

# 3. CONVERSION TO CHECKLISTS
    for counter, seq_record in enumerate(seq_records): # Looping through records
        try:
            outdict = {}
    # 3.1. Extraction of marker abbreviations
            marker_abbrevs = ClOps.Parser().parse_marker_abbrevs(seq_record, GlobVars.standard_qualifiers)

    # 3.2. Check if minimal feature prerequisites present; if not, skip this seq_record
            try:
                PreOps.Checker().checkMinimalFeaturePrerequisites(checklist_type, marker_abbrevs)
            except Exception as e:
                GlobVars.warnings.append('WARNING: %s \nSelected checklist type: `%s`; Identified marker abbrevs: `%s` \nSkipping record `%s`...' % (e, checklist_type, marker_abbrevs, seq_record.id))
                continue # Close current loop iteration and continue with next iteration

    # 3.3. Conversion to checklist format
            outdict = ClOps.Parser().mandatoryQualifiers(seq_record, marker_abbrevs, counter, checklist_type, env_sample)

    # 3.4. Optional qualifiers will be added as long as outdict true
            if outdict:
                optional_qualifiers = ClOps.Parser().optionalQualifiers(seq_record, GlobVars.GlobalVariables().getQualifiers(checklist_type, 'o'))
                outdict.update(optional_qualifiers)

    # 3.5. Write parsed record information to handle
            outp_handle.append(outdict)

        except Exception as error:
            raise error

# 4. CLOSE OUTFILE (unless uncaught exceptions)
    outp_file = open(path_to_outfile,"w")
    ClOps.Writer().writer(checklist_type, outp_handle, outp_file)
    outp_file.close()

# 5. RETURN TRUE (unless uncaught exceptions)
    return True


"""
got the program working but accuracy is not getting above 0.333, maybe an
issue with the target data im not sure
"""
import tensorflow as tf
import pandas as pd
import numpy as np
from IPython.display import display
from sklearn import datasets

iris = datasets.load_iris()
df = pd.DataFrame(data= np.c_[iris['data'], iris['target']], columns= iris['feature_names'] + ['target'])
df1 = pd.DataFrame(iris.data, columns=iris.feature_names, )

print("complete df")
display(df)
print("df head")
display(df.head())
print("df types")
display(df.dtypes)
print("df columns")
display(df.columns)
# print("df1 data")
# display(df1.data)

target = df.target
dataset = tf.data.Dataset.from_tensor_slices((df1.values, target.values))
train_dataset = dataset.shuffle(len(df)).batch(1)

def get_compiled_model():
  model = tf.keras.Sequential([
    tf.keras.layers.Dense(4, activation='relu'),
    # tf.keras.layers.Dense(10, activation='relu'),
    tf.keras.layers.Dense(3, activation='sigmoid')
  ])

  model.compile(optimizer='adam',
                loss='binary_crossentropy',
                metrics=['accuracy'])
  return model

model = get_compiled_model()
model.fit(train_dataset, epochs=200)
# model.evaluate(x_test,  y_test, verbose=2)

# (x_train, y_train), (x_test, y_test) = df1.data
# x_train, x_test = x_train / 255.0, x_test / 255.0
# # Build the tf.keras.Sequential model by stacking layers. Choose an optimizer and loss function for training:
#
#
# model = tf.keras.models.Sequential([
#   tf.keras.layers.Flatten(input_shape=(28, 28)),
#   tf.keras.layers.Dense(128, activation='relu'),
#   tf.keras.layers.Dropout(0.2),
#   tf.keras.layers.Dense(10, activation='softmax')
# ])
#
# model.compile(optimizer='adam',
#               loss='sparse_categorical_crossentropy',
#               metrics=['accuracy'])
# # Train and evaluate the model:
#
#
# model.fit(x_train, y_train, epochs=5)
#
# model.evaluate(x_test,  y_test, verbose=2)


"""Our "development" dependencies

Users should *not* need to install these. If users see a load()
statement from these, that's a bug in our distribution.
"""

load("@bazel_tools//tools/build_defs/repo:http.bzl", _http_archive = "http_archive")
load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")

def http_archive(name, **kwargs):
    maybe(_http_archive, name = name, **kwargs)

def rules_pandoc_docx_internal_deps():
    "Fetch deps needed for local development"
    http_archive(
        name = "io_bazel_rules_go",
        sha256 = "19ef30b21eae581177e0028f6f4b1f54c66467017be33d211ab6fc81da01ea4d",
        urls = ["https://github.com/bazelbuild/rules_go/releases/download/v0.38.0/rules_go-v0.38.0.zip"],
    )

    http_archive(
        name = "bazel_gazelle",
        sha256 = "ecba0f04f96b4960a5b250c8e8eeec42281035970aa8852dda73098274d14a1d",
        urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.29.0/bazel-gazelle-v0.29.0.tar.gz"],
    )

    # Override bazel_skylib distribution to fetch sources instead
    # so that the gazelle extension is included
    # see https://github.com/bazelbuild/bazel-skylib/issues/250
    http_archive(
        name = "bazel_skylib",
        sha256 = "3b620033ca48fcd6f5ef2ac85e0f6ec5639605fa2f627968490e52fc91a9932f",
        strip_prefix = "bazel-skylib-1.3.0",
        urls = ["https://github.com/bazelbuild/bazel-skylib/archive/refs/tags/1.3.0.tar.gz"],
    )

    http_archive(
        name = "io_bazel_stardoc",
        sha256 = "3fd8fec4ddec3c670bd810904e2e33170bedfe12f90adf943508184be458c8bb",
        urls = ["https://github.com/bazelbuild/stardoc/releases/download/0.5.3/stardoc-0.5.3.tar.gz"],
    )

    http_archive(
        name = "aspect_bazel_lib",
        sha256 = "dce068f085e9eabfec6d795caaabdbbe4a73550810f3cae3035aff7162e42b3c",
        strip_prefix = "bazel-lib-1.26.2",
        url = "https://github.com/aspect-build/bazel-lib/archive/refs/tags/v1.26.2.tar.gz",
    )


import os
import torch
import torch.nn as nn
# import curve
import platform
import dataset
from torch.utils.data import DataLoader,Dataset
import numpy as np
import argparse, time, random
import yaml
from yaml.loader import SafeLoader
from evaluation import *
import glob
import model
import net
from saver import Saver
from tqdm import tqdm
import matplotlib.pyplot as plt
from matplotlib.ticker import MultipleLocator
from sklearn.metrics import roc_curve, auc
from tensorboardX import SummaryWriter
from sklearn.metrics import cohen_kappa_score
from sklearn.metrics import accuracy_score
from sklearn.metrics import precision_score, recall_score, f1_score
from sklearn.metrics import confusion_matrix
import gc
from sklearn import metrics
import h5py
from utils import *
import pandas as pd
from skimage import io

from apex import amp
"""
label 2016={ 0:'G2_O', 1:'G3_O', 2:'G2_A', 3:'G3_A', 4:'G2_OA', 5:'G3_OA', 6:'GBM'}
label 2021={ 0:'G2_O', 1:'G3_O', 2:'G2_A', 3:'G3_A', 4:'G4_A', 5:'GBM'}
"""

def train(opt):

    gpuID = opt['gpus']

    ############## Init #####################################

    device = torch.device('cuda:{}'.format(gpuID[0])) if gpuID else torch.device('cpu')

    Res_pretrain= net.Res50_pretrain().cuda(gpuID[0])
    # Res_pretrain.to(device)
    Res_pretrain = nn.DataParallel(Res_pretrain, device_ids=gpuID)
    Res_pretrain.eval()

    ###############  Datasets #######################

    trainDataset = dataset.Our_Dataset(phase='Train',opt=opt)
    valDataset = dataset.Our_Dataset(phase='Val', opt=opt)
    testDataset = dataset.Our_Dataset(phase='Test',opt=opt)
    trainLoader = DataLoader(trainDataset, batch_size=opt['batchSize'],
                             num_workers=opt['nThreads'], shuffle=True)
    valLoader = DataLoader(valDataset, batch_size=opt['Val_batchSize'],
                             num_workers=opt['nThreads'], shuffle=True)
    testLoader = DataLoader(testDataset, batch_size=opt['Test_batchSize'],
                            num_workers=opt['nThreads'], shuffle=True)

    train_bar = tqdm(trainLoader)
    for packs in train_bar:
        img = packs[0][0] #(N,3,224,224)
        imgPath = packs[2][0]
        patches_coor = packs[3][0] # list N,2
        if torch.cuda.is_available():
            img = img.cuda(gpuID[0])
        N=img.detach().cpu().numpy().shape[0]

        feature = Res_pretrain(img) # N 1024
        feature = torch.unsqueeze(feature, dim=0)
        feature_save = feature.detach().cpu().numpy()
        feature_save=np.float16(feature_save)
        print(feature_save.shape)
        if not os.path.exists(opt['dataDir'] +'Res50_feature_2000_fixdim0_512/'):
            os.makedirs(opt['dataDir'] +'Res50_feature_2000_fixdim0_512/')
        with h5py.File(opt['dataDir'] +'Res50_feature_2000_fixdim0_512/'+ imgPath+'.h5', 'w') as f:
            f['Res_feature'] = feature_save
            f['patches_coor'] = patches_coor

    train_bar = tqdm(valLoader)
    for packs in train_bar:
        img = packs[0][0]  # (N,3,224,224)
        imgPath = packs[2][0]
        patches_coor = packs[3][0]  # list N,2
        if torch.cuda.is_available():
            img = img.cuda(gpuID[0])
        N = img.detach().cpu().numpy().shape[0]

        feature = Res_pretrain(img)  # N 1024
        feature = torch.unsqueeze(feature, dim=0)
        feature_save = feature.detach().cpu().numpy()
        feature_save = np.float16(feature_save)
        print(feature_save.shape)
        if not os.path.exists(opt['dataDir'] + 'Res50_feature_2000_fixdim0_512/'):
            os.makedirs(opt['dataDir'] + 'Res50_feature_2000_fixdim0_512/')
        with h5py.File(opt['dataDir'] + 'Res50_feature_2000_fixdim0_512/' + imgPath + '.h5', 'w') as f:
            f['Res_feature'] = feature_save
            f['patches_coor'] = patches_coor
    #
    train_bar = tqdm(testLoader)
    for packs in train_bar:
        img = packs[0][0]  # (N,3,224,224)
        imgPath = packs[2][0]
        patches_coor = packs[3][0]  # list N,2
        if torch.cuda.is_available():
            img = img.cuda(gpuID[0])
        N = img.detach().cpu().numpy().shape[0]

        feature = Res_pretrain(img)  # N 1024
        feature = torch.unsqueeze(feature, dim=0)
        feature_save = feature.detach().cpu().numpy()
        feature_save = np.float16(feature_save)
        print(feature_save.shape)
        if not os.path.exists(opt['dataDir'] + 'Res50_feature_2000_fixdim0_512/'):
            os.makedirs(opt['dataDir'] + 'Res50_feature_2000_fixdim0_512/')
        with h5py.File(opt['dataDir'] + 'Res50_feature_2000_fixdim0_512/' + imgPath + '.h5', 'w') as f:
            f['Res_feature'] = feature_save
            f['patches_coor'] = patches_coor
        # features_WSI=[]
        # for i in range(N):
        #     feature=Res_pretrain(torch.unsqueeze(img[i],dim=0))
        #     feature=feature.detach().cpu().numpy()
        #     features_WSI.append(feature)
        # features_WSI=np.asarray(features_WSI) #(N,1024)
        # feature_save = np.expand_dims(features_WSI, axis=0)
        # feature_save = np.float16(feature_save)
        # print(feature_save.shape)
        # if not os.path.exists(opt['dataDir'] + 'Res50_feature_1200_iso_512/'):
        #     os.makedirs(opt['dataDir'] + 'Res50_feature_1200_iso_512/')
        # with h5py.File(opt['dataDir'] + 'Res50_feature_1200_iso_512/' + imgPath + '.h5', 'w') as f:
        #     f['Res_feature'] = feature_save
        #     f['patches_coor'] = patches_coor

        a=1

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('--opt', type=str, default='config/miccai.yml')
    args = parser.parse_args()
    with open(args.opt) as f:
        opt = yaml.load(f, Loader=SafeLoader)
    #
    # k1 = h5py.File(opt['dataDir'] + 'Res50_feature_1/TCGA-DH-5143-01Z-00-DX1.h5')['Res50_feature'][:][0,100,:]
    # k2 = h5py.File( './TCGA-DH-5143-01Z-00-DX1.h5')['Res50_feature'][:][0,100,:]

    # k3 = h5py.File(opt['dataDir'] + 'Res50_feature_1/TCGA-06-0876-01Z-00-DX1.h5')['Res50_feature'][:][0,0,100,:]
    # k4 = h5py.File(opt['dataDir'] + 'Res50_feature_2/TCGA-06-0876-01Z-00-DX1.h5')['Res50_feature'][:][0,100,:]
    #
    # k5 = h5py.File(opt['dataDir'] + 'Res50_feature_1/TCGA-CS-6290-01A-01-TS1.h5')['Res50_feature'][:][0,0,100,:]
    # k6 = h5py.File(opt['dataDir'] + 'Res50_feature_2/TCGA-CS-6290-01A-01-TS1.h5')['Res50_feature'][:][0,100,:]

    a = 1
    train(opt)

    a=1


"""
Spatial statistics.
"""

__author__ = "Steven Kearnes"
__copyright__ = "Copyright 2014, Stanford University"
__license__ = "3-clause BSD"

import numpy as np
from scipy.spatial.distance import cdist, is_valid_dm

def distance(a, b):
    """
    Calculate pairwise distances between examples in two datasets.

    Parameters
    ----------
    a, b : ndarray
        Descriptor datasets with examples on the first axis and descriptors
        on the second axis.
    """
    return cdist(a, b)

def spread(d, t):
    """
    Calculate the spread between two sets of compounds.

    Given a matrix containing distances between two sets of compounds, A
    and B, calculate the fraction of compounds in set A that are closer
    than t to any compound in set B.

    If the distance matrix is symmetric and has zero diagonals, the
    diagonal values are ignored.

    Parameters
    ----------
    d : ndarray
        Distance matrix with compounds from set A on first axis.
    t : float
        Distance threshold.
    """

    # test for symmetric distance matrix
    # diagonal values should be ignored
    if is_valid_dm(d):
        d = np.copy(d)
        d[np.diag_indices_from(d)] = np.inf
    s = np.mean(np.any(d < t, axis=1))
    return s

def sum_of_spreads(d, coeff, min_t=0, max_t=3, step=None, diff=None):
    """
    Calculate the sum of spreads (or spread differences) across a range of
    distance thresholds.

    Parameters
    ----------
    d : ndarray
        Distance matrix.
    coeff : float
        Coefficient used to rescale distance thresholds.
    min_t : float, optional (default 0)
        Minimum distance threshold (before rescaling).
    max_t : float, optional (default 3)
        Maximum distance threshold (before rescaling).
    step : float, optional
        Step size for determining values to sample between min_t and max_t.
        If not provided, defaults to max_t / 500.
    diff : ndarray, optional
        Distance matrix. If provided, the spread will be calculated for this
        distance matrix and subtracted from the spread of d at each distance
        threshold.
    """
    if step is None:
        step = max_t / 500.
    n_steps = int((max_t - min_t) / step)
    thresholds = coeff * np.linspace(min_t, max_t, n_steps)
    if diff is not None:
        ss = np.sum([spread(d, t) - spread(diff, t)
                     for t in thresholds])
    else:
        ss = np.sum([spread(d, t) for t in thresholds])
    return ss


# -*-coding:utf-8-*-
import os

from Brats20183D import *
import pandas as pd

class Config2(Config):
    SavePathDict = {}
    SavePathDict['Unet3D'] = r'./logs\Unet3D\epoch=149-valid_loss=0.25-et_valid_mean_dice=0.74.ckpt'
    SavePathDict['VNet'] = r'./logs\VNet\epoch=69-valid_loss=0.62-et_valid_mean_dice=0.36.ckpt'
    SavePathDict['UNetR'] = r'./logs\UNetR\epoch=42-valid_loss=0.33-et_valid_mean_dice=0.39.ckpt'
    SavePathDict['SwinBTS'] = r'./logs\SwinBTS\epoch=53-valid_loss=0.22-et_valid_mean_dice=0.73.ckpt'
    SavePathDict['TransBTS'] = r'./logs\TransBTS\epoch=51-valid_loss=0.24-et_valid_mean_dice=0.71.ckpt'
    SavePathDict['AttentionUnet'] = r'./logs\AttentionUnet\epoch=49-valid_loss=0.50-et_valid_mean_dice=0.62.ckpt'
    SavePathDict['SwinPureUnet3D'] = r'./logs\SwinPureUnet3D\epoch=38-valid_loss=0.31-et_valid_mean_dice=0.66.ckpt'
    SavePathDict['SwinUnet3D'] = r'./logs\SwinUnet3D\epoch=51-valid_loss=0.22-et_valid_mean_dice=0.73.ckpt'

class Brats2018Model2(Brats2018Model):
    def __init__(self, cfg=Config2()):
        super(Brats2018Model2, self).__init__(cfg)
        self.metrics = [
            DiceMetric(include_background=True,
                       reduction="none"),
            DiceMetric(include_background=True,
                       reduction="none")
        ]

    def validation_step(self, batch, batch_idx):
        with torch.no_grad():
            cfg = self.cfg
            x = batch['image']
            y = batch['label'].float()

            # 使用滑动窗口进行推理
            y_hat = sliding_window_inference(x, roi_size=cfg.roi_size, overlap=cfg.overlap,
                                             sw_batch_size=1, predictor=self.net)
            loss, dices = self.shared_step(y_hat, y, 1)

            return {'valid_loss': loss, 'valid_dice': dices}

    def validation_epoch_end(self, outputs):
        cfg = self.cfg
        losses, mean_dice = self.shared_epoch_end(outputs, 'valid_loss', 1)
        # mean_dice = torch.nan_to_num(mean_dice)

        res_df = pd.DataFrame(columns=['ET', 'TC', 'WT'])
        mean_dice_1 = mean_dice.detach().cpu().numpy()
        res_df['ET'] = mean_dice_1[:, 0]
        res_df['TC'] = mean_dice_1[:, 1]
        res_df['WT'] = mean_dice_1[:, 2]
        if not os.path.exists('./csv_files'):
            os.mkdir('./csv_files')
        res_df.to_csv(os.path.join('./csv_files', cfg.model_name) + '.csv', index=False)

def main(model_name: str = 'Unet3D', seed: int = 3407, cfg=Config2()):
    cfg.model_name = model_name
    cfg.NeedTrain = False
    cfg.seed = seed
    setseed(cfg)

    data = Brats2018DataSet(cfg)
    model = Brats2018Model(cfg)

    early_stop = EarlyStopping(
        monitor='valid_mean_loss',
        patience=5,
    )

    check_point = ModelCheckpoint(dirpath=f'./logs/{cfg.model_name}',
                                  save_last=False,
                                  save_top_k=3, monitor='valid_mean_loss', verbose=True,
                                  filename='{epoch}-{valid_loss:.2f}-{et_valid_mean_dice:.2f}')
    trainer = pl.Trainer(
        accelerator='gpu',
        devices=[2],
        reload_dataloaders_every_n_epochs=1000,
        max_epochs=cfg.max_epoch,
        min_epochs=cfg.min_epoch,
        gpus=1,
        # auto_select_gpus=True, # 这个参数针对混合精度训练时，不能使用
        # auto_lr_find=True,
        auto_scale_batch_size=True,
        logger=TensorBoardLogger(save_dir=f'./logs', name=f'{cfg.model_name}'),
        callbacks=[early_stop, check_point],
        precision=16,
        accumulate_grad_batches=16,
        num_sanity_val_steps=0,
        log_every_n_steps=400,
        gradient_clip_val=1e3,
        gradient_clip_algorithm='norm',
    )

    if cfg.NeedTrain:
        trainer.fit(model, data)
        trainer.save_checkpoint(f'./trained_models/{cfg.model_name}/TrainedModel.ckpt')
    else:
        save_path = cfg.SavePathDict[cfg.model_name]

        argsDict = {'cfg': cfg}
        model = Brats2018Model2.load_from_checkpoint(save_path, **argsDict)  # 这是个类方法，不是对象方法
        model.eval()
        model.freeze()

    # predict_data = Brats2021DataSet()
    # trainer.predict(model, datamodule=predict_data)
    trainer.validate(model, datamodule=data)

if __name__ == '__main__':
    # torch.multiprocessing.set_sharing_strategy('file_system')
    # os.environ['CUDA_LAUNCH_BLOCKING'] = '1'

    g_seeds = [42]

    model_names = ['Unet3D', 'VNet', 'UNetR', 'SwinBTS', 'TransBTS', 'AttentionUnet',
                   'SwinUnet3D', 'SwinPureUnet3D']
    # g_seeds = [2022, 3407]
    # model_names = ['Unet3D', 'SwinUnet3D', 'SwinPureUnet3D', 'UNetR']
    for g_seed in g_seeds:
        for name in model_names:
            main(name, g_seed)


'''
Faça um programa que leia uma quantidade indeterminada de números positivos e conte quantos deles estão nos seguintes intervalos: [0-25], [26-50], [51-75] e [76-100].
A entrada de dados deverá terminar quando for lido um número negativo
'''

from os import system

n = 0
XXIV = 0
L = 0
LXXIV = 0
C = 0
while n >= 0:
    try:
        n = int(input('Digite um número: '))
    except:
        system('clear')
        print('Número inválido! Tente novamente')
        continue
    if n < 0:
        break
    elif n <= 25:
        XXIV += 1
    elif n <= 50:
        L += 1
    elif n <= 75:
        LXXIV += 1
    elif n <= 100:
        C += 1

print(f'[0-25]: {XXIV} números.')
print(f'[26-50]: {L} números.')
print(f'[51-75]: {LXXIV} números.')
print(f'[76-100]: {C} números.')

Python/FATEC/1° Semestre - Algoritmo e Lógica de Programação/Python - Lista 01 - P2/EX-25.py


import common
from MenuTree import *
from codegen import MenuHandler

class PerlCodeGenerator:
    def get_properties_code(self, obj):
        return []

    def get_init_code(self, obj):
        prop = obj.properties
        plgen = common.code_writers['perl']
        out = []
        append = out.append
        menus = obj.properties['menubar']
        ids = []
        # We need to keep track of tmpnames used.
        tmpsused = {}

        def append_items(menu, items):
            for item in items:
                if item.name == '---': # item is a separator
                    append('%s->AppendSeparator();\n' % menu)
                    continue
                name, val = plgen.generate_code_id(None, item.id)
                if not name and ( not val or val == '-1'):
                    id = 'Wx::NewId()'
                else:
                    if name: ids.append(name)
                    id = val

                if item.children:
                    if item.name:
                        name = item.name
                    else:
                        name = '%s_sub' % menu
                        if not tmpsused.has_key(name):
                            tmpsused[name] = 1
                            append('my %s;\n' % name)

                    append('%s = Wx::Menu->new();\n' % name)
                    append_items(name, item.children)
                    append('%s->Append(%s, %s, %s, %s);\n' %
                           (menu, id, plgen.quote_str(item.label),
                            name, plgen.quote_str(item.help_str)))
                else:
                    item_type = 0
                    if item.checkable == '1':
                        item_type = 1
                    elif item.radio == '1':
                        item_type = 2

                    if item.name: itemname = '$self->{%s} = ' % item.name
                    else: itemname = ''

                    if item_type:
                        append('%s%s->Append(%s, %s, %s, %s);\n' %
                               (itemname, menu, id, plgen.quote_str(item.label),
                                plgen.quote_str(item.help_str), item_type))
                    else:

                        append('%s%s->Append(%s, %s, %s);\n' %
                               (itemname, menu, id, plgen.quote_str(item.label),
                                plgen.quote_str(item.help_str)))
        #print 'menus = %s' % menus

        if obj.is_toplevel: obj_name = '$self'
        else: obj_name = '$self->{%s}' % obj.name

        append('my $wxglade_tmp_menu;\n') # NOTE below name =
        for m in menus:
            menu = m.root
            if menu.name: name = '$self->{%s}' % menu.name
            else: name = '$wxglade_tmp_menu'
            append('%s = Wx::Menu->new();\n' % name)
            if menu.children:
                append_items(name, menu.children)
            append('%s->Append(%s, %s);\n' %
                   (obj_name, name, plgen.quote_str(menu.label)))

        return ids + out

    def get_code(self, obj):
        """\
        function that generates Perl code for the menubar of a wxFrame.
        """
        klass = obj.base;
        if klass != obj.klass : klass = obj.klass;
        else: klass = klass.replace('wx','Wx::',1);

        plgen = common.code_writers['perl']
        init = [ '\n\n', '# Menu Bar\n\n', '$self->{%s} = %s->new();\n' %
                 (obj.name, klass) ]
##                  '$self->SetMenuBar($self->{%s});\n' % obj.name ]
        init.extend(self.get_init_code(obj))
        init.append('$self->SetMenuBar($self->{%s});\n' % obj.name)
        init.append('\n# Menu Bar end\n\n')
        return init, [], []

    # 2004-12-05
    def get_events(self, obj):
        pygen = common.code_writers['perl']
        cn = pygen.cn
        out = []

        #print 'get_events', obj.properties['menubar']

        def do_get(item):
            ret = []
            if item.name:
                #val = '#self.%s' % item.name # see py_codegen.py, ~480
                val = item.name
            else:
                name, val = pygen.generate_code_id(None, item.id)
                if not val: val = '-1' # but this is wrong anyway...
            if item.handler:
                ret.append((val, 'EVT_MENU', item.handler))
            if item.children:
                for c in item.children:
                    ret.extend(do_get(c))
            return ret

        for menu in obj.properties['menubar']:
            out.extend(do_get(menu.root))
        return out

# end of class PerlCodeGenerator

def initialize():
    common.class_names['EditMenuBar'] = 'wxMenuBar'
    common.toplevels['EditMenuBar'] = 1

    plgen = common.code_writers.get('perl')
    if plgen:
        plgen.add_widget_handler('wxMenuBar', PerlCodeGenerator())
        plgen.add_property_handler('menus', MenuHandler)


# import selenium webdriver
from selenium import webdriver
import pandas as pd
import time
from selenium.webdriver.common.by import By
from selenium_stealth import stealth
# set browser / browser options
from selenium.webdriver.chrome.options import Options

df = pd.read_csv("subscription.csv", usecols=[1])

options = Options()
driver_path = '/usr/local/bin/chromedriver'
options.binary_location = '/Applications/Brave Browser.app/Contents/MacOS/Brave Browser'
options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
options.add_argument('--disable-blink-features=AutomationControlled')
driver = webdriver.Chrome(options=options, executable_path = driver_path)
stealth(driver,
    languages=["en-US", "en"],
    vendor="Google Inc.",
    platform="Win32",
    webgl_vendor="Intel Inc.",
    renderer="Intel Iris OpenGL Engine",
    fix_hairline=True,
    )
# get page
driver.get("https://www.google.com")
time.sleep(30)

for channel in df["Channel Url"]:
    try:
        driver.get(channel)
        time.sleep(1)
        driver.find_element(By.XPATH, '//*[@id="subscribe-button"]/ytd-subscribe-button-renderer/tp-yt-paper-button').click()
    except:
        continue


#funcao um, diagrama na foto 1

valor_Total = (float(input("Valor total da empresa: ")))

sum = (str(input("Nome do primeiro sócio: ")))
pcum = (float(input("Porcentagem do primeiro sócio: ")))

sdois = (str(input("Nome do segundo sócio: ")))
pcdois = (float(input("Porcentagem do segundo sócio: ")))

stres = (str(input("Nome do terceiro sócio: ")))
pctres = (float(input("Porcentagem do terceiro sócio: ")))

a = valor_Total / pcum
b = valor_Total / pcdois
c = valor_Total / pctres

socios(sum, pcum, sdois, pcdois, stres, pctres)

def socios(sum, pcum, sdois, pcdois, stres, pctreis):
  print(f"O valor total da empresa é: {valor_Total}. \n Propriedade individual - {sum}: R$ {a}. \n  Propriedade individual - {sdois}: R$ {b} \n Propriedade individual - {stres}: R$ {c}.")


import os
from tkinter import image_names
import cv2 as cv
import numpy as np

src_path = "/home/ur10/continual_data/motor_rider_data_merge/"
files = os.listdir(src_path)
sorted_files = sorted(files)

#TODO - get the image bounding box in the correct shape
# Get the Intersection between the motorcycle and the rider
# Store the points as input data

def read_annotation():
    total_motor_boxes = []
    total_rider_boxes = []
    for filename in sorted_files:
        motor_boxes = []
        rider_boxes = []
        if filename.endswith('.txt'):
            image_name = filename.replace('.txt', '.jpg')
            strings = []
            with open(src_path+filename) as file:
                if os.path.getsize(src_path+filename) <= 0:
                    print("No annotation")

                img  = cv.imread(src_path+image_name)
                height, width = img.shape[0], img.shape[1]
                m_box = []
                r_box = []

                for line in file:
                    clas, x, y, w, h = map(float, line.split(' '))

                    if clas == 0: # rider class
                        strings.append(line)
            with open(src_path+filename, 'w') as file:
                if len(strings) == 0:
                    new_line = '0 0.0 0.0 0.0 0.0'
                    strings.append(new_line)
                for line in strings:
                    new_line = '0' + line[1:]
                    print(new_line)
                    file.write(new_line)
                # input_boxes = get_motor_rider_intersection(m_box, r_box)
                # for pair_box in input_boxes:
                #     color = list(np.random.random(size=3) * 256)
                #     for box in pair_box:
                #         cv.rectangle(img, (box[0], box[1]), (box[2], box[3]), color)
                # cv.imshow('Imaeg',img)
                # cv.waitKey(0)
                # cv.destroyAllWindows()
    return motor_boxes, rider_boxes

def check_intersection(box1, box2):
    x1, y1, width1, height1 = box1[0], box1[1], abs(box1[2]- box1[0]), abs(box1[3] - box1[1])
    x2, y2, width2, height2 = box2[0], box2[1], abs(box2[2]- box2[0]), abs(box2[3] - box2[1])

    # Calculate the coordinates of the corners of the boxes
    top_left1 = (x1, y1)
    top_right1 = (x1 + width1, y1)
    bottom_left1 = (x1, y1 + height1)
    bottom_right1 = (x1 + width1, y1 + height1)

    top_left2 = (x2, y2)
    top_right2 = (x2 + width2, y2)
    bottom_left2 = (x2, y2 + height2)
    bottom_right2 = (x2 + width2, y2 + height2)

    # blank_image = np.ones((1400, 1200, 3), np.uint8) * 255
    # cv.rectangle(blank_image,top_left1, bottom_right1, (0,0,255))
    # cv.rectangle(blank_image, top_left2, bottom_right2, (0,0,255))
    # cv.imshow('Imaeg', blank_image)
    # cv.waitKey(0)
    # cv.destroyAllWindows()

    # if (bottom_left1[1] < top_left2[1]) or (bottom_right1[1] > top_left2[1]):
    #     return False
    # if (bottom_right1[0] < top_left2[0]) or (bottom_left1[0] > top_right2[0]):
    #     return False
    # if (bottom_left2[1] < top_left1[1]) or (bottom_right2[1] > top_left1[1]):
    #     return False
    # if (bottom_right2[0]  < top_left1[0]) or (bottom_left2[0] > top_right1[0]):
    #     return False
    #
    # return True

    # # Check for intersection
    if (top_left1[0] <= bottom_right2[0] and bottom_right1[0] >= top_left2[0] and
        top_left1[1] <= bottom_right2[1] and bottom_right1[1] >= top_left2[1]):
        return True  # Boxes intersect
    else:
        return False  # Boxes do not intersect

def get_motor_rider_intersection(motor_boxes, rider_boxes):
    input_boxes = []

    for m_box in motor_boxes:
        pair_boxes = [m_box]
        for r_box in rider_boxes:
            if check_intersection(m_box, r_box):
                if len(pair_boxes) < 5:
                    pair_boxes.append(r_box)
        input_boxes.append(pair_boxes)

    return input_boxes

def get_input_data(input_boxes, width, height):
    input_data = []

    for pair_boxes in input_boxes:
        pair_data = []
        for box in pair_boxes:
            x_c = (box[0] + box[2])/(2*width)
            y_c = (box[1] + box[3])/(2*height)
            w_m = abs(box[0]-box[1])/width
            h_m = abs(box[1]-box[3])/height

            pair_data.append([x_c,y_c, w_m, h_m])
        input_data.append(pair_data)
    return input_data

def main():
    motor_boxes, rider_boxes = read_annotation()
    # input_boxes = get_motor_rider_intersection(motor_boxes, rider_boxes)

if __name__ == "__main__":
    main()

		# count = count + 1
# print(len(empty_files))


#!/usr/bin/env python
from pathlib import Path
import pylivestream as pls
import pytest

R = Path(__file__).parent

inifn = R / 'test.ini'

@pytest.mark.parametrize('site', ['periscope', 'youtube', 'facebook'])
def test_disk(site):
    p = pls.SaveDisk(inifn, '')
    assert p.site == 'file'
    assert p.video_kbps == 3000

if __name__ == '__main__':
    pytest.main(['-x', __file__])


# Yevgeniy Gorbachev
# SoftDev1 pd1
# K<n> -- <Title><Topic/Summary>
# 2019-09-16

import csv #awww yeah
import random

occus = csv.reader(open("occupations.csv"))
lines = [pair for pair in occus]
lines = [(pair[0], float(pair[1])) for pair in lines[1:-2]]
occu_dict = dict(lines)
print('\n'.join(str(item) for item in occu_dict.items()))

def select_random(occupations):
    selected = random.random() * 100
    running_chance = 0
    for pair in occupations:
        running_chance += pair[1]
        if selected < running_chance:
            return pair[0]

def rand_occupations(trials):
    actual_chance = {}
    for pair in lines:
        actual_chance[pair[0]]=0
    # print(actual_chance)
    for i in range(trials):
        occu = select_random(lines)
        if occu != None:
            actual_chance[occu] += 100/trials
    print(actual_chance)

print(select_random(lines))


# -*- coding: utf-8 -*-

"""
Turkish sentence and word tokenizers
"""

__all__ = ['SentenceTokenizer', 'WordTokenizer']
__version__ = '0.0.0.1'

import os
import pathlib
from enum import Enum
from typing import Optional, Tuple, Union

import regex as re

from trtokenizer.errors import NonPrefixFileNotExistError

class SentenceTokenizer:
    class PrefixType(Enum):
        DEFAULT: int
        NUMERIC_ONLY: int

        DEFAULT = 1
        NUMERIC_ONLY = 2

    __slots__ = (
        # Dictionary of non-breaking prefixes; keys are string prefixes, values are PrefixType enums
        '__non_breaking_prefixes',
        'non_breaking_prefix_file',
        'pre_compiled_regexes'
    )

    def __repr__(self):
        if self.non_breaking_prefix_file:
            return f'Sentence tokenizer integrated with "{self.non_breaking_prefix_file}" look-up table'
        else:
            return f'Sentence tokenizer not integrated with look-up table'

    def __init__(self, non_breaking_prefix_file: Optional[str] = None):
        """Sentence tokenizer

        Parameters
        ----------
        non_breaking_prefix_file : str, optional
            path to non-breaking prefix file.

        Raises
        ------
        NonBreakingPrefixFileError
            when non breaking file for given language not exist in abbrev_sentence_tokenizer folder

        """

        self.non_breaking_prefix_file: Optional[str] = non_breaking_prefix_file
        self.__non_breaking_prefixes: dict
        self.pre_compiled_regexes: dict
        line: str
        prefix_type: int
        item: int

        if self.non_breaking_prefix_file is None:
            self.non_breaking_prefix_file = str(
                pathlib.Path(__file__).parent.resolve() / "tr_non_suffixes"
            )

        if not os.path.isfile(self.non_breaking_prefix_file):
            raise NonPrefixFileNotExistError(self.non_breaking_prefix_file)

        self.__non_breaking_prefixes = {}
        with open(self.non_breaking_prefix_file, mode='r', encoding='utf-8') as fp_prefix_file:
            for line in fp_prefix_file.readlines():

                if '#NUMERIC_ONLY#' in line:
                    prefix_type = SentenceTokenizer.PrefixType.NUMERIC_ONLY
                else:
                    prefix_type = SentenceTokenizer.PrefixType.DEFAULT

                # Remove comments
                line = re.sub(pattern=r'#.*', repl='', string=line, flags=re.DOTALL | re.UNICODE)

                line = line.strip()

                if not line:
                    continue

                self.__non_breaking_prefixes[line] = prefix_type

        prefix_type = SentenceTokenizer.PrefixType.DEFAULT
        for item in range(1, 10000):
            self.__non_breaking_prefixes[str(item)] = prefix_type

        # boost regex performance
        self.pre_compiled_regexes = {
            1: re.compile(
                r'([?!]) *([\'"([\u00bf\u00A1\p{Initial_Punctuation}]*[\p{Uppercase_Letter}\p{Other_Letter}])',
                re.U
            ),
            2: re.compile(  # + changed to *
                r'(\.[\.]+) *([\'"([\u00bf\u00A1\p{Initial_Punctuation}]*[\p{Uppercase_Letter}\p{Other_Letter}])',
                re.U
            ),
            3: re.compile(
                r'([?!\.][\ ]*[\'")\]\p{Final_Punctuation}]+) +([\'"([\u00bf\u00A1\p{Initial_Punctuation}]*[\ ]*'
                r'[\p{Uppercase_Letter}\p{Other_Letter}])',
                re.U
            ),
            4: re.compile(
                r'([?!\.]) +([\'"[\u00bf\u00A1\p{Initial_Punctuation}]+[\ ]*[\p{Uppercase_Letter}\p{Other_Letter}])',
                re.U
            ),
            5: re.compile(r' +', re.U),
            6: re.compile(r'([\w\.\-]*)([\'\"\)\]\%\p{Final_Punctuation}]*)(\.+)$', re.U),
            7: re.compile(r'[\p{Uppercase_Letter}\p{Lowercase_Letter}\p{Other_Letter}\-]+', re.U),
            8: re.compile(r'(\.)[\p{Uppercase_Letter}\p{Other_Letter}\-]+(\.+)$', re.U),
            9: re.compile(r'^([ ]*[\'"([\u00bf\u00A1\p{Initial_Punctuation}]*[ ]*[\p{Uppercase_Letter}'
                          r'\p{Other_Letter}0-9])',
                          re.U),
            10: re.compile('^[0-9]+', re.U),
            11: re.compile(r' +'),
            12: re.compile(r'\n '),
            13: re.compile(r' \n ')
        }

    def tokenize(self, text: Optional[str], mode: Optional[str] = None) -> Tuple:
        """Sentence tokenize

        Parameters
        ----------
        text: str
            Text to be split into individual sentences
        mode: None, str
            if given as table, sentence tokenizer is performed only splitting text using enters (\n)
            other wise considers Turkish abbreviations and grammar rules

        Returns
        -------

        """

        words: list
        i: int
        w: list
        forward_word: str

        if text is None:
            return ()

        if not text:
            return ()

        if mode not in {'table', 'only_index'}:

            # Add sentence breaks as needed:

            # Non-period end of sentence markers (?!) followed by sentence starters
            text = self.pre_compiled_regexes[1].sub(repl='\\1\n\\2', string=text)

            # Multi-dots followed by sentence starters
            text = self.pre_compiled_regexes[2].sub(repl='\\1\n\\2', string=text)

            # Add breaks for sentences that end with some sort of punctuation inside a quote or parenthetical and are
            # followed by a possible sentence starter punctuation and upper case
            text = self.pre_compiled_regexes[3].sub(repl='\\1\n\\2', string=text)

            # Add breaks for sentences that end with some sort of punctuation are followed by a
            # sentence starter punctuation and upper case
            text = self.pre_compiled_regexes[4].sub(repl='\\1\n\\2', string=text)

            # Special punctuation cases are covered. Check all remaining periods
            words = self.pre_compiled_regexes[5].split(string=text)

            text = ''
            for i in range(0, len(words) - 1):
                match = self.pre_compiled_regexes[6].search(string=words[i])

                if match:
                    w = words[i].split('.')
                    if self.pre_compiled_regexes[7].search(string=w[-1]):
                        forward_word = w[-1]
                    else:
                        forward_word = words[i + 1]
                    prefix = match.group(1)
                    starting_punctuation = match.group(2)

                    def is_prefix_honorific(prefix_: str, starting_punct_: str) -> bool:
                        """Check if \\1 is a known honorific and \\2 is empty."""
                        if prefix_:
                            if prefix_ in self.__non_breaking_prefixes:
                                if self.__non_breaking_prefixes[prefix_] == SentenceTokenizer.PrefixType.DEFAULT:
                                    if not starting_punct_:
                                        return True
                        return False

                    if is_prefix_honorific(prefix_=prefix, starting_punct_=starting_punctuation):
                        # Not breaking
                        pass

                    elif self.pre_compiled_regexes[8].search(words[i]):
                        # Not breaking - upper case acronym
                        pass

                    elif self.pre_compiled_regexes[9].search(string=forward_word):
                        def is_numeric(prefix_: str, starting_punct_: str, next_word: str):
                            """The next word has a bunch of initial quotes, maybe a space, then either upper case or a
                            number."""
                            if prefix_:
                                if prefix_ in self.__non_breaking_prefixes:
                                    if self.__non_breaking_prefixes[prefix_] == \
                                            SentenceTokenizer.PrefixType.NUMERIC_ONLY:
                                        if not starting_punct_:
                                            if self.pre_compiled_regexes[10].search(string=next_word):
                                                return True
                            return False

                        if not is_numeric(prefix_=prefix, starting_punct_=starting_punctuation,
                                          next_word=forward_word):
                            if words[i + 1] == forward_word:
                                words[i] = ''.join([w[0], '.\n'])
                            else:
                                words[i] = ''.join(['.'.join(w[:-1]), '.\n', forward_word])
                        # We always add a return for these unless we have a numeric non-breaker and a number start
                text = ''.join([text, words[i], " "])

            # We stopped one token from the end to allow for easy look-ahead. Append it now.
            text = ''.join([text, words[-1]])
            text = self.pre_compiled_regexes[11].sub(repl=' ', string=text)
            text = self.pre_compiled_regexes[12].sub(repl='\n', string=text)
            text = self.pre_compiled_regexes[13].sub(repl='\n', string=text)
            text = text.strip()

        return tuple(text.split('\n'))

class WordTokenizer:
    __slots__ = 'pre_compiled_regexes'

    def __init__(self):
        suffixes: str
        numbers: str
        any_word: str
        punctuations: str

        suffixes = r"[a-zğçşöüı]{3,}' ?[a-zğçşöüı]+"
        numbers = r"%\d{2,}[.,:/\d-]+"
        any_word = r"[a-zğçşöüı_+%\.()@&`’/\\\d-]+"
        punctuations = r"[a-zğçşöüı]*[,!?;:]"
        self.pre_compiled_regexes = re.compile(
            "|".join(
                [suffixes,
                 numbers,
                 any_word,
                 punctuations
                 ]
            ), re.I
        )

    def tokenize(self, sentence: str) -> Tuple:
        words: Union[list, tuple]
        dots: str

        try:
            words = self.pre_compiled_regexes.findall(sentence)
        except (re.error, TypeError):
            return ()
        else:
            # If last word ends with dot, it should be another word
            words = tuple(words)
            if words:
                end_dots = re.search(r'\b(\.+)$', words[-1])
                if end_dots:
                    dots = end_dots.group(1)
                    words = words[:-1] + (words[-1][:-len(dots)],) + (dots,)
            return words


import json
import logging
from typing import Any, Dict, Iterable, List, Optional

import dateutil.parser

from datahub.configuration import ConfigModel
from datahub.configuration.common import AllowDenyPattern
from datahub.emitter.mce_builder import DEFAULT_ENV, get_sys_time
from datahub.ingestion.api.common import PipelineContext
from datahub.ingestion.api.source import Source, SourceReport
from datahub.ingestion.api.workunit import MetadataWorkUnit
from datahub.ingestion.source.dbt_types import (
    POSTGRES_TYPES_MAP,
    SNOWFLAKE_TYPES_MAP,
    resolve_postgres_modified_type,
)
from datahub.metadata.com.linkedin.pegasus2avro.common import AuditStamp
from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
    DatasetLineageTypeClass,
    UpstreamClass,
    UpstreamLineage,
)
from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import DatasetSnapshot
from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent
from datahub.metadata.com.linkedin.pegasus2avro.schema import (
    BooleanTypeClass,
    DateTypeClass,
    MySqlDDL,
    NullTypeClass,
    NumberTypeClass,
    SchemaField,
    SchemaFieldDataType,
    SchemaMetadata,
    StringTypeClass,
    TimeTypeClass,
)
from datahub.metadata.schema_classes import DatasetPropertiesClass

logger = logging.getLogger(__name__)

class DBTConfig(ConfigModel):
    manifest_path: str
    catalog_path: str
    sources_path: Optional[str]
    env: str = DEFAULT_ENV
    target_platform: str
    load_schemas: bool
    node_type_pattern: AllowDenyPattern = AllowDenyPattern.allow_all()

class DBTColumn:
    name: str
    comment: str
    index: int
    data_type: str

    def __repr__(self):
        fields = tuple("{}={}".format(k, v) for k, v in self.__dict__.items())
        return self.__class__.__name__ + str(tuple(sorted(fields))).replace("'", "")

class DBTNode:
    dbt_name: str
    database: str
    schema: str
    dbt_file_path: str
    node_type: str  # source, model
    materialization: str  # table, view, ephemeral
    name: str  # name, identifier
    columns: List[DBTColumn]
    upstream_urns: List[str]
    datahub_urn: str
    max_loaded_at: Optional[str]

    def __repr__(self):
        fields = tuple("{}={}".format(k, v) for k, v in self.__dict__.items())
        return self.__class__.__name__ + str(tuple(sorted(fields))).replace("'", "")

def get_columns(catalog_node: dict) -> List[DBTColumn]:
    columns = []

    raw_columns = catalog_node["columns"]

    for key in raw_columns:
        raw_column = raw_columns[key]

        dbtCol = DBTColumn()
        dbtCol.comment = raw_column["comment"]
        dbtCol.data_type = raw_column["type"]
        dbtCol.index = raw_column["index"]
        dbtCol.name = raw_column["name"]
        columns.append(dbtCol)
    return columns

def extract_dbt_entities(
    nodes: Dict[str, Dict[str, Any]],
    catalog: Dict[str, Dict[str, Any]],
    sources_results: List[Dict[str, Any]],
    load_catalog: bool,
    target_platform: str,
    environment: str,
    node_type_pattern: AllowDenyPattern,
) -> List[DBTNode]:

    sources_by_id = {x["unique_id"]: x for x in sources_results}

    dbt_entities = []
    for key, node in nodes.items():
        dbtNode = DBTNode()

        # check if node pattern allowed based on config file
        if not node_type_pattern.allowed(node["resource_type"]):
            continue
        dbtNode.dbt_name = key
        dbtNode.database = node["database"]
        dbtNode.schema = node["schema"]
        dbtNode.dbt_file_path = node["original_file_path"]
        dbtNode.node_type = node["resource_type"]
        dbtNode.max_loaded_at = sources_by_id.get(key, {}).get("max_loaded_at")
        if "identifier" in node and not load_catalog:
            dbtNode.name = node["identifier"]
        else:
            dbtNode.name = node["name"]

        if "materialized" in node["config"].keys():
            # It's a model
            dbtNode.materialization = node["config"]["materialized"]
            dbtNode.upstream_urns = get_upstreams(
                node["depends_on"]["nodes"],
                nodes,
                load_catalog,
                target_platform,
                environment,
            )
        else:
            # It's a source
            dbtNode.materialization = catalog[key]["metadata"][
                "type"
            ]  # get materialization from catalog? required?
            dbtNode.upstream_urns = []

        if (
            dbtNode.materialization != "ephemeral" and load_catalog
        ):  # we don't want columns if platform isn't 'dbt'
            logger.debug("Loading schema info")
            dbtNode.columns = get_columns(catalog[dbtNode.dbt_name])
        else:
            dbtNode.columns = []

        dbtNode.datahub_urn = get_urn_from_dbtNode(
            dbtNode.database,
            dbtNode.schema,
            dbtNode.name,
            target_platform,
            environment,
        )

        dbt_entities.append(dbtNode)

    return dbt_entities

def loadManifestAndCatalog(
    manifest_path: str,
    catalog_path: str,
    sources_path: Optional[str],
    load_catalog: bool,
    target_platform: str,
    environment: str,
    node_type_pattern: AllowDenyPattern,
) -> List[DBTNode]:
    with open(manifest_path, "r") as manifest:
        dbt_manifest_json = json.load(manifest)

    with open(catalog_path, "r") as catalog:
        dbt_catalog_json = json.load(catalog)

    if sources_path is not None:
        with open(sources_path, "r") as sources:
            dbt_sources_json = json.load(sources)
            sources_results = dbt_sources_json["results"]
    else:
        sources_results = {}

    manifest_nodes = dbt_manifest_json["nodes"]
    manifest_sources = dbt_manifest_json["sources"]

    all_manifest_entities = {**manifest_nodes, **manifest_sources}

    catalog_nodes = dbt_catalog_json["nodes"]
    catalog_sources = dbt_catalog_json["sources"]

    all_catalog_entities = {**catalog_nodes, **catalog_sources}

    nodes = extract_dbt_entities(
        all_manifest_entities,
        all_catalog_entities,
        sources_results,
        load_catalog,
        target_platform,
        environment,
        node_type_pattern,
    )

    return nodes

def get_urn_from_dbtNode(
    database: str, schema: str, name: str, target_platform: str, env: str
) -> str:

    db_fqn = f"{database}.{schema}.{name}".replace('"', "")
    return f"urn:li:dataset:(urn:li:dataPlatform:{target_platform},{db_fqn},{env})"

def get_custom_properties(node: DBTNode) -> Dict[str, str]:
    return {
        "dbt_node_type": node.node_type,
        "materialization": node.materialization,
        "dbt_file_path": node.dbt_file_path,
    }

def get_upstreams(
    upstreams: List[str],
    all_nodes: Dict[str, dict],
    load_catalog: bool,
    target_platform: str,
    environment: str,
) -> List[str]:
    upstream_urns = []

    for upstream in upstreams:
        dbtNode_upstream = DBTNode()

        dbtNode_upstream.database = all_nodes[upstream]["database"]
        dbtNode_upstream.schema = all_nodes[upstream]["schema"]
        if "identifier" in all_nodes[upstream] and not load_catalog:
            dbtNode_upstream.name = all_nodes[upstream]["identifier"]
        else:
            dbtNode_upstream.name = all_nodes[upstream]["name"]

        upstream_urns.append(
            get_urn_from_dbtNode(
                dbtNode_upstream.database,
                dbtNode_upstream.schema,
                dbtNode_upstream.name,
                target_platform,
                environment,
            )
        )

    return upstream_urns

def get_upstream_lineage(upstream_urns: List[str]) -> UpstreamLineage:
    ucl: List[UpstreamClass] = []

    actor = "urn:li:corpuser:dbt_executor"
    sys_time = get_sys_time()

    for dep in upstream_urns:
        uc = UpstreamClass(
            dataset=dep,
            auditStamp=AuditStamp(actor=actor, time=sys_time),
            type=DatasetLineageTypeClass.TRANSFORMED,
        )
        ucl.append(uc)

    return UpstreamLineage(upstreams=ucl)

# See https://github.com/fishtown-analytics/dbt/blob/master/core/dbt/adapters/sql/impl.py
_field_type_mapping = {
    "boolean": BooleanTypeClass,
    "date": DateTypeClass,
    "time": TimeTypeClass,
    "numeric": NumberTypeClass,
    "text": StringTypeClass,
    "timestamp with time zone": DateTypeClass,
    "timestamp without time zone": DateTypeClass,
    "integer": NumberTypeClass,
    "float8": NumberTypeClass,
    **POSTGRES_TYPES_MAP,
    **SNOWFLAKE_TYPES_MAP,
}

def get_column_type(
    report: SourceReport, dataset_name: str, column_type: str
) -> SchemaFieldDataType:
    """
    Maps known DBT types to datahub types
    """
    TypeClass: Any = _field_type_mapping.get(column_type)

    if TypeClass is None:

        # attempt Postgres modified type
        TypeClass = resolve_postgres_modified_type(column_type)

    # if still not found, report the warning
    if TypeClass is None:

        report.report_warning(
            dataset_name, f"unable to map type {column_type} to metadata schema"
        )
        TypeClass = NullTypeClass

    return SchemaFieldDataType(type=TypeClass())

def get_schema_metadata(
    report: SourceReport, node: DBTNode, platform: str
) -> SchemaMetadata:
    canonical_schema: List[SchemaField] = []
    for column in node.columns:
        field = SchemaField(
            fieldPath=column.name,
            nativeDataType=column.data_type,
            type=get_column_type(report, node.dbt_name, column.data_type),
            description=column.comment,
            nullable=False,  # TODO: actually autodetect this
            recursive=False,
        )

        canonical_schema.append(field)

    actor = "urn:li:corpuser:dbt_executor"
    sys_time = get_sys_time()

    last_modified = sys_time

    if node.max_loaded_at is not None:
        last_modified = int(
            dateutil.parser.parse(node.max_loaded_at).timestamp() * 1000
        )

    return SchemaMetadata(
        schemaName=node.dbt_name,
        platform=f"urn:li:dataPlatform:{platform}",
        version=0,
        hash="",
        platformSchema=MySqlDDL(tableSchema=""),
        created=AuditStamp(time=sys_time, actor=actor),
        lastModified=AuditStamp(time=last_modified, actor=actor),
        fields=canonical_schema,
    )

class DBTSource(Source):
    """Extract DBT metadata for ingestion to Datahub"""

    @classmethod
    def create(cls, config_dict, ctx):
        config = DBTConfig.parse_obj(config_dict)
        return cls(config, ctx, "dbt")

    def __init__(self, config: DBTConfig, ctx: PipelineContext, platform: str):
        super().__init__(ctx)
        self.config = config
        self.platform = platform
        self.report = SourceReport()

    def get_workunits(self) -> Iterable[MetadataWorkUnit]:
        platform = self.platform
        nodes = loadManifestAndCatalog(
            self.config.manifest_path,
            self.config.catalog_path,
            self.config.sources_path,
            self.config.load_schemas,
            self.config.target_platform,
            self.config.env,
            self.config.node_type_pattern,
        )

        for node in nodes:

            dataset_snapshot = DatasetSnapshot(
                urn=node.datahub_urn,
                aspects=[],
            )
            custom_properties = get_custom_properties(node)

            dbt_properties = DatasetPropertiesClass(
                description=node.dbt_name,
                customProperties=custom_properties,
                tags=[],
            )
            dataset_snapshot.aspects.append(dbt_properties)

            upstreams = get_upstream_lineage(node.upstream_urns)
            if upstreams is not None:
                dataset_snapshot.aspects.append(upstreams)

            if self.config.load_schemas:
                schema_metadata = get_schema_metadata(self.report, node, platform)
                dataset_snapshot.aspects.append(schema_metadata)

            mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot)
            wu = MetadataWorkUnit(id=dataset_snapshot.urn, mce=mce)
            self.report.report_workunit(wu)

            yield wu

    def get_report(self):
        return self.report

    def close(self):
        pass


import classes.movies as mov
import classes.database as db
import classes.datamanip as dm

table_name = input("Enter table name: ")
chunksize = eval(input("Enter chunksize: "))
startchunk = eval(input("Enter startchunk (0 if first time): ")) - 1
dfidlist = mov.downloadIDlist()
print("ID list downloaded")

numOfChunks = round(dfidlist.shape[0] // chunksize) + 1

print(f"There will be {numOfChunks} chunks")

db.connect()
print("Creating table")
db.createMoviesTable(table_name)
db.shutdown()

for i in range(numOfChunks):
    print("Downloading movies")
    df, startchunk = mov.downloadMoviesInChunks(dfidlist, chunksize, startchunk)
    print("Cleaning data")
    df = dm.cleanup(df)
    print("Data cleaned")
    db.connect()
    print("Uploading data")
    db.uploadMovieData(df, table_name)
    print("Data uploaded")
    db.shutdown()


"""
    user
    ~~~

    :author: dilless(Huangbo)
    :date: 2022/7/10
"""
import asyncio
import datetime
import hashlib
import random
import uuid
from typing import Optional

from fastapi import APIRouter, Path, Query, Depends
from sqlmodel import Session, select
from starlette.requests import Request

import crud
import models
from api import deps
from app import schemas
from core.config import settings
from db.mysql import engine
from utils.redis_ import aio_redis, get_login_code_key, get_login_user_obj_key, SignUtils

router = APIRouter()

@router.post('/code',
             response_model=schemas.GenericResponseModel)
async def send_code(
        *,
        phone: str = Query(),
):
    code = ''.join(random.sample(list(map(str, range(0, 10))), 6))
    await aio_redis.setex(get_login_code_key(phone), settings.LOGIN_CODE_EXPIRE, code)
    print('发送验证码成功，验证码：{}'.format(code))
    return schemas.GenericResponseModel()

@router.post('/login',
             response_model=schemas.GenericResponseModel)
async def login(
        *,
        login_model: schemas.UserLoginModel,
):
    cached_code = await aio_redis.get(get_login_code_key(login_model.phone))
    if login_model.code is None or cached_code != login_model.code:
        return schemas.GenericResponseModel(success=False, error_msg='验证码错误')

    user = crud.user_crud.get_user_by_phone(phone=login_model.phone)
    if user is None:
        user = crud.user_crud.create_user_with_phone(phone=login_model.phone)

    token = hashlib.md5(str(user.id).encode()).hexdigest()
    base_user = schemas.UserBaseModel.parse_obj(user)
    await aio_redis.hmset(get_login_user_obj_key(token), base_user.dict(exclude_none=True))
    await aio_redis.expire(get_login_user_obj_key(token), settings.LOGIN_USER_OBJ_EXPIRE)

    return schemas.GenericResponseModel(data=token)

@router.post('/logout',
             response_model=schemas.GenericResponseModel)
async def logout(
        *,
        user: schemas.UserBaseModel = Depends(deps.verify_user),
):
    pass

@router.get('/me',
            response_model=schemas.GenericResponseModel)
async def me(
        *,
        user: schemas.UserBaseModel = Depends(deps.verify_user),

):
    return schemas.GenericResponseModel(data=schemas.UserBaseModel.parse_obj(user))

@router.post('/sign',
             response_model=schemas.GenericResponseModel)
async def sign(
        *,
        user: schemas.UserBaseModel = Depends(deps.verify_user),
):
    await SignUtils.sign(datetime.datetime.now(), user.id)
    return schemas.GenericResponseModel()

@router.get('/sign/count',
            response_model=schemas.GenericResponseModel)
async def sign_count(
        *,
        user: schemas.UserBaseModel = Depends(deps.verify_user),
):
    count = await SignUtils.sign_count(datetime.datetime.now(), user.id)
    return schemas.GenericResponseModel(data=count)

@router.get('/{user_id}',
            response_model=schemas.GenericResponseModel)
async def get_user(
        *,
        user_id: int = Path(),
):
    user = crud.user_crud.get_by_id(pk=user_id)
    return schemas.GenericResponseModel(data=schemas.UserBaseModel.parse_obj(user))

@router.get('/info/{user_id}',
            response_model=schemas.GenericResponseModel)
async def info(
        *,
        user_id: int = Path()
):
    with Session(engine) as sess:
        statement = select(models.User).where(models.UserInfo.user_id == user_id)
        result = sess.exec(statement).first()
    return schemas.GenericResponseModel(data=result)

if __name__ == '__main__':
    async def do_job(u, lock):
        code = await send_code(phone=u.phone)
        login_obj = schemas.UserLoginModel(
            phone=u.phone,
            code=code,
        )
        login_resp = await login(login_model=login_obj)
        token = login_resp.data
        async with lock:
            with open('../../../token.txt', 'a', encoding='utf-8') as f:
                f.write(token + '\n')

    async def main():
        with Session(engine) as sess:
            users = sess.exec(select(models.User)).all()[:1000]
        lock = asyncio.Lock()
        for u in users:
            await do_job(u, lock)

    asyncio.run(main())