"""Functions for loading data files from the SP experiment

"""
import numpy as np
from os import path
import os
import re
from collections import namedtuple
import yaml
import MT
import convert

relaxedNames="""Tot Pos Det
Tot Pos Det 2
Tot Pos Det 3
Cell Temp
Ferrite Temp
Oven Temp
Temperature Controller Output
Breadboard Temp
Room Temp
G10 Lid Temp
Tilt Sensor 1
Tilt Sensor 2
WavelengthErr
Pump Current Mon
Pump Temp Mon
Probe Current Mon
Probe Temp Mon
Water Flow
Apparatus Position
""".splitlines()
mediumNames="""Accelerometer
Position Purple
Position Green
Fluxgate X
Fluxgate Y
Fluxgate Z
Tiltmeter X
Tiltmeter Y
Vert Pos Det
Horiz Pos Det
Vert Pos Det 2
Horiz Pos Det 2
Vert Pos Det 3
Horiz Pos Det 3
""".splitlines()

def getSpinningDataNames(timestamp, bMotorData=False):
    if not bMotorData:
        base_name='SP_aparatus {0}'.format(timestamp)
    else:
        base_name='SP_motor {0}'.format(timestamp)
    base_dir=os.path.join(os.getenv('SP_DATA_DIR'), base_name + ' Bundle/')
    return base_dir, base_name

def getAllTimeStamps(bApparatus=True, bMotor=False):
    data_dir=os.path.join(os.getenv('SP_DATA_DIR'))
    ap_dir_regex=re.compile('SP_aparatus (\d+(?:\.\d*)?|\.\d+) Bundle$')
    mo_dir_regex=re.compile('SP_motor (\d+(?:\.\d*)?|\.\d+) Bundle$')

    regExL=[]
    if bApparatus:
        regExL.append(ap_dir_regex)
    if bMotor:
        regExL.append(mo_dir_regex)
    outL=[]
    for reg in regExL:
        match_list=[ap_dir_regex.match(line) for line in os.listdir(data_dir)]
        stamps=[m.groups()[0] for m in match_list if m is not None]
        stamps.sort()
        outL.append(stamps)
    if len(outL)==1:
        outL=outL[0]
    return outL

#old
def loadSet(timestamp, cal=None, subL=["Fast", "Medium", "Relaxed"], startTime=-np.inf, endTime=np.inf):
    """OLD Load all the data associated with @timestamp
    """
    if cal is None:
        print("Haven't implemented loading of calibration data yet, Your data will stay in Volts")
        cal=1
    base_dir, base_name=getSpinningDataNames(timestamp)

    def splitRecArray(rA):
        t=rA['sd']
        names=list(rA.dtype.names)
        names.remove('sd')
        y=rA[names].view(rA.dtype[-1]).reshape(rA.size,-1)
        return [t, y.T, names]

    #startI=0
    fastD=None;
    medD=None;
    relD=None;
    if "Fast" in subL:
        fastRec=loadSPBinary.loadBins('{0}{1} {2}'.format(base_dir, base_name, 'Fast'), totalDataType=[('sd','>f8'), ('s1', '>f4')])
        if fastRec is not None:
            fastD=splitRecArray(fastRec)
            fastD[1]=fastD[1].squeeze()
            fastD[1]*=cal

            startI, endI=fastD[0].searchsorted([startTime, endTime])
            fastD[0]=fastD[0][startI:endI]
            fastD[1]=fastD[1][startI:endI]

    if "Medium" in subL:
        medRec=loadSPBinary.loadBins('{0}{1} {2}'.format(base_dir, base_name, 'Medium'), useMetaFile=True)
        if medRec is not None:
            renameFieldMaybe(medRec, 'Sidereal Days', 'sd')
            medD=splitRecArray(medRec)
            startI, endI=medD[0].searchsorted([startTime, endTime])
            medD[0]=medD[0][startI:endI]
            medD[1]=medD[1][:,startI:endI]

    if "Relaxed" in subL:
        relRec=loadSPBinary.loadBins('{0}{1} {2}'.format(base_dir, base_name, 'Relaxed'), useMetaFile=True)
        if relRec is not None:
            renameFieldMaybe(relRec, 'Sidereal Days', 'sd')
            relD=splitRecArray(relRec)
            startI, endI=relD[0].searchsorted([startTime, endTime])
            relD[0]=relD[0][startI:endI]
            relD[1]=relD[1][:,startI:endI]
    if "Slow" in subL:
        slRec=loadSPBinary.loadBins('{0}{1} {2}'.format(base_dir, base_name, 'Slow'), useMetaFile=True)
        if slRec is not None:
            renameFieldMaybe(slRec, 'Sidereal Days', 'sd')
            slD=splitRecArray(slRec)

    return SPDataSet(fastD, medD=medD, relD=relD)

set_info_defaults={
            'bUseful': 'unknown',
            'bDataConfirmed':False,
            'acquisition':{'sampleRate': 50, 'timeConstant': 0.003},
            #'rotationSpec':{'delay': 9,
            #                'interval':30,
            #                'numRotations':20,
            #                'rotAngles': [-180],
            #                'extraRotAngle': -90.,
            #                'startingAngle': 0,
            #                },
            'timeStamp':'unknown',
            'notes': 'Default',
            #'windows':{}
            'windows':{
                'sig':{
                    'duration': 4,
                    'offset': -6.6,
                    },
                'bg':{
                    'duration': 1,
                    'offset': 0,
                    },
                    }
            }
def loadSetInfo(dir_name):
    ts=dir_name.split()[-2]
    bWriteNew=False
    filepath=os.path.join(dir_name, 'set_info.yaml')
    try:
        f=open(filepath)
        infoD=yaml.load(f)
        f.close()
    except IOError:
        print("No set_info.yaml exists in {}, making defaults".format(dir_name))
        bWriteNew=True
        infoD= {}

    loadedKeys=infoD.keys()
    if 'bandwidth' in loadedKeys: #It's a matlab generated yaml file
        print("updating matlab generated yaml")
        #newD=copy(set_info_defaults)
        newD={
            'bUseful':infoD['bUseful'],
            'bDataConfirmed':infoD['bDataConfirmed'],
            'acquisition':{'sampleRate': infoD['bandwidth'][1],
                            'timeConst': infoD['bandwidth'][0]
                         },
            'rotationSpec':{'delay': infoD['delay'], 'interval':infoD['interval'],
                            'numRotations':infoD['numRotations'],
                            'rotAngles': [infoD['rotationAngles'][0]],
                            'extraRotAngle':infoD['rotationAngles'][1],
                            'startingAngle':MT.cardinalToDeg(infoD['startPosition']),
                            },
            'timeStamp':infoD['timestamp'],
            'notes':infoD['notes']+'\nConverted from matlab',
            }
        infoD=newD
        writeSetInfo(dir_name, infoD)

    if infoD.has_key('rotationSpec') and infoD['rotationSpec'].has_key('startPosition'):
        infoD['rotationSpec']['startingAngle']= MT.cardinalToDeg(infoD['rotationSpec']['startPosition'])
        infoD['rotationSpec'].pop('startPosition')

    newLoadedKeys=infoD.keys()
    for key in set_info_defaults.keys():
        if key not in newLoadedKeys:
            infoD[key]=set_info_defaults[key]
            print("Loading {0} from defaults, it may not be right. Go fix this in set_info.yaml!".format(key))

    if bWriteNew:
        infoD['timeStamp']=ts
        writeSetInfo(dir_name, infoD)
    return infoD

def writeSetInfo(dir_name, D):
        filepath=os.path.join(dir_name, 'set_info.yaml')
        print("writing new set_info.yaml file...")
        fout=open(filepath,'w')
        fout.write(yaml.dump(D))
        fout.close()
        print("{0} was written.".format(filepath))

def loadZeroFile(fname):
    proc_dict={
            1: 'cBz',
            2: 'By',
            3: 'Bx',
            35: 'Cal',
            29: 'Pol',
    }
    dat=np.loadtxt(fname).T;
    t=dat[0]
    proc_num=dat[1]
    val=dat[2]
    slope=dat[3]
    sigma=dat[4]
    ZeroProc=namedtuple('ZeroProc', ['t', 'val', 'slope', 'sigma'])
    zD={}
    for num, name in proc_dict.iteritems():
        I=np.where(proc_num==float(num))[0]
        zD[name]=ZeroProc(t=t[I], val=val[I], slope=slope[I], sigma=sigma[I])
        #DataFrame =namedtuple('DataFrame', ['sd', 'y'])
    return zD

def getAllSPSensor(names, slc=None, timeRange=None):
    data_dir=os.path.join(os.getenv('SP_DATA_DIR'))
    ap_dir_regex=re.compile('SP_aparatus (\d+(?:\.\d*)?|\.\d+) Bundle$')
    mo_dir_regex=re.compile('SP_motor (\d+(?:\.\d*)?|\.\d+) Bundle$')

    ap_match_list=[ap_dir_regex.match(line) for line in os.listdir(data_dir)]
    ap_stamps=[m.groups()[0] for m in ap_match_list if m is not None]
    mo_match_list=[mo_dir_regex.match(line) for line in os.listdir(data_dir)]
    mo_stamps=[m.groups()[0] for m in mo_match_list if m is not None]

    mo_stamps.sort()
    ap_stamps.sort()
    ioff()
    sdL=[]
    if slc is None:
        [firstI, lastI]=searchsorted(ap_stamps, timeRange)
        slc=slice(firstI, lastI)
    #slc=slice(-20,-15)
    relIdxs=[relaxedNames.index(name) for name in names if name in relaxedNames]
    medIdxs=[mediumNames.index(name) for name in names if name in mediumNames]
    #slIdxs=[slowNames.index(name) for name in names if name is in slowNames]
    #medList=[idx for name in names if name is in mediumNames]
    #slList=[idx for name in names if name is in slNames]

    allMedL=[]
    allRelL=[]
    subSetL=[]
    if medIdxs:
        subSetL.append("Medium")
    if relIdxs:
        subSetL.append("Relaxed")

    for stamp in ap_stamps[slc]:
        thisTsYL=[]
        dat=loadSet(stamp, subL=subSetL)
        relL=[]
        medL=[]
        #tsL=[]
        for idxs, D, l in [(relIdxs, dat.relD, relL), (medIdxs, dat.medD, medL)]:
            if idxs and D:
                l.append([D[0]]+[D[1][idx] for idx in idxs])
                #tsL.append(D[0])
            #else:
                #tsL.append(None)
                #l.append(None)

        if medL:
            allMedL.append(medL)
        if relL:
            allRelL.append(relL)

        if 0:
            try:
                dat=loadSet(stamp, subL=subSet)
                if subSet=='Relaxed':
                    D=dat.relD
                elif subSet=='Medium':
                    D=dat.medD
                elif subSet=='Slow':
                    D=dat.slowD
                if D:
                    sd=D[0].copy()
                    y=D[1][D[2].index(name)].copy()
                    del dat
                    sdL.append(sd)
                    yL.append(y)

            except ValueError:
                print("\nTimestamp {0} didn't load, maybe no '{1}' data? \n \n".format(stamp, subSet))
        if allRelL:
            allRelL=dstack(allRelL).squeeze()
        if allMedL:
            allMedL=dstack(allMedL).squeeze()
    return allMedL, allRelL

def loadBins(root_path, useMetaFile=True, nVars=1, endian='', dType=None, totalDataType=None):
    """Load all the binary files in that match the root path.

    They match the root path with regex 'root_path( \d+)?.bin',
    that is anything that starts with root_path, ends with .bin, and optionally has a number after a space. If there are numbers after the space, the files are loaded and the data appended in the correct order.
    """
    dirname=path.dirname(root_path)
    rootname=path.basename(root_path)
    r=re.compile('{0}( \d+)?\.bin'.format(rootname))
    dlist=os.listdir(dirname);
    dlist=[st for st in dlist if r.match(st)]
    if not dlist:
        print("No data for {0}".format(root_path))
        return None
    if len(dlist)>1:
        dlist.sort( key=lambda st: int(r.match(st).groups()[0]) ) # dlist is now sorted

    dat_list=[loadABin(path.join(dirname, fname), useMetaFile=useMetaFile, nVars=nVars, endian=endian, totalDataType=totalDataType) for fname in dlist]
    # Combine these into a single file...

    dat=np.hstack(dat_list)
    #datnames=dat_list[0].keys()
    return dat

def loadABin(filepath, nVars=1, useMetaFile=True, endian='', totalDataType=None):
    """Load a single binary file at filepath
    """
    #This is all just establishing how the data is stored in the file
    if totalDataType is None: #If a data type wasn't given, we'll try to work it out:
        if useMetaFile: #Usually the data type is stored in a meta file:
            fname_components=path.splitext(filepath)[0].split(' ')
            if fname_components[-1].isdigit():
                fname_components.pop()
            meta_file_path=' '.join(fname_components) + '.npmeta'
            npmeta_path=' '.join(fname_components) + '.npmeta'
            if path.exists(npmeta_path):
                mfile=open(npmeta_path)
                lines=mfile.readlines()
                pairs=[line.strip().split(',') for line in lines if not line.isspace() and not line.startswith('#') ]# should be tuples of e.g. ('signal', 'f8');
                defaultDtype='>f4'
                totalDataType=[]
                for p in pairs:
                    name=p[0].strip()
                    if len(p)>1:
                        dt=p[1].strip()
                        defaultDtype=dt;
                    else:
                        dt=defaultDtype;
                    totalDataType.append((name, dt));
            else: #If there was no appropriate unambiguous meta-file, look for the old style:
                meta_file_path=' '.join(fname_components) + '.meta'
                if not path.exists(meta_file_path): #Sometimes they have the extension .meta.txt for no good reason, so check for that too
                    meta_file_path=meta_file_path+'.txt'
                if path.exists(meta_file_path):
                    convert.spmeta2npmeta(meta_file_path) #This convert the old style to the new (Corrects for some changes, but not all)
                    print("Found an old meta-file only. Will attempt to make something useful out of it (old style is ambiguous)")
                    return loadABin(filepath, nVars, useMetaFile) #We'll run the function again with the new meta file exists
                    #mfile=open(meta_file_path)
                else:
                    raise IOError("Looking for a meta-file (for {0}) but none exists".format(' '.join(fname_components)))
                sigNames=mfile.readlines()
                sigNames=[name.strip() for name in sigNames]
                if sigNames[0]=='Sidereal Days':
                    sigNames=sigNames[1:]
                nVars=len(sigNames)
                sigDataType=[(name, endian+'f8') for name in sigNames]
                timeDataType=[('sd', endian+'f8')]
                totalDataType=timeDataType+sigDataType

        else:
            timeDataType=[('sd', endian+'f8')]
            sigDataType=[('s{0}'.format(n), endian+'f8') for n in range(1,nVars+1)]
            totalDataType=timeDataType+sigDataType

    # This is where we actually read it
    dat=np.fromfile(filepath, totalDataType)

    return dat;

if __name__=="__main__":
    #ddir='/media/morgan/6a0066c2-6c13-49f4-9322-0b818dda2fe8/Princeton Data/South Pole/SP_motor 5253.67 Bundle/'
    #fpath=ddir+'SP_motor 5253.67 sensors 8.bin'
    ddir='/media/morgan/6a0066c2-6c13-49f4-9322-0b818dda2fe8/Princeton Data/South Pole/SP_aparatus 5256.38 Bundle/'
    #ddir='/media/morgan/6a0066c2-6c13-49f4-9322-0b818dda2fe8/Princeton Data/South Pole/SP_motor 5256.38 Bundle/'
    fname='SP_aparatus 5256.38 Medium 6.bin'
    fname_f='SP_aparatus 5256.38 Fast'
    ddir2='/media/morgan/6a0066c2-6c13-49f4-9322-0b818dda2fe8/Princeton Data/South Pole/SP_motor 5253.67 Bundle/'
    fname2='SP_motor 5253.67 sensors'
    fpath=ddir+fname
    fpath_f=ddir+fname_f

    dat=loadABin(fpath, useMetaFile=True);
    dat_f=loadBins(fpath_f, useMetaFile=False, totalDataType=[('sd','>f8'), ('s1', '>f4')]);
    dat2=loadBins(ddir+'SP_aparatus 5256.38 Medium')
    dat3=loadBins(ddir2+fname2, useMetaFile=True)


#Importing needed modules
import numpy as np
import matplotlib.pyplot as plt

class grid_environment():

    def __init__(self, path = "grid20.txt", epi_length_limit = 10000):

        mdppath = 'mdp.txt'

        #storing the file as strings line by line
        mdpdata=[]

        #Saving arm true means to the array-band (indices indicates arms)
        mdp = open(str(mdppath), "r")
        for x in mdp:
            mdpdata.append(x)
        mdp.close()
        #Closing mdp file.

        #Determining the nature of mdp
        mdptype1=[]
        for word in mdpdata[-2].split():
            try:
                mdptype1.append(str(word))
            except (ValueError, IndexError):
                pass
        mdptype=mdptype1[1]

        #Discount factor
        gamma1=[]
        for word in mdpdata[-1].split():
            try:
                gamma1.append(float(word))
            except (ValueError, IndexError):
                pass

        self.gamma=float(gamma1[0])

        #Number of states
        states=[]
        for word in mdpdata[0].split():
            try:
                states.append(int(word))
            except (ValueError, IndexError):
                pass
        self.numS=int(states[0])
        #Number of actions
        actions=[]
        for word in mdpdata[1].split():
            try:
                actions.append(int(word))
            except (ValueError, IndexError):
                pass
        self.numA=int(actions[0])
        #Start state
        start=[]
        for word in mdpdata[2].split():
            try:
                start.append(int(word))
            except (ValueError, IndexError):
                pass

        self.startS=int(start[0])
        #Terminal states for episodic mdps
        if(mdptype=='episodic'):
            terminal=[]
            for word in mdpdata[3].split():
                try:
                    terminal.append(int(word))
                except (ValueError, IndexError):
                    pass
            self.no_of_termS = len(terminal)
            self.terS = terminal

        #T-matrix dimensions numS*numS*numA
        #R-matrix dimensions numS*numA*numS

        self.T = np.zeros((self.numS,self.numA,self.numS))
        self.R = np.zeros((self.numS,self.numA,self.numS))

        for i in range(4,len(mdpdata)-2):
            trans=[]
            for word in mdpdata[i].split():
                try:
                    trans.append(float(word))
                except (ValueError, IndexError):
                    pass
            trans
            s1=int(trans[0])
            ac=int(trans[1])
            s2=int(trans[2])
            r=float(trans[3])
            p=float(trans[4])
            self.T[s1,ac,s2]=p
            self.R[s1,ac,s2]=r

        #useful variables
        #numS,numA,startS,terS,mdptype,gamma,T,R
        self.path = path
        self.info = None
        self.tau = 0
        self.tau_limit = epi_length_limit
        self.termination = False
        self.truncation = False

        #Encoder
        #Importing needed modules

        # #Reading arguments for program from shell call
        gridpath = path

        #storing the file as strings line by line
        griddata=[]

        #Saving arm true means to the array-band (indices indicates arms)
        grid = open(str(gridpath), "r")
        for x in grid:
            griddata.append(x)
        grid.close()
        #Closing mdp file

        #Grid Representation
        grid=[]
        for i in range(len(griddata)):
            row=[]
            for word in griddata[i].split():
                try:
                    row.append(int(word))
                except (ValueError, IndexError):
                    pass
            grid.append(row)

        n=len(grid)

        self.allstates=[]
        nonterm=[]
        self.validstates=[]
        endstates=[]
        startstate=[]

        for i in range(n):
            for j in range(n):

                self.allstates.append((i,j))
                if(grid[i][j]!=1):
                    self.validstates.append((i,j))
                if(grid[i][j]!=1 and grid[i][j]!=3):
                    nonterm.append((i,j))
                if(grid[i][j]==2):
                    startstate.append((i,j))
                if(grid[i][j]==3):
                    endstates.append((i,j))

    #to get the state number corresponding to the coordinate of a valis state
    def ctostates(self,x,y):
        s=0
        for i in range(self.numS):
            if(self.validstates[i][0]==x and self.validstates[i][1]==y):
                break
            s=s+1
        return(s)

    def ctoallstates(self,x,y):
        s=0
        for i in range(self.numS):
            if(self.allstates[i][0]==x and self.allstates[i][1]==y):
                break
            s=s+1
        return(s)

    def statestoc(self,s):
        return [self.validstates[s][0], self.validstates[s][1]]

    def allstatestoc(self,s):
        return [self.allstates[s][0], self.allstates[s][1]]

    def render(self, animation_save = False, N = 0, tau = 0):
        plt.figure(figsize=(3.5, 3.5))
        grid = np.loadtxt(self.path, dtype = int)
        [x,y] = self.statestoc(self.current_state)
        [sx, sy] = self.statestoc(self.startS)
        grid[sx,sy] = 0
        grid[x][y] = 4
        plt.imshow(grid, cmap=plt.cm.CMRmap, interpolation='nearest') #
        plt.xticks([]), plt.yticks([])
        if(animation_save == True):
            # Figure figure when rendered in environment
            plt.savefig(f'./img_N_{N}/img_{tau}.png')
        else:
            plt.show()

    def render_c_matrix(self, c):
        plt.figure(figsize=(3.5, 3.5))
        if c is not None:
            grid = c
        plt.imshow(grid, cmap=plt.cm.CMRmap, interpolation='nearest')
        plt.xticks([]), plt.yticks([])
        plt.show()

    def reset(self, seed = 10, randomise_goal = False):

        self.startS = np.random.randint(0,self.numS)
        if(randomise_goal):
            self.end_state = np.random.randint(0,self.numS)

        while(self.startS == self.end_state):
            self.startS = np.random.randint(0,self.numS)
            if(randomise_goal):
                self.end_state = np.random.randint(0,self.numS)

        self.T[self.end_state,:,self.end_state] = 1

        self.current_state = self.startS
        self.termination = False
        self.truncation = False
        self.tau = 0
        return self.current_state, self.info

    def step(self, action):
        self.tau += 1
        n_s = np.argmax(self.T[self.current_state, action, :])
        reward = self.R[self.current_state, action, n_s]
        if(n_s == self.end_state):
            self.termination = True
            reward = 10
        if(self.tau > self.tau_limit):
            self.truncation = True
        self.current_state = n_s
        return n_s, reward, self.termination, self.truncation, self.info

    def get_trueB(self):
        true_B = np.zeros((self.numS, self.numS, self.numA))
        for i in range(self.numS):
            for j in range(self.numA):
                true_B[:,i,j] = self.T[i,j,:]
        return true_B

all trial and error/backup_15.06.2023/4_grid_stochastic_mutating_goal/grid_env.py


import torch
import numpy as np

from torch.utils.data import Dataset
import cv2
from visdrone_db import VisDroneDataBase

class VisDroneDataset(Dataset):
    def __init__(self, path, device='cpu'):
        self.db = VisDroneDataBase(path)
        self.device= device

    def __getitem__(self, index):
        numpy_img, annotations = self.db[index]
        # w h c -> c w h
        numpy_img = numpy_img.transpose((2, 0, 1))
        numpy_img = np.asarray(numpy_img, dtype=np.float32) / 255
        [bbox, cls_label, info] = annotations
        bbox = np.asarray(bbox, dtype=np.float32)
        numpy_img = torch.from_numpy(numpy_img)
        bbox = torch.from_numpy(bbox)
        cls_label = torch.LongTensor(cls_label)
        target = {"boxes":bbox.to(self.device), "labels":cls_label.to(self.device),
                  "scores": torch.Tensor(info[0]).to(self.device)}
        return numpy_img.to(self.device), target

    def __len__(self):
        return len(self.db)

def collater(data):
    imgs = [s['img'] for s in data]
    annots = [s['annot'] for s in data]
    scales = [s['scale'] for s in data]

    imgs = torch.from_numpy(np.stack(imgs, axis=0))

    max_num_annots = max(annot.shape[0] for annot in annots)

    if max_num_annots > 0:

        annot_padded = torch.ones((len(annots), max_num_annots, 5)) * -1

        for idx, annot in enumerate(annots):
            if annot.shape[0] > 0:
                annot_padded[idx, :annot.shape[0], :] = annot
    else:
        annot_padded = torch.ones((len(annots), 1, 5)) * -1

    imgs = imgs.permute(0, 3, 1, 2)

    return {'img': imgs, 'annot': annot_padded, 'scale': scales}

class Resizer(object):
    """
    Convert ndarrays in sample to Tensors.
    """

    def __init__(self, img_size=512):
        self.img_size = img_size

    def __call__(self, sample):
        image, annots = sample['img'], sample['annot']
        height, width, _ = image.shape
        if height > width:
            scale = self.img_size / height
            resized_height = self.img_size
            resized_width = int(width * scale)
        else:
            scale = self.img_size / width
            resized_height = int(height * scale)
            resized_width = self.img_size

        image = cv2.resize(image, (resized_width, resized_height), interpolation=cv2.INTER_LINEAR)

        new_image = np.zeros((self.img_size, self.img_size, 3))
        new_image[0:resized_height, 0:resized_width] = image

        annots[:, :4] *= scale

        return {'img': torch.from_numpy(new_image).to(torch.float32), 'annot': torch.from_numpy(annots), 'scale': scale}

class Augmenter(object):
    """Convert ndarrays in sample to Tensors."""

    def __call__(self, sample, flip_x=0.5):
        if np.random.rand() < flip_x:
            image, annots = sample['img'], sample['annot']
            image = image[:, ::-1, :]

            rows, cols, channels = image.shape

            x1 = annots[:, 0].copy()
            x2 = annots[:, 2].copy()

            x_tmp = x1.copy()

            annots[:, 0] = cols - x2
            annots[:, 2] = cols - x_tmp

            sample = {'img': image, 'annot': annots}

        return sample

class Normalizer(object):

    def __init__(self, mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)):
        self.mean = np.array([[mean]])
        self.std = np.array([[std]])

    def __call__(self, sample):
        image, annots = sample['img'], sample['annot']

        return {'img': ((image.astype(np.float32) - self.mean) / self.std), 'annot': annots}


from PyQt5 import QtWidgets
from trainer.app.controller.app_controller import AppController
from trainer.app.model.app_model import AppModel
from trainer.app.view.app_view import AppView

RU = 'ru'

def main():
    import sys
    app = QtWidgets.QApplication(sys.argv)
    main_window = QtWidgets.QMainWindow()

    model = AppModel()
    model.set_up_parameters()
    view = AppView(main_window, model.load_locale(model.locale))
    AppController(model, view)

    main_window.show()
    sys.exit(app.exec_())

if __name__ == "__main__":
    main()


from django.contrib import admin
from django.utils.safestring import mark_safe
from django.utils.translation import gettext_lazy as _
from .models import (
    Chat,
    GroupChat,
    Message,
    TextMessage,
    ImageMessage,
    FileMessage,
)

class GroupChatInline(admin.TabularInline):
    """ Group Chat in line for ChatAdmin """
    model = GroupChat
    extra = 0

@admin.register(Chat)
class ChatAdmin(admin.ModelAdmin):
    """ Chat Model for admin """
    list_display = ('uuid', 'chat_type')
    radio_fields = {'chat_type': admin.HORIZONTAL}
    inlines = (GroupChatInline,)

class TextMessageInline(admin.TabularInline):
    """ TextMessage in line for MessageAdmin """
    model = TextMessage
    extra = 0

class ImageMessageInline(admin.TabularInline):
    """ ImageMessage in line for MessageAdmin """
    model = ImageMessage
    extra = 0
    readonly_fields = ('get_preview',)

    def get_preview(self, obj):
        return mark_safe(f'<img src="{obj.content.url}" height="100">')
    get_preview.short_description = _('Preview')

class FileMessageInline(admin.TabularInline):
    """ FileMessage in line for MessageAdmin """
    model = FileMessage
    extra = 0

@admin.register(Message)
class MessageAdmin(admin.ModelAdmin):
    """ Message Model for admin """
    list_display = ('uuid', 'chat', 'sender', 'msg_type', 'created_at')
    readonly_fields = ('created_at', 'updated_at')
    inlines = (TextMessageInline, ImageMessageInline, FileMessageInline)


from fastapi import FastAPI, WebSocket
from fastapi.responses import HTMLResponse
from fastapi.middleware.cors import CORSMiddleware
from schemas import ChatResponse

app = FastAPI()

app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

# resp = ChatResponse(sender="bot", message=token, type="stream")
import json

def receiverHandler(received_data):
    try:
        return ChatResponse(
            sender=received_data.get("sender"),
            message=received_data.get("message"),
            type=received_data.get("type"),
        )
    except Exception as e:
        print(e)

import asyncio

@app.websocket("/ws")
async def websocket_endpoint(websocket: WebSocket):
    await websocket.accept()
    res = """
       Hello, I am a bot. I can help you with the following:
       Rap, talk shit
       Smoke fatties!
    """
    while True:
        data = await websocket.receive_text()
        received = receiverHandler(json.loads(data))
        print(received)
        for i in res:
            response = ChatResponse(sender="bot", message=i, type="stream")
            await asyncio.sleep(0.05)
            await websocket.send_json(response.dict())
        await websocket.send_json(
            ChatResponse(sender="bot", message="MSG_DONE", type="stream").dict()
        )


djobradovic@singidunum.ac.rs

recnik = {
    'a':0,
    'b':0
}

def analiziraj_liniju(linija, recnik):
    for t in linija:
        t = t.lower()
        if t in recnik:
            recnik[t] += 1
        else:
            recnik[t] = 1

c = 0
with open('hobit.txt') as file:
    for linija in file:
        analiziraj_liniju(linija, recnik)
print(recnik)

najveci = None
for rec in recnik:
    print(rec, recnik[rec])
    if rec != ' ':
        if najveci == None:
            najveci = {
                'karakter': rec,
                'broj ponavljanja':recnik[rec]
            }
        else:
            if recnik[rec] > najveci['broj ponavljanja']:
                najveci = {
                    'karakter': rec,
                    'broj ponavljanja':recnik[rec]
                }

print(najveci)

print('U datoteci ima {} karaktera'.format(c))


from heapq import heappushpop, heappush, heappop

class MedianFinder:

    # to find median fast, we need a sorted array. how do we get a sorted array for a running stream of inputs?
    # think of the different type of sorting algorithm. cant be we run sort() everytime a new number is added to the array.
    # the only possible sorting algorithm to use is heap sort.
    # the trick is then to realise that to find the median of a sorted array, we need to know the middle element in the array
    # to do this, we use 2 heaps, the first heap to store numbers on the smaller half of the sorted array and the other to store the larger numbers
    # note that for the heap storing the smaller half, we are using max heap while for the heap storing the larger half, we are using min heap
    # since we are using python heapq, the default is a min heap. to convert to a max heap, multiply the value by -1
    # next, note that at any one point in time, both the heaps should have either the same size, or one heap larger by 1
    # if one heap is larger than the other by 1 element, we know that the total number of elements in the array is odd and the median value
    # is the min / max of the heap with 1 extra element
    # for my implementation, i am using the max heap to store the extra element if there are odd number of elements
    def __init__(self):
        self.max_heap = []
        self.min_heap = []

    # https://leetcode.com/problems/find-median-from-data-stream/discuss/1330646/C%2B%2BJavaPython-MinHeap-MaxHeap-Solution-Picture-explain-Clean-and-Concise
    # for the implementation of add number, the pseudo code is as follows:
    # first, we always add the number to the max_heap
    # then, we will take out the largest number in max_heap and put it into min_heap
    # this is to account for the chance that the newly added number should belong to the larger half of the array (on the min_heap side)
    # since we do this, there is a chance that the size of min heap is larger than max heap.
    # remember that at any one time, the min heap size should either be same size as max heap or max_heap.size-1 as we are storing the odd number element on max heap
    # if the size is not balanced to the afore mentioned requirements, we pass back 1 element from the min_heap to the max_heap
    def addNum(self, num: int) -> None:
        largest_in_max_heap = heappushpop(self.max_heap, num*-1)
        heappush(self.min_heap, largest_in_max_heap*-1)
        if len(self.min_heap) > len(self.max_heap):
            smallest_in_min_heap = heappop(self.min_heap)
            heappush(self.max_heap, smallest_in_min_heap*-1)

    # finding the median is simple, if odd number, return the top element of the larger size heap
    # in my implementation, the larger size heap is always the max heap
    # if both heaps are the same size, return the average value of the top element in min and max heap
    def findMedian(self) -> float:
        if len(self.max_heap) > len(self.min_heap):
            return float(self.max_heap[0]*-1)
        assert len(self.max_heap) == len(self.min_heap)
        return (self.max_heap[0]*-1+self.min_heap[0])/2.0

if __name__ == "__main__":
    finder = MedianFinder()
    nums = [5,1,3,4,2]
    for num in nums:
        finder.addNum(num)
        print(finder.findMedian())

# Your MedianFinder object will be instantiated and called as such:
# obj = MedianFinder()
# obj.addNum(num)
# param_2 = obj.findMedian()


import ezdxf
from ezdxf.math import Vec3
from ezdxf.enums import MTextEntityAlignment

# This is the only way to create MTEXT entities where the "width" attribute
# is missing or equals 0.
# MTEXT entities created by AutoCAD or BricsCAD ALWAYS have a
# "width" attribute > 0.

CONTENT = "This is a long MTEXT line without line wrapping!\\PThe second line."

doc = ezdxf.new(setup=True)
doc.layers.new("MTEXT", dxfattribs={"color": ezdxf.const.RED})
msp = doc.modelspace()
attribs = {
    "char_height": 0.7,
    "style": "OpenSans",
    "layer": "MTEXT",
}

def add_mtext(
    location: Vec3, attachment_point: MTextEntityAlignment, size: float = 2
):
    msp.add_line(location - (size, 0), location + (size, 0))
    msp.add_line(location - (0, size), location + (0, size))
    mtext = msp.add_mtext(CONTENT, attribs)
    mtext.set_location(location, attachment_point=attachment_point)

params = [
    ((0, 0), MTextEntityAlignment.BOTTOM_LEFT),
    ((100, 0), MTextEntityAlignment.BOTTOM_CENTER),
    ((200, 0), MTextEntityAlignment.BOTTOM_RIGHT),
    ((0, 100), MTextEntityAlignment.MIDDLE_LEFT),
    ((100, 100), MTextEntityAlignment.MIDDLE_CENTER),
    ((200, 100), MTextEntityAlignment.MIDDLE_RIGHT),
    ((0, 200), MTextEntityAlignment.TOP_LEFT),
    ((100, 200), MTextEntityAlignment.TOP_CENTER),
    ((200, 200), MTextEntityAlignment.TOP_RIGHT),
]

for location, attachment_point in params:
    add_mtext(Vec3(location), attachment_point)

doc.set_modelspace_vport(300, (100, 100))
doc.saveas("mtext_simple_all_alignments.dxf")


from ex115.lib.interface import *
from ex115.lib.arquivo import *
from  time import sleep

arq = 'cursoemvideo.txt'

if not arquivoExite(arq):
    criarArquivo(arq)

while True:
    resposta = menu(['Ver pessoas cadastradas','Cadastrar pessoa','Sair do sitema'])
    if resposta == 1:
        # opeção de lista um conteudo de um arquivo
        lerArquivo(arq)
    elif resposta == 2 :
        #opção de cadastra uma nova pessoa
        cabeçario('Novo Cadastro')
        nome = str(input('Nome: ')).strip()
        idade = leiaint('Idade: ')
        cadastrar(arq,nome,idade)
    elif resposta == 3:
        cabeçario('Saindo do sistema.. até logo')
        break
    else:
        print('\033[31mOpeção invalida tente novamente\033[m')
    sleep(2)

Alguns codigo feito no Curso de Python/modulos/ex115/sistema.py


def get_cookiecutter_config(template, default_config=None, version=None):
    """Obtains the configuration used for cookiecutter templating

    Args:
        template: Path to the template
        default_config (dict, optional): The default configuration
        version (str, optional): The git SHA or branch to use when
            checking out template. Defaults to latest version

    Returns:
        tuple: The cookiecutter repo directory and the config dict
    """
    default_config = default_config or {}
    config_dict = cc_config.get_user_config()
    repo_dir, _ = cc_repository.determine_repo_dir(
        template=template,
        abbreviations=config_dict['abbreviations'],
        clone_to_dir=config_dict['cookiecutters_dir'],
        checkout=version,
        no_input=True)
    context_file = os.path.join(repo_dir, 'cookiecutter.json')
    context = cc_generate.generate_context(
        context_file=context_file,
        default_context={**config_dict['default_context'], **default_config})
    return repo_dir, cc_prompt.prompt_for_config(context)


from bisect import bisect

import pandas as pd
import numpy as np

from mortgage_scenarios import MortgageLoanRunner, LoanPartIterator, get_monthly_rate
from mortgage_scenarios.core import group_by_year

"""
This script contains prototypes for experimental features
They are to be implemented in a proper way in the main code

experimental features
---------------------
LTV changes
-----------
loans with variable rates based on discounts that are computed monthy
based on the current LTV (loan to value)

interest rate scenario
---------------------
computed what-if scenario on interest rate after the fixed-rate period ends
(what if the rate is 5% when the new rate is negotiated)

prepayments
-----------
doing additional payments at given periods
"""

####
# Settings
####
start_period = '2020-10'
rates_addon = 0.000175

USE_LTV = True
houseprice = 631000
#####

def get_ltv_tranch(amount, houseprice):
    ltv_boundaries = (0.675, 0.9, 1.0)
    ltv_tranch_names = ('<67.5%', '67.5% - 90%', '90% - 100%', '>100%')
    ltv = amount / houseprice
    ltv_index = bisect(ltv_boundaries, ltv)
    return ltv_index, ltv_tranch_names[ltv_index]

PERIODS = 30  # noqa

pd.options.display.precision = 2
pd.options.display.float_format = '{:,.0f}'.format

rates = np.array([0.0215, 0.0195, 0.0195])

rates += rates_addon
fixed = 0
loan1 = LoanPartIterator(92500, rates[0], PERIODS, future=92500, fixed=fixed, yearly=True)
loan2 = LoanPartIterator(198183, rates[1], PERIODS, fixed=fixed, yearly=True)
loan3 = LoanPartIterator(144000, rates[2], PERIODS, fixed=fixed, yearly=True)
loans = [loan1, loan2, loan3]

# ----

mortgage = MortgageLoanRunner()
for loan in loans:
    mortgage.add_loanpart(loan)

print('total loan %.2f euro' % mortgage.current_amount)
# TODO: set rates based on current ltv
ltv_info = get_ltv_tranch(mortgage.current_amount, houseprice)

eventlog = []
while True:
    events = []

    if mortgage.period == 0:
        loan3.start_amount -= (400000 - 326000)
        loan3.reset()

    # ltv check
    new_ltv = get_ltv_tranch(mortgage.current_amount, houseprice)
    if USE_LTV and new_ltv != ltv_info and new_ltv[0] == 0:
        events.append('ltv < 67.5%')
        mortgage.replace_loanpart(loan1, loan1.new_loanpart_with_rate(
            get_monthly_rate(0.019)))
        mortgage.replace_loanpart(loan2, loan2.new_loanpart_with_rate(
            get_monthly_rate(.0175)))
        mortgage.replace_loanpart(loan3, loan3.new_loanpart_with_rate(
            get_monthly_rate(.0175)))
        ltv_info = new_ltv

    if mortgage.period == 20*12-1:
        # simulates interest jump
        new_rate = get_monthly_rate(0.05)
        for loan in mortgage.loanparts[:]:
            mortgage.replace_loanpart(loan, loan.new_loanpart_with_rate(new_rate))

    try:
        mortgage.step()
    except StopIteration:
        break

    # repayment check
    if mortgage.period in [10, 20]:
        print('do repayment')
        events.append('repayment')

    eventlog.append(";".join(events))

df = mortgage.to_dataframe()
df['events'] = eventlog
df['ltv'] = df['amount'] / houseprice

# print(df)
df2 = mortgage.to_dataframe()

df_agg = group_by_year(df2, start_period)
print(df_agg[['amount_end', 'interest', 'payment', 'repayment']])


#this scripty script will be for trying to scrape all the best g damn flights to yaurope

#notes
    #good thing is that the airport codes in the site are not case sensitive no no need to convert all strings into uppercase letters
    #CANNOT search more than ten airports at the same time, need to break up EU list into groups of 10
    #also, all airport codes entered must be separated by a comma or wont work

    #ok for the button issue, lamba method gives function memory as to what index in the loop should be called by what function
        #instead of just looking it up at run time it remembers what index it was assigned in the loop
        #this is good cause now each button opens a unique link instead of just the last link
        #but problem is it only calls it from the last list_of_links list created
        #so all buttons are linked in order but only to last created list_of_links list which is a problem when searching more than one date
        #i think the soln is to append each list_of_links list to another list so list within a list or nested list
        #then use same lambda method to remember both the index within the list but also the index of the list itself - kinda confusing
            #IT WORKED LETS FUCKING GO

    #STALE ELEMENT ERRORS:
        #occurs when element requested isnt attached to the page
        #i think shorter delays cause this to occur more frequently cause the page isnt fully loaded and its trying to pull all 15 flight cards
        #longer delay after giving the driver the URL but before requesting the elements will give it time to load and will probably fix the problem
        #only thing is that delay may be dependent on the users wifi speed and with slow ass wifi may not be sufficient

from glob import glob
from time import sleep
from urllib import request
from venv import create
import pandas as pd
from selenium import webdriver
from bs4 import BeautifulSoup
import os
from tkinter import *

from webdriver_manager.chrome import ChromeDriverManager

from tkcalendar import Calendar
from datetime import datetime

from datetime import timedelta, date
import webbrowser
import requests

#this package is for pulling out substring between two strings lol
import re
from tkinter import ttk

import unidecode #this is for getting rid of those weird \xa characters in the price data

#first thing to do is get current date to set on calendar widget
date_time = datetime.today()
year = date_time.year
month = date_time.month
day = date_time.day

#IMPORT ALL EU AIRPORTS
EU_airport_list = ['TIA', 'EVN', 'GRZ', 'INN', 'KLU', 'LNZ', 'SZG', 'VIE', 'GYD', 'MSQ', 'ANR', 'BRU', 'CRL',
'LGG', 'OST', 'SJJ', 'TZL', 'BOJ', 'SOF', 'VAR', 'DBV', 'PUY', 'SPU', 'ZAD', 'ZAG', 'LCA', 'PFO', 'BRQ', 'PRG',
'AAL', 'AAR', 'BLL', 'CPH', 'FAE', 'TLL', 'HEL', 'OUL', 'RVN', 'TMP', 'TKU', 'VAA', 'AJA', 'BIA', 'EGC', 'BIQ',
'BOD', 'BES', 'FSC', 'LIL', 'LYS', 'MRS', 'MPL', 'NTE', 'NCE', 'BVA', 'CDG', 'ORY', 'SXB', 'RNS', 'RUN', 'TLN',
'TLS', 'KUT', 'TBS', 'FMM', 'BER', 'SXF', 'TXL', 'BRE', 'CGN', 'DTM', 'DRS', 'DUS', 'FRA', 'HHN', 'FDH', 'HAM',
'HAJ', 'FKB', 'LEJ', 'MUC', 'FMO', 'NUE', 'PAD', 'STR', 'NRN', 'ATH', 'CHQ', 'CFU', 'HER', 'KGS', 'JMK', 'RHO',
'JTR', 'SKG', 'ZTH', 'BUD', 'DEB', 'KEF', 'ORK', 'DUB', 'NOC', 'KIR', 'SNN', 'AHO', 'AOI', 'BRI', 'BGY', 'BLQ',
'BDS', 'CAG', 'CTA', 'CIY', 'FLR', 'GOA', 'SUF', 'LIN', 'MXP', 'NAP', 'OLB', 'PMO', 'PEG', 'PSR', 'PSA', 'CIA',
'FCO', 'TPS', 'TSF', 'TRN', 'VCE', 'VRN', 'ALA', 'TSE', 'PRN', 'RIX', 'KUN', 'VNO', 'LUX', 'MLA', 'KIV', 'TGD',
'TIV', 'AMS', 'EIN', 'GRQ', 'MST', 'RTM', 'SKP', 'AES', 'BGO', 'BOO', 'HAU', 'KRS', 'OSL', 'TRF', 'SVG', 'TOS',
'TRD', 'GDN', 'KTW', 'KRK', 'POZ', 'SZZ', 'WAW', 'WMI', 'WRO', 'FAO', 'LIS', 'FNC', 'PDL', 'OPO', 'OTP', 'CLJ',
'IAS', 'TSR', 'SVX', 'KRR', 'DME', 'SVO', 'VKO', 'ZIA', 'OVB', 'LED', 'AER', 'BEG', 'INI', 'BTS', 'KSC', 'LJU',
'ALC', 'LEI', 'OVD', 'BCN', 'BIO', 'FUE', 'GRO', 'LPA', 'GRX', 'IBZ', 'XRY', 'SPC', 'ACE', 'MAD', 'AGP', 'MAH',
'PMI', 'RMU', 'REU', 'SDR', 'SCQ', 'SVQ', 'TFN', 'TFS', 'VLC', 'ZAZ', 'GOT', 'MMX', 'ARN', 'BMA', 'NYO', 'VST',
'BSL', 'BRN', 'GVA', 'ZRH', 'ADA', 'ESB', 'AYT', 'DLM', 'IST', 'SAW', 'ADB', 'BJV', 'TZX', 'HRK', 'KBP', 'IEV',
'LWO', 'ODS', 'ABZ', 'BHD', 'BFS', 'BHX', 'BRS', 'CWL', 'DSA', 'EMA', 'EDI', 'EXT', 'GLA', 'PIK', 'HUY', 'JER',
'LBA', 'LPL', 'LCY', 'LGW', 'LHR', 'LTN', 'SEN', 'STN', 'MAN', 'NCL', 'SOU']

#because there are two options for arrival airports (all EU, or typed response), use booleans to figure out which one was used

arrivals_were_typed = FALSE

arrivals_were_not_typed = FALSE

#######################################################CREATE ROOT WINDOW###########################################################
#make this into a GUI and then an executable
#create root app
root = Tk()
screen_width = root.winfo_screenwidth()
screen_height = root.winfo_screenheight()
adjusted_width = screen_width/1.3
adjusted_height = screen_height/1.3
screenWidth_int = int(adjusted_width)
screenHeight_int = int(adjusted_height)
screenWidth_String = str(screenWidth_int)
screenHeight_String = str(screenHeight_int)
gui_size = screenWidth_String + "x" + screenHeight_String
root.geometry(gui_size) #allows x,y size of GUI window to be set in terms of pixel number
root.resizable(True, True)
root.configure(background = 'old lace')

masterNotebook = ttk.Notebook(root)
masterNotebook.grid(row = 0, column = 0)
style = ttk.Style()
style.theme_use('default')
style.configure('TNotebook.Tab', background="gray70", font=('Helvetica','14'))
style.map("TNotebook", background= [("selected", "gray70")])

#now create the initial page
search_page = Frame(masterNotebook, width = adjusted_width, height = adjusted_height, bg = 'old lace')
search_page.pack(fill = "both", expand = 1)
masterNotebook.add(search_page, text= "SEARCH FLIGHTS")

#########################################################START BUILDING FRONT END###################################################
#need to have option for selecting date range, also airport (either all or space to type in specific ones)

################DATE SELECT################################################################
date_range_label = Label(search_page, text = "select date range", font = (40), bg = 'old lace')
date_range_label.place(relx = 0.15, rely = 0.15, anchor = "center")

#create calendar to select from
start_cal = Calendar(search_page, selectmode = 'day', year = year, month = month, day = day)
start_cal.place(relx = 0.15, rely = 0.3, anchor = "center")

def getStart():

    global start_date

    start_date = start_cal.get_date() #this is string type
    start_cal_button['state'] = DISABLED
    end_cal_button['state'] = NORMAL

start_cal_button = Button(search_page, text = "select start date", bg = 'cornflower blue', command = getStart)
start_cal_button.place(relx = 0.15, rely = 0.45, anchor = "center")

#NOW CREATE END CALENDAR
end_cal = Calendar(search_page, selectmode = 'day', year = year, month = month, day = day)
end_cal.place(relx = 0.15, rely = 0.6, anchor = "center")

def getEnd():
    end_date = end_cal.get_date() #this is string type
    end_cal_button['state'] = DISABLED
    departure_entry['state'] = NORMAL

    confirm_departure['state'] = NORMAL

    #in here want to properly strucutre travel date data so that it can be fed into the URL later on
    new_start_date = start_date.split('/')
    new_end_date = end_date.split('/')
    #now pull out day month year data
    start_month = int(new_start_date[0])
    start_day = int(new_start_date[1])
    start_year = int(new_start_date[2]) + 2000

    end_month = int(new_end_date[0])
    end_day = int(new_end_date[1])
    end_year = int(new_end_date[2]) + 2000

    #random method for day difference
    def daterange(date1, date2):
        for n in range(int((date2 - date1).days) + 1):
            yield date1 + timedelta(n)

    start_dt = date(start_year, start_month, start_day)
    end_dt = date(end_year, end_month, end_day)

    global formatted_dates_array #this will be accessed later

    formatted_dates_array = [] #this array will now contain all properly formatted strings for date in URL

    for dt in daterange(start_dt, end_dt):
        formatted_dates_array.append(dt.strftime("%Y-%m-%d"))

end_cal_button = Button(search_page, text = "select end date", bg = 'cornflower blue', command = getEnd)
end_cal_button.place(relx = 0.15, rely = 0.75, anchor = "center")
end_cal_button['state'] = DISABLED
###############################################################################################

#################DEPARTURE AIRPORTS################################################################
departure_airport_label = Label(search_page, text = "type departure airport code(s) - separate with ,", font = (40), bg = 'old lace')
departure_airport_label.place(relx = 0.5, rely = 0.22, anchor = "center")

departure_entry = Entry(search_page)
departure_entry.place(relx = 0.5, rely = 0.28, height = 30, width = 400, anchor = "center")
departure_entry['state'] = DISABLED

#now create button for submitting departure options
def submitDeparture():
    departure_entry['state'] = DISABLED
    confirm_departure['state'] = DISABLED

    arrival_entry['state'] = NORMAL
    #all_airports_button['state'] = NORMAL #haven't built this function yet, leave this disabled for now
    confirm_arrival['state'] = NORMAL

    global DEPARTURE_AIRPORTS

    #now get entry as string for IATA codes
    DEPARTURE_AIRPORTS = departure_entry.get()

confirm_departure = Button(search_page, text = "confirm departure airports", bg = 'cornflower blue', command = submitDeparture)
confirm_departure.place(relx = 0.5, rely = 0.35, anchor = "center")
confirm_departure['state'] = DISABLED

#also maybe create button to database to search airport codes
def searchIATA():
    webbrowser.open('https://www.iata.org/en/publications/directories/code-search/')

IATA_button = Button(search_page, text = "search IATA codes", command = searchIATA)
IATA_button.place(relx = 0.5, rely = 0.15, anchor = "center")
#####################################################################################################

################ARRIVAL AIRPORTS####################################################################
arrival_airport_label = Label(search_page, text = "type arrival airport code(s) - separate with ,", font = (40), bg = 'old lace')
arrival_airport_label.place(relx = 0.8, rely = 0.29, anchor = "center")

arrival_entry = Entry(search_page)
arrival_entry.place(relx = 0.8, rely = 0.35, height = 30, width = 400, anchor = "center")
arrival_entry['state'] = DISABLED

def submitArrival():
    confirm_arrival['state'] = DISABLED
    all_airports_button['state'] = DISABLED
    arrival_entry['state'] = DISABLED

    cheapest_button['state'] = NORMAL
    fastest_button['state'] = NORMAL
    best_button['state'] = NORMAL

    global ARRIVAL_AIRPORTS

    #grab entry as string
    ARRIVAL_AIRPORTS = arrival_entry.get()
    #now set the boolean true
    global arrivals_were_typed
    arrivals_were_typed = TRUE

confirm_arrival = Button(search_page, text = "confirm arrival airports", bg = 'cornflower blue', command = submitArrival)
confirm_arrival.place(relx = 0.8, rely = 0.42, anchor = "center")
confirm_arrival['state'] = DISABLED

def allAirports(): #pretty obvious whats gonna happen in here
    confirm_arrival['state'] = DISABLED
    all_airports_button['state'] = DISABLED
    arrival_entry['state'] = DISABLED

    #search_button['state'] = NORMAL

    global ARRIVAL_AIRPORTS

    global ARRIVAL_AIRPORTS_0
    global ARRIVAL_AIRPORTS_1
    global ARRIVAL_AIRPORTS_2
    global ARRIVAL_AIRPORTS_3
    global ARRIVAL_AIRPORTS_4
    global ARRIVAL_AIRPORTS_5
    global ARRIVAL_AIRPORTS_6
    global ARRIVAL_AIRPORTS_7
    global ARRIVAL_AIRPORTS_8
    global ARRIVAL_AIRPORTS_9
    global ARRIVAL_AIRPORTS_10
    global ARRIVAL_AIRPORTS_11
    global ARRIVAL_AIRPORTS_12
    global ARRIVAL_AIRPORTS_13
    global ARRIVAL_AIRPORTS_14
    global ARRIVAL_AIRPORTS_15
    global ARRIVAL_AIRPORTS_16
    global ARRIVAL_AIRPORTS_17
    global ARRIVAL_AIRPORTS_18
    global ARRIVAL_AIRPORTS_19
    global ARRIVAL_AIRPORTS_20
    global ARRIVAL_AIRPORTS_21
    global ARRIVAL_AIRPORTS_22
    global ARRIVAL_AIRPORTS_23
    global ARRIVAL_AIRPORTS_24
    global ARRIVAL_AIRPORTS_25
    global ARRIVAL_AIRPORTS_26

    #now set arrival airports to all EU airports
    ARRIVAL_AIRPORTS = EU_airport_list
    #need to split into groups of 10 to search with
    ARRIVAL_AIRPORTS_0 = ARRIVAL_AIRPORTS[0:10]
    ARRIVAL_AIRPORTS_1 = ARRIVAL_AIRPORTS[10:20]
    ARRIVAL_AIRPORTS_2 = ARRIVAL_AIRPORTS[20:30]
    ARRIVAL_AIRPORTS_3 = ARRIVAL_AIRPORTS[30:40]
    ARRIVAL_AIRPORTS_4 = ARRIVAL_AIRPORTS[40:50]
    ARRIVAL_AIRPORTS_5 = ARRIVAL_AIRPORTS[50:60]
    ARRIVAL_AIRPORTS_6 = ARRIVAL_AIRPORTS[60:70]
    ARRIVAL_AIRPORTS_7 = ARRIVAL_AIRPORTS[70:80]
    ARRIVAL_AIRPORTS_8 = ARRIVAL_AIRPORTS[80:90]
    ARRIVAL_AIRPORTS_9 = ARRIVAL_AIRPORTS[90:100]
    ARRIVAL_AIRPORTS_10 = ARRIVAL_AIRPORTS[100:110]
    ARRIVAL_AIRPORTS_11 = ARRIVAL_AIRPORTS[110:120]
    ARRIVAL_AIRPORTS_12 = ARRIVAL_AIRPORTS[120:130]
    ARRIVAL_AIRPORTS_13 = ARRIVAL_AIRPORTS[130:140]
    ARRIVAL_AIRPORTS_14 = ARRIVAL_AIRPORTS[140:150]
    ARRIVAL_AIRPORTS_15 = ARRIVAL_AIRPORTS[150:160]
    ARRIVAL_AIRPORTS_16 = ARRIVAL_AIRPORTS[160:170]
    ARRIVAL_AIRPORTS_17 = ARRIVAL_AIRPORTS[170:180]
    ARRIVAL_AIRPORTS_18 = ARRIVAL_AIRPORTS[180:190]
    ARRIVAL_AIRPORTS_19 = ARRIVAL_AIRPORTS[190:200]
    ARRIVAL_AIRPORTS_20 = ARRIVAL_AIRPORTS[200:210]
    ARRIVAL_AIRPORTS_21 = ARRIVAL_AIRPORTS[210:220]
    ARRIVAL_AIRPORTS_22 = ARRIVAL_AIRPORTS[220:230]
    ARRIVAL_AIRPORTS_23 = ARRIVAL_AIRPORTS[230:240]
    ARRIVAL_AIRPORTS_24 = ARRIVAL_AIRPORTS[240:250]
    ARRIVAL_AIRPORTS_25 = ARRIVAL_AIRPORTS[250:260]
    ARRIVAL_AIRPORTS_26 = ARRIVAL_AIRPORTS[260:270]

    #now set boolean true
    global arrivals_were_not_typed
    arrivals_were_not_typed = TRUE

all_airports_button = Button(search_page, text = "all EU airports", bg = 'cornflower blue', command = allAirports)
all_airports_button.place(relx = 0.8, rely = 0.5, anchor = "center")
all_airports_button['state'] = DISABLED #this should be unlocked after the other two things are dealt with

or_label = Label(search_page, text = "OR", font = (50), bg = 'old lace')
or_label.place(relx = 0.8, rely = 0.46, anchor = "center")

#CREATE BUTTONS TO SELECT BETWEEN CHEAPEST, FASTEST, AND BEST FLIGHTS
def cheapestFunction():
    cheapest_button['state'] = DISABLED
    fastest_button['state'] = DISABLED
    best_button['state'] = DISABLED

    search_button['state'] = NORMAL

    global search_type
    search_type = 'price_a'

cheapest_button = Button(search_page, text = "cheapest", bg = 'cornflower blue', command = cheapestFunction)
cheapest_button.place(relx = 0.43, rely = 0.6, anchor = "center")
cheapest_button['state'] = DISABLED

def fastestFunction():
    cheapest_button['state'] = DISABLED
    fastest_button['state'] = DISABLED
    best_button['state'] = DISABLED

    search_button['state'] = NORMAL

    global search_type
    search_type = 'duration_a'

fastest_button = Button(search_page, text = "fastest", bg = 'cornflower blue', command = fastestFunction)
fastest_button.place(relx = 0.5, rely = 0.6, anchor = "center")
fastest_button['state'] = DISABLED

def bestFunction():
    cheapest_button['state'] = DISABLED
    fastest_button['state'] = DISABLED
    best_button['state'] = DISABLED

    search_button['state'] = NORMAL

    global search_type
    search_type = 'bestflight_a'

best_button = Button(search_page, text = "best", bg = 'cornflower blue', command = bestFunction)
best_button.place(relx = 0.57, rely = 0.6, anchor = "center")
best_button['state'] = DISABLED
########################################################################################################

###############################################START BUILDING BACKEND########################################################################

#scrape price, departure and arrival location, trip time, and link to offer

all_links = [] #defined outside the function so the data in this array doesn't get overwritten

list_of_lists = []

def searchFlights():
    search_button['state'] = DISABLED

    #THERE ARE TWO POSSIBLE CASES IN HERE
        #one is where arrivals_were_typed = TRUE
        #the other is where arrivals_were_not_typed = TRUE
        #searching will be different for these two methods
    #now here do all the scraping shyte

    dates_iterator = 0

    while dates_iterator <= len(formatted_dates_array) - 1:

        travel_date = formatted_dates_array[dates_iterator] #this will be stuffed into the link

        try: #to handle the stupid element is not attached to the page error

            if arrivals_were_typed == TRUE:

                driver = webdriver.Chrome(ChromeDriverManager().install())
                url = f'https://www.ca.kayak.com/flights/{DEPARTURE_AIRPORTS}-{ARRIVAL_AIRPORTS}/{travel_date}?sort={search_type}'

                driver.get(url)
                sleep(5) #the smaller the delay, seems like more stale element errors occur, with 5 seconds they occur every time. With 10 seconds only sometimess

                #now can begin to pull out flight info
                flight_rows = driver.find_elements("xpath", '//div[@class="inner-grid keel-grid"]') #comes from inspecting the element of the flight card on the website
                #print(flight_rows) #sweet it works and is pulling the web elements out on the page

                global list_prices
                global start_and_end_location
                global trip_time
                global offer_link

                list_prices = []
                start_and_end_location = []
                trip_time = []
                offer_link = []

                for WebElement in flight_rows:
                    elementHTML = WebElement.get_attribute('outerHTML')
                    elementSoup = BeautifulSoup(elementHTML, 'html.parser')

                    #extract prices
                    temp_price = elementSoup.find("div", {"class": "col-price result-column js-no-dtog"}) #again from inspecting the element
                    price = temp_price.find("span", {"class": "price option-text"})
                    #this could be confusing but basically this span tag price thing is statically created whereas the actualy price text is dynamically created
                        #cant automate the dynamically created extraction so went up one element within the element heirarchy
                        #this element is statically created but is used all across the page on every flight card
                        #howeveer, since we are laready working within an element, this should only exist once within this element and so we can call it and extract the data i think

                    list_prices.append(price.text) #so append to the price list and pull out the text

                    #departure and arrival location
                    location = elementSoup.find("div", {"class": "bottom"})
                    start_and_end_location.append(location.text)

                    #trip time
                    time = elementSoup.find("div", {"class": "section duration allow-multi-modal-icons"})
                    trip_time.append(time.text)

                    #finally, try to get link to offer so user can click from results, for this need the HREF i think but not sure how to get it
                    #use xpath of button?, in the HTML look for the object that says role = button since thats the actual button
                    #sleep(5)
                    #view_deal_button = '//*[@id="c4wGi-mb-bE-130ff2b47dc"]' #right click copy xpath to get this

                    #offer_link.append(elementSoup.find_all(['a'],{'class':['booking-link']}))
                    offer_link.append(elementSoup.find_all(href = True)) #this method is way better and actually pulls out the correct number of href links unlike the other one that pulls out 45 ish
                    #links embedded in the output
                        #need to have kayak.,com or whatever added to the string.
                        #also, every & symbol in the link gets an added 'amp;' for some weird reason
                        #so that needs to be .replace("amp;", "") before the link can be clickable by the user

                        #since there are 15 objects per page, the length of offer_link is 15, can loop through each index and then figure out how to pull out the link somehow
                            #maybe step one is to convert each index into text or string somehow then do the string search to find the link

                #before converting to string, need to separate out all indeces from the object
                #maybe just append to a list of strings

                list_of_strings = []
                i = 0

                while i <= len(offer_link) - 1:

                    list_of_strings.append(str(offer_link[i]))

                    i = i + 1

                #now with list of strings all categorized can loop through the new list and pull out the links
                list_of_links = []
                k = 0

                kayak_string = 'https://www.ca.kayak.com'

                while k <= len(list_of_strings) - 1: #LETS FUCKING GO THIS SHIT WORKS

                    #use the re library to syphon out the links lol
                    temp_link = re.search('href="(.*)" id="', list_of_strings[k])

                    temp_flight_link = kayak_string + temp_link.group(1)
                    #now remove all the amperand crap
                    flight_link = temp_flight_link.replace('amp;', '') #replace amp; with nothing to take it out of link

                    list_of_links.append(flight_link)

                    all_links.append(flight_link) #this list will contain every link from the whole search and will be used to generate the buttons

                    k = k + 1

                #now append the list of links to the list of lists
                list_of_lists.append(list_of_links) #now each index of this should contain an entire list of links...i think

                #need to clean the outputs - links are good but the other three are full of junk
                    #then maybe can write to a file or something, still not sure how the data should be presented to the user
                #list_prices
                #start_and_end_location
                #trip_time

                list_prices_cleaned = []
                derp = 0

                while derp <= len(list_prices) - 1:

                    temp_val = list_prices[derp]
                    temp_val_0 = temp_val.replace('\n', '')
                    temp_val_0 = unidecode.unidecode(temp_val_0) #gets rid of the \xa character

                    list_prices_cleaned.append(temp_val_0)

                    derp = derp + 1

                list_locations_cleaned = []
                derpderp = 0

                while derpderp <= len(start_and_end_location) - 1:

                    temp_val = start_and_end_location[derpderp]
                    list_locations_cleaned.append(temp_val.replace('\n', ''))

                    derpderp = derpderp + 1

                list_trip_time_cleaned = []
                derpderpderp = 0

                while derpderpderp <= len(trip_time) - 1:

                    temp_val = trip_time[derpderpderp]
                    list_trip_time_cleaned.append(temp_val.replace('\n',''))

                    derpderpderp = derpderpderp + 1

                #now should have all cleaned strings
                #NEED TO ADD IN DATE FOR THESE FLIGHTS BUT THIS CAN COME FROM USER SELECTION ON CALENDAR AS NOT ON FLIGHT CARD ON SITE
                    #now need to figure out how to display data
                    #maybe make a pop up window with an option to export to csv

                create_new_page = Frame(masterNotebook, width = adjusted_width, height = adjusted_height, bg = 'old lace')
                create_new_page.pack(fill = "both", expand = 1)
                masterNotebook.add(create_new_page, text= f"{formatted_dates_array[dates_iterator]}") #now this should create new tab w title of each date selected in the date range picker
                root.update() #push the update onto the screen

                ###########NOW CAN START ADDING IN EACH FLIGHT RESULT INTO THE PAGE###############
                #results_iterator = 0
                y_start_val = 0.15

                for i in range(len(list_of_links)): #the list could be any of them just chose links arbitrarily

                    #FIRST SHOW DEPARTING AND ARRIVING AIRPORT
                    location_label = Label(create_new_page, text = f"{list_locations_cleaned[i]}", bg = 'old lace')
                    location_label.place(relx = 0.2, rely = y_start_val, anchor = "center")

                    #NOW DO PRICE
                    price_label = Label(create_new_page, text = f"{list_prices_cleaned[i]}", bg = 'old lace')
                    price_label.place(relx = 0.4, rely = y_start_val, anchor = "center")

                    #NOW TRIP TIME
                    trip_time_label = Label(create_new_page, text = f"{list_trip_time_cleaned[i]}", bg = 'old lace')
                    trip_time_label.place(relx = 0.6, rely = y_start_val, anchor = "center")

                    #try again the button crap lol
                    def linkFunction(var, new_var):
                        #pull out list from list of lists
                        temp_list = list_of_lists[new_var]
                        webbrowser.open(temp_list[var])

                    link_button = Button(create_new_page, text = "Link to Offer", bg = 'cornflower blue', command = lambda var = i, new_var = dates_iterator: linkFunction(var, new_var))
                    link_button.place(relx = 0.8, rely = y_start_val, anchor = "center")

                    #results_iterator = results_iterator + 1
                    y_start_val = y_start_val + 0.05

                dates_iterator = dates_iterator + 1 #i think this goes here but not sure tbh lol

            else: #this is then the case where all EU airports were selected as arrivals which the code is gonna be harder for this section
                #for now just work on the typed arrivals section
                pass

        except Exception as e:
            pass #since the stale element error occurs before the iterator is reached, it never actually increments up
            #so when the computer passes through the exception it goes back to start of while loop and the iterator it reads is the same as before
            #so it just keeps retrying until it passes which is what was desired
            print("error occured") #doesn't really do anything, just to see how many errors occur

    #buttons have to be created outside the while loop otherwise they keep getting overwritten
    #here create buttons based on filled list of all links pulled across all pages and dates

search_button = Button(search_page, text = "search flights", fg = 'blue', bg = 'dark goldenrod', font = 'bold', command = searchFlights)
search_button.place(relx = 0.5, rely = 0.7, anchor = 'center', height = 40, width = 100)
search_button['state'] = DISABLED

#add in reset page button
def resetSearch():
    #in here would reset all buttons and such to make another search without having to quit the app

    global arrivals_were_typed
    global arrivals_were_not_typed

    arrivals_were_typed = FALSE
    arrivals_were_not_typed = FALSE

reset_button = Button(search_page, text = "reset search parameters", command = resetSearch)
reset_button.place(relx = 0.5, rely = 0.8, anchor = "center")

#############################################################END LOOP##############################################################################
root.mainloop()


import json
import logging
import os
import zipfile
from multiprocessing import Process

from ..common import *

logging.basicConfig(format="%(levelname)s:\t  %(message)s", level=logging.DEBUG)
_logger = logging.getLogger()

class VM:
    def __init__(
            self,
            new: bool = False,  # False to open an exist VM, True to create a new VM
            name: str = "",  # If you want to create a new vm, this is used to specify the name of the vm
            cpu: int = 2,  # When creating a new vm, you should set the cpu core with this pragma
            ram: float = 4,  # Used to set the ram size when creating a new vm, make sure that the unit is Gigabyte
            template: str = "",  # Initialize the new vm with the template you choose here
            directory: str = ""  # If you are opening an existed vm, use this to specify the directory of it
    ):

        self.name: str = ""
        self.vm_process: Process = Process()
        self.config: dict = {}

        if new:
            if not template:
                _logger.error("No given template of a new VM")
                raise ValueError("No given template of a new VM")

            self.new(name, cpu, ram, template)
        else:
            if not directory:
                _logger.error("No given directory of any VM")
                raise ValueError("No given directory of any VM")

            self.load(directory)

    def load(self, directory: str):
        self.name = directory.split("/")[len(directory.split("/")) - 1]
        _logger.info(f"Loading vm: {self.name}")

        try:
            with open(f"{directory}/config.json", "rt") as f:
                self.config = json.loads(f.read())
        except FileNotFoundError:
            _logger.error("Broken VM")
            raise FileNotFoundError("Broken VM")

    def new(self, name: str, cpu: int, ram: float, template: str):
        _logger.info(f"Creating a new vm with template: {template}")

        template_file = zipfile.ZipFile(template)
        template_file.extractall(f"vms/{name}")

        _logger.info(f"Initializing vm: {name}")
        self.load(f"vms/{name}")
        self.reconfig(cpu=cpu, ram=ram)

    def run(self) -> int:
        _logger.info(f"Starting up vm: {self.name}")

        port = -1

        for i in range(8090, 65535):
            if not check_port(i):
                port = i
                break

        if port == -1:
            _logger.error(f"No empty port exist, cannot start vm: {name}")
            raise IOError(f"No empty port exist, cannot start vm: {name}")

        self.vm_process = Process(target=lambda: os.popen(self.config["run_command"].format(cpu, ram, port)))
        self.vm_process.start()

        return port

    def kill(self):
        _logger.warning(f"Killing vm: {name} by sending a SIGKILL signal")
        self.vm_process.kill()

    def reconfig(self, cpu: int = 0, ram: float = 0):
        if cpu:
            self.config["cpu"] = cpu
        if ram:
            self.config["ram"] = ram

        _logger.info(f"Configuring vm: {self.name}")
        with open(f"vms/{self.name}/config.json", "wt") as f:
            f.write(json.dumps(self.config))

    def is_living(self):
        _logger.info(f"Getting vm's stature, current stature is: {self.vm_process.is_alive()}")
        return self.vm_process.is_alive()

    def get_name(self):
        return self.name


"""
segments a TIFF image stack
"""
import argparse
import os
import torch
import torchreg
import torchreg.transforms.functional as f
import pytorch_lightning as pl
from .registration_model import RegistrationModel
from tqdm import tqdm
import torchreg.viz as viz
import numpy as np

def main(hparams):
    device = "cuda" if torch.cuda.is_available() else "cpu"

    # load model
    model = RegistrationModel.load_from_checkpoint(checkpoint_path=hparams.weights)
    model.eval()
    model = model.to(device)

    transformer = torchreg.nn.SpatialTransformer()

    print(
        f"Evaluating model for dataset {model.hparams.dataset}, loss {model.hparams.loss}, lambda {model.hparams.lam}"
    )

    # init trainer
    trainer = pl.Trainer(gpus=1 if torch.cuda.is_available() else 0)

    # test (pass in the model)
    trainer.test(model)

    # create grid animation
    test_set = model.test_dataloader().dataset
    for i in tqdm(range(1), desc="registering.."):
        (I_0, S_0), (I_1, S_1) = test_set[i]
        (I_0, S_0), (I_1, S_1) = (
            (I_0.unsqueeze(0).to(device), S_0.unsqueeze(0).to(device)),
            (I_1.unsqueeze(0).to(device), S_1.unsqueeze(0).to(device)),
        )
        flow = model.forward(I_0, I_1)
        I_m = transformer(I_0, flow)
        S_m = transformer(S_0.float(), flow, mode="nearest").round().long()

        print(f"accuracy of {i}: ", torch.mean((S_m == S_1).float()))

        os.makedirs(os.path.dirname(hparams.out), exist_ok=True)
        affine = np.array(
            [
                [-1.0, 0.0, 0.0, 80.0],
                [0.0, 0.0, 1.0, -112.0],
                [0.0, -1.0, 0.0, 96.0],
                [0.0, 0.0, 0.0, 1.0],
            ]
        )
        f.save_tensor_as_nii(
            os.path.join(hparams.out, f"{i}i_0.nii.gz"),
            I_0[0],
            affine=affine,
            dtype=np.float32,
        )
        f.save_tensor_as_nii(
            os.path.join(hparams.out, f"{i}s_0.nii.gz"),
            S_0[0],
            affine=affine,
            dtype=np.uint8,
        )
        f.save_tensor_as_nii(
            os.path.join(hparams.out, f"{i}i_1.nii.gz"),
            I_1[0],
            affine=affine,
            dtype=np.float32,
        )
        f.save_tensor_as_nii(
            os.path.join(hparams.out, f"{i}s_1.nii.gz"),
            S_1[0],
            affine=affine,
            dtype=np.uint8,
        )
        f.save_tensor_as_nii(
            os.path.join(hparams.out, f"{i}i_m.nii.gz"),
            I_m[0],
            affine=affine,
            dtype=np.float32,
        )
        f.save_tensor_as_nii(
            os.path.join(hparams.out, f"{i}s_m.nii.gz"),
            S_m[0],
            affine=affine,
            dtype=np.uint8,
        )
        f.save_tensor_as_np(os.path.join(hparams.out, f"{i}flow.np"), flow[0])

if __name__ == "__main__":
    # commandline parser
    parser = argparse.ArgumentParser()
    # add PROGRAM level args
    parser.add_argument(
        "--weights",
        type=str,
        default="./weights/brain-mri/registration/ncc+supervised/weights.ckpt",
        help="model checkpoint to initialize with",
    )
    parser.add_argument(
        "--out",
        type=str,
        default="./out/brain-mri/registration/ncc+supervised/",
        help="path to save the result in",
    )

    hparams = parser.parse_args()
    main(hparams)


def imprime(texto = 'String vazia!'):
    print(texto)

def saudacao(msg = 'Olá', nome = 'Usuário'):
    # Quantos comandos quiser
    nome = nome.replace('e', '3')
    msg = msg.replace('e', '3')
    print(msg, nome)

imprime('Oie')
imprime()

saudacao('Olá', 'Filipe')
saudacao(nome = 'Leopoldo')
saudacao(nome = 'Mark', msg = 'Oh hi')
saudacao()


# -*- coding: utf-8 -*-
# ------------------------------------------------------------------------------
# Created by Mingfei Chen (chenmingfei.lasia@bytedance.com)
# Created On: 2020-1-8
# ------------------------------------------------------------------------------
import torch
import torch.nn as nn
import torch.nn.functional as F

from units.units import Scale, fcosfpn_conv2d

class FPN(nn.Module):

    def __init__(self,
                 in_channels,
                 out_channels,
                 num_outs,
                 start_level=0,
                 end_level=-1,
                 add_extra_convs=True,
                 extra_convs_on_inputs=False,
                 relu_before_extra_convs=True,
                 ):
        """ FCOSFPN. FPN based on lateral backbone, top-down paramid with extra pyramid layers.

        Args:
            in_channels (list): Channel num of input from the backbone
            out_channels (int): FPN output channel num
            num_outs (int): FPN output layer num
            end_level (int, -1): FPN layer num based on in_channel list length
            add_extra_convs (bool): Extra FPN layers
            extra_convs_on_inputs (bool): Extra FPN layers based on backbone input or FPN P5 layer
            relu_before_extra_convs (bool): ReLU activation before each extra layer

        Forward:
            fpn_in(list): len FPN_In_Layer_Num(4)  `([BS X In_Channel_1 X W_1 X H_1, ...])`

        Return:
            fpn_out(list): len FPN_Out_Layer_Num(5)  `([BS X Out_Channel X W_1 X H_1, ...])`

        """
        super(FPN, self).__init__()
        assert isinstance(in_channels, list)
        self.in_channels = in_channels
        self.out_channels = out_channels
        self.num_ins = len(in_channels)
        self.num_outs = num_outs
        self.relu_before_extra_convs = relu_before_extra_convs

        if end_level == -1:
            self.backbone_end_level = self.num_ins
            assert num_outs >= self.num_ins - start_level
        else:
            # if end_level < inputs, no extra level is allowed
            self.backbone_end_level = end_level
            assert end_level <= len(in_channels)
            assert num_outs == end_level - start_level
        self.start_level = start_level
        self.end_level = end_level
        self.add_extra_convs = add_extra_convs
        self.extra_convs_on_inputs = extra_convs_on_inputs
        self.lateral_convs = nn.ModuleList()
        self.fpn_convs = nn.ModuleList()

        for i in range(self.start_level, self.backbone_end_level):
            # backbone for top features
            l_conv = fcosfpn_conv2d(
                in_channels[i],
                out_channels,
                1)
            # fpn backbone for bottom features
            fpn_conv = fcosfpn_conv2d(
                out_channels,
                out_channels,
                3,
                padding=1)
            self.lateral_convs.append(l_conv)
            self.fpn_convs.append(fpn_conv)
        # add extra conv layers (e.g., RetinaNet)
        extra_levels = num_outs - self.backbone_end_level + self.start_level
        if add_extra_convs and extra_levels >= 1:
            for i in range(extra_levels):
                if i == 0 and self.extra_convs_on_inputs:
                    in_channels = self.in_channels[self.backbone_end_level - 1]
                else:
                    in_channels = out_channels
                extra_fpn_conv = fcosfpn_conv2d(
                    in_channels,
                    out_channels,
                    3,
                    stride=2,
                    padding=1)
                self.fpn_convs.append(extra_fpn_conv)
        self.init_weights()

    def init_weights(self):
        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):
                nn.init.constant_(m.weight, 1)
                nn.init.constant_(m.bias, 0)

    def forward(self, inputs):
        assert len(inputs) == len(self.in_channels)
        # build laterals C3 C4 C5
        laterals = [
            lateral_conv(inputs[i + self.start_level])
            for i, lateral_conv in enumerate(self.lateral_convs)
        ]
        # build top-down path
        used_backbone_levels = len(laterals)
        for i in range(used_backbone_levels - 1, 0, -1):
            # upsample P4 P5
            laterals[i - 1] += F.interpolate(
                laterals[i], scale_factor=2, mode='nearest')
        # build outputs
        # append the bottom two outputs
        outs = [
            self.fpn_convs[i](laterals[i]) for i in range(used_backbone_levels)
        ]
        # append P6 P7
        if self.num_outs > len(outs):
            # when no extra layers, use max pool to get more levels on top of outputs
            if not self.add_extra_convs:
                for i in range(self.num_outs - used_backbone_levels):
                    outs.append(F.max_pool2d(outs[-1], 1, stride=2))
            # add conv layers on top of original feature maps (RetinaNet)
            else:
                # append the mid layer
                if self.extra_convs_on_inputs:
                    # apply the top input layer as base layer for extra convs
                    orig = inputs[self.backbone_end_level - 1]
                    outs.append(self.fpn_convs[used_backbone_levels](orig))
                else:
                    # appy P5 as base layer for extra convs
                    outs.append(self.fpn_convs[used_backbone_levels](outs[-1]))
                # append two top layers
                for i in range(used_backbone_levels + 1, self.num_outs):
                    # apply ReLU before extra convs
                    if self.relu_before_extra_convs:
                        outs.append(self.fpn_convs[i](F.relu(outs[-1])))
                    else:
                        outs.append(self.fpn_convs[i](outs[-1]))
        return tuple(outs)

if __name__ == "__main__":
    from torch.autograd import Variable
    neck = FPN(in_channels=[256, 512, 1024, 2048],
               out_channels=256,
               start_level=1,
               add_extra_convs=True,
               extra_convs_on_inputs=False,
               num_outs=5,
               relu_before_extra_convs=True)
    x = []
    in_dim = 256
    w = 200
    h = 256
    for i in range (4):
        x.append(Variable(torch.randn(2, in_dim, w, h)))
        in_dim = in_dim * 2
        w = int(w / 2)
        h = int(h / 2)
    outs = neck(x)
    for i in range (len(outs)):
        print(outs[i].shape)


from . import AbstractExperiment
import experimentalCollections as tc
import networkx as nx
import matplotlib.pyplot as plt
import numpy as np
from matplotlib import cm
import json
import pandas as pd
import convLib
from convLib import Annotator
import utils
import time
from retrieval.Analyzers import StandardAnalyzer

from convLib import contexts, Annotator

import experimentalCollections as EC

from sklearn.preprocessing import normalize
from sklearn.metrics import r2_score

from scipy.stats import gaussian_kde, pearsonr
from scipy.spatial.distance import jensenshannon
from collections import Counter

import seaborn as sns
import random

from retrieval.CAsTutils import loadQueries
#'first_query', r"FU",
systems_order = ['context_query', 'linear_prev',  'RM3_prev', 'RM3_seq', 'allennlp_simple']
systems_labels = [ r"CU", r"LP", r"RM3p", r"RM3s", r"anCB"]

class Intratopic_analysis(AbstractExperiment):

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    def run_experiment(self):
        sns.set_theme(style="whitegrid")
        #sns.set_context("paper",
        #                rc={"font.size": 24, "axes.titlesize": 24, "xtick.labelsize": 24, "ytick.labelsize": 24,
        #                     "legend.fontsize":24, "axes.labelsize": 24})

        measures = pd.read_csv(f"../../data/measures/full_valid_100.csv")

        avgs_all = measures\
                        [['topic', 'perm', 'qtype', 'name', 'nDCG@3']]\
                        .groupby(['topic', 'qtype', 'name', 'perm'])\
                        .aggregate(['mean'])\
                        .droplevel(1, axis=1)\
                        .reset_index() \

        avgs_all = avgs_all.groupby(['name', 'qtype', 'perm']).aggregate(['mean']).droplevel(1, axis=1).reset_index()

        avgs = measures[measures['name'].isin(systems_order)]\
                        [['topic', 'perm', 'qtype', 'name', 'nDCG@3']]\
                        .groupby(['topic', 'qtype', 'name', 'perm'])\
                        .aggregate(['mean'])\
                        .droplevel(1, axis=1)\
                        .reset_index()

        '''
        corrs = []
        for qtype in ['original', 'resolved']:
            avgs_t = avgs[avgs['qtype']==qtype][['name', 'topic', 'perm', 'nDCG@3']].copy()
            corr = avgs_t.groupby(['topic']).apply(rankingSimilarity)
            corr = pd.DataFrame([[qtype] + l for l in list(corr)], columns = ['qtype', 'topic', 'correlation', 'pvalue'])
            corrs.append(corr)
        corrs = pd.concat(corrs)
        fname = "../../data/correlation_valid_permutations.pdf"

        fig, ax = plt.subplots(figsize=(18, 12))
        h = sns.barplot(data=corrs, x='topic', y='correlation', hue='qtype', ax=ax)
        handles, labels = ax.get_legend_handles_labels()
        ax.legend(handles=handles[0:], labels=labels[0:])
        ax.set(xlabel= "Conversation id", ylabel=r"Correlation (Spearman's $\rho$)")
        plt.savefig(fname)
        '''
        #for qtype in ['original', 'resolved']:
        for qtype in ['original']:
            avgs_t = avgs[(avgs['qtype']==qtype) & (avgs['topic'].isin(["32", "33", "37", "56", "58", "59", "61", "67", "69", "77", "78", "79"]))][['name', 'topic', 'perm', 'nDCG@3']].copy()
            Z = avgs_t.groupby(["topic"]).apply(pairwise_best).reset_index()
            print(Z)
            Z = Z.drop(columns=["level_1"])
            Z = Z.rename({"topic":"Conv. id"}, axis='columns')
            print(Z)
            grid = sns.FacetGrid(Z, col="Conv. id", col_wrap=4)
            grid.map(draw_heatmap, 's1', 's2', 'n_best', cbar=False, annot=True,  fmt='.3g')
            grid.set_titles(size=18)
            fname = f"../../data/figures/best_between_two_{qtype}.pdf"
            plt.savefig(fname)

def draw_heatmap(*args, **kwargs):
    data = pd.concat(args, axis=1)
    d = data.pivot(index=args[0].name, columns=args[1].name, values=args[2].name)
    d = d.sort_values('s1', key=make_sorter(systems_order))[systems_order]
    g = sns.heatmap(d, **kwargs)
    g.set_xticklabels(systems_labels, fontsize=18)
    g.set_yticklabels(systems_labels, fontsize=18)
    g.set_ylabel("")
    g.set_xlabel("")

def rankingSimilarity(scores, sim='pearson', index='name'):
    old_scores = scores[scores['perm']==0][['name', 'nDCG@3']]
    new_scores = scores[scores['perm']!=0][['name', 'nDCG@3']]

    temp_scores = pd.merge(old_scores, new_scores, how="inner", on=["name"])

    return [scores['topic'].unique()[0]] + list(pearsonr(temp_scores['nDCG@3_x'], temp_scores['nDCG@3_y']))

def pairwise_best(avgs_t):
    models = avgs_t['name'].unique()
    tmp = avgs_t.pivot_table(index='perm', columns='name', values='nDCG@3').reset_index()

    output=[[i, j, (np.sum(tmp[i]>tmp[j]))] for i in models for j in models if i!=j]
    output = pd.DataFrame(output, columns=["s1", "s2", "n_best"])

    return output

def make_sorter(l):
    """
    Create a dict from the list to map to 0..len(l)
    Returns a mapper to map a series to this custom sort order
    """
    sort_order = {k:v for k,v in zip(l, range(len(l)))}
    return lambda s: s.map(lambda x: sort_order[x])


# coding=utf-8

import requests
from bs4 import BeautifulSoup
import re
import csv
import time
import datetime
import os
import pymongo

re_r = '1[3|4|5|7|8]\d{9}'  # 1代表第一个号码为1;;;[3|4|5|7|8]第二位指定为这几个数；；；\d代表0~9，{9}循环9次

one_url = ['https://www.atobo.com.cn/Companys/s-p26-y{}/'.format(str(i)) for i in range(10,50)]

one_title_url = '#setcolor_area > div.product_contextlist.bplist > ul > li'
one_href_url = ' div > ul > li.p_name > div > ul > li.pp_name > a.CompanyName'

name = 'body > div.contextf > div > div > div.cont-r > div:nth-of-type(1) > div > div > ul:nth-of-type(3) > li.card-right'
telephone = 'body > div.contextf > div > div > div.cont-r > div:nth-of-type(1) > div > div > ul:nth-of-type(4) > li.card-right'
phone = 'body > div.contextf > div > div > div.cont-r > div:nth-of-type(1) > div > div > ul:nth-of-type(5) > li.card-right'

def get_html_url():
    print()
    for var_url in one_url:
        time.sleep(100)
        url_html = requests.get(var_url)
        soup = BeautifulSoup(url_html.text, 'html.parser')
        for var_one in soup.select(one_title_url):
            one_href = var_one.select(one_href_url)
            for two_url in one_href:
                url_url = two_url.get('href')
                home_url = 'https:' + url_url
                print(home_url)
                time.sleep(5)
                # # 调用只获得手机号码的方法
                # get_phone(home_url)
                # 调用获得手机号和姓名
                get_name_phone(home_url)

# 获得手机号和姓名
def get_name_phone(url):
    arr_two = []
    two_url = requests.get(url)
    soup = BeautifulSoup(two_url.text, 'html.parser')
    var_name1 = soup.select(name)
    var_telephone1 = soup.select(telephone)
    var_phone1 = soup.select(phone)
    for var_name, var_telephone, var_phone in zip(var_name1,var_telephone1,var_phone1):
        data_two = (
            var_name.text,
            var_telephone.text,
            var_phone.text
        )
        arr_two.append(data_two)
        print(arr_two)
        set_name_phone(set_data_time('name'), arr_two)

# 存储姓名，手机号，座机号
def set_name_phone(folder, _arr1):
    if os.access(folder, os.F_OK):
        with open(folder, 'a+', newline='')as folders:
            writers = csv.writer(folders)
            writers.writerows(_arr1)
            folders.close()
    else:
        with open(folder, 'a+', newline='')as folders:
            writers = csv.writer(folders)
            writers.writerow(['name', 'telephone', 'phone'])
            writers.writerows(_arr1)
            folders.close()

# 获得手机号码
def get_phone(url):
    arr_data = []
    url_html = requests.get(url).text
    re_phone = re.findall(re_r, url_html, re.M | re.S | re.I)
    re_phone = set(re_phone)
    for car_phone in re_phone:
        data_a = (
            car_phone
        )
        arr_data.append(data_a)
    set_csv(set_data_time('phone'), arr_data)

# 设置文件生成时间
def set_data_time(st):
    time_ = time.localtime(time.time())
    data_day = datetime.date.today()
    folder = 'E:\sc\四川%s-%s点%s.csv' % (data_day, time_.tm_hour, st)
    return folder

# 文件存储
def set_csv(folder, _arr1):
    if os.access(folder, os.F_OK):
        with open(folder, 'a+', newline='')as folders:
            writers = csv.writer(folders)
            writers.writerows(_arr1)
            folders.close()
    else:
        with open(folder, 'a+', newline='')as folders:
            writers = csv.writer(folders)
            writers.writerow(['phone'])
            writers.writerows(_arr1)
            folders.close()

if __name__ == '__main__':
    print('开始运行！！！')
    # get_phone('https://hyb8816.atobo.com.cn/')
    get_html_url()


import os
import subprocess
import numpy as np
import matplotlib.pyplot as plt
import mplbm_utils as mplbm

def download_geometry(filename, url):

    download_command = f'wget {url} -O {filename}'
    try:
        subprocess.run(download_command.split(' '))
    except FileNotFoundError:
        raise InterruptedError(f'wget was not found. Please make sure it is installed on your system.')
    return

def run_2_phase_sim(inputs):
    # Steps
    # 1) create geom for palabos
    # 2) create palabos input file
    # 3) run 2-phase sim

    if inputs['simulation type'] == '1-phase':
        raise KeyError('Simulation type set to 1-phase...please change to 2-phase.')
    sim_directory = inputs['input output']['simulation directory']

    # 1) Create Palabos geometry
    print('Creating efficient geometry for Palabos...')
    mplbm.create_geom_for_palabos(inputs)

    # 2) Create simulation input file
    print('Creating input file...')
    mplbm.create_palabos_input_file(inputs)

    # 3) Run 2-phase simulation
    print('Running 2-phase simulation...')
    num_procs = inputs['simulation']['num procs']
    input_dir = inputs['input output']['input folder']
    simulation_command = f"mpirun -np {num_procs} ../../src/2-phase_LBM/ShanChen {input_dir}2_phase_sim_input.xml"
    file = open(f'{sim_directory}/{input_dir}run_shanchen_sim.sh', 'w')
    file.write(f'{simulation_command}')
    file.close()

    simulation_command_subproc = f'bash {sim_directory}/{input_dir}run_shanchen_sim.sh'
    subprocess.run(simulation_command_subproc.split(' '))

    return

def run_rel_perm_sim(inputs):

    # Rel Perm Steps
    # 1) create 1-phase palabos input file for relperms
    # 2) create geoms for rel perm
    # 3) run 1-phase sim to get rel perms

    if inputs['simulation type'] == '1-phase':
        raise KeyError('Simulation type set to 1-phase...please change to 2-phase.')
    sim_directory = inputs['input output']['simulation directory']

    # 1) Create geoms for rel perm
    print('Creating rel perm geometries...')
    user_geom_name = inputs['domain']['geom name']
    inputs = mplbm.create_geom_for_rel_perm(inputs)

    # 2) Create simulation input file
    print('Creating input file...')
    inputs['simulation type'] = 'rel perm'
    mplbm.create_palabos_input_file(inputs)
    inputs['domain']['geom name'] = user_geom_name

    # 3) Write rel perm bash file
    print('Running rel perm simulation...')
    num_procs = inputs['simulation']['num procs']
    input_dir = inputs['input output']['input folder']
    output_dir = inputs['input output']['output folder']
    simulation_command = f"mpirun -np {num_procs} ../../src/1-phase_LBM/permeability {input_dir}relperm_input.xml"
    file = open(f'{sim_directory}/{input_dir}run_relperm_sim.sh', 'w')
    file.write(f'{simulation_command}')
    file.close()

    simulation_command_subproc = f'bash {sim_directory}/{input_dir}run_relperm_sim.sh'
    make_4relperm_folder = f'mkdir {sim_directory}/{output_dir}/4relperm'
    subprocess.run(make_4relperm_folder.split(' '))
    subprocess.run(simulation_command_subproc.split(' '))

    return

def process_and_plot_results(inputs):

    # Process data
    mplbm.create_pressure_data_file(inputs)
    mplbm.create_relperm_data_file(inputs)

    # Load and plot data
    sim_dir = inputs['input output']['simulation directory'] + '/'
    output_dir = inputs['input output']['output folder']
    Sw = np.loadtxt(f'{sim_dir + output_dir}data_Sw.txt')
    Pc = np.loadtxt(f'{sim_dir + output_dir}data_Pc.txt')
    krw = np.loadtxt(f'{sim_dir + output_dir}data_krw.txt')
    krnw = np.loadtxt(f'{sim_dir + output_dir}data_krnw.txt')

    plt.figure(figsize=[10,8])
    mplbm.plot_capillary_pressure_data(Sw, Pc)
    plt.savefig(sim_dir + 'pc_curve.png', dpi=300)

    plt.figure(figsize=[10,8])
    mplbm.plot_rel_perm_data(Sw, krw, krnw)
    # plt.yscale('log')
    plt.savefig(sim_dir + 'relperm_curve.png', dpi=300)

    plt.figure(figsize=[6, 10])
    mplbm.plot_pc_and_rel_perm(Sw, Pc, krw, krnw)
    plt.savefig(sim_dir + 'pc_and_relperm_curve.png', dpi=300)

    return

drp_url = 'www.digitalrocksportal.org/projects/65/images/71108/download/'
file_name = 'input/rg_theta60_phi10.raw'
download_geometry(file_name, drp_url)

input_file = 'input.yml'
inputs = mplbm.parse_input_file(input_file)  # Parse inputs
inputs['input output']['simulation directory'] = os.getcwd()  # Store current working directory
run_2_phase_sim(inputs)  # Run 2 phase sim
run_rel_perm_sim(inputs)  # Run rel perm
process_and_plot_results(inputs)  # Plot results

# plt.show()


from scrapy.spiders import SitemapSpider

from locations.structured_data_spider import StructuredDataSpider
from locations.user_agents import BROWSER_DEFAULT

class HarveysSpider(SitemapSpider, StructuredDataSpider):
    name = "harveys"
    item_attributes = {"brand": "Harvey's", "brand_wikidata": "Q1466184"}
    allowed_domains = ["harveys.ca"]
    sitemap_urls = ["https://www.harveys.ca/en/locations/sitemap.xml"]
    sitemap_rules = [(r"/locations/[-\w]+/[-\w]+/[-\w]+", "parse_sd")]
    wanted_types = ["Restaurant"]
    user_agent = BROWSER_DEFAULT


from tkinter import *

# https://pypi.org/project/tkmacosx/
# pip3 install tkmacosx
from tkmacosx import *

# Hello Mr Wagner

classes = ['SDS A', 'Fundamentals C', 'CSP D', 'CSP E', 'APCS F']

# ==================================================================================================================================
# ==================================================================================================================================
# ==================================================================================================================================

def DrawText(pic, text):

    global wordsLabel
    global puppyLabel
    puppyLabel.config(image=pic)

    #frame2.delete()
    wordsLabel.config(text=text)

    pass

# ==================================================================================================================================
# ==================================================================================================================================
# ==================================================================================================================================

root = Tk()
root.title("Mr. Wagner Card")
root.configure(background='light pink')

frame1 = Frame(root, bg='sky blue', bd=10)
frame1.pack(side=TOP)

label1 = Label(frame1, text="Mr. Wagner,", fg="black", bg="sky blue", width=25, height=1)
label1.pack(side=TOP)

mainCanvas = Canvas(frame1, width=400, height=400, bg='sky blue')
mainCanvas.pack(side=TOP)

# this adds photo to main canvas
puppy = PhotoImage(file='dapup.png')
puppyPhoto1 = PhotoImage(file='puppy1.png')
puppyPhoto2 = PhotoImage(file='puppy2.png')
puppyPhoto3 = PhotoImage(file='puppy3.png')
puppyPhoto4 = PhotoImage(file='puppy4.png')
puppyPhoto5 = PhotoImage(file='puppy5.png')

puppyLabel = Label(mainCanvas, image=puppy)
puppyLabel.pack()

wordsLabel = Label(mainCanvas, text='Click a class below.', fg="black", bg="light grey")
wordsLabel.pack(side=TOP)

frame2 = Frame(root)
frame2.pack(side=BOTTOM)

c1 = Button(frame2, text=classes[0], borderless=1, bg='white', command=lambda: DrawText(puppyPhoto1, '"We are very sorry for your loss and you are in our thoughts and prayers.\nAs students of yours for two years now, we’ve been able to see your positive attitude and quirky\nhumor firsthand and can’t imagine what you’re going through right now. We can’t wait to see you and hear some more dad jokes!" -' + classes[0]))
c1.pack(side=LEFT)

c2 = Button(frame2, text=classes[1], borderless=1, bg='white', command=lambda: DrawText(puppyPhoto2, '"On behalf of our programming class, we want to tell you that we love you and support you." -' + classes[1]))
c2.pack(side=LEFT)

c3 = Button(frame2, text=classes[2], borderless=1, bg='white', command=lambda: DrawText(puppyPhoto3, '"We think about you every day, and we are deeply sorry for your loss." -' + classes[2]))
c3.pack(side=LEFT)

c4 = Button(frame2, text=classes[3], borderless=1, bg='white', command=lambda: DrawText(puppyPhoto4, '"We have all been thinking of you during this time, we missed you and we can\'t wait to see you in class." -' + classes[3]))
c4.pack(side=LEFT)

c5 = Button(frame2, text=classes[4], borderless=1, bg='white', command=lambda: DrawText(puppyPhoto5, '"We missed you, and we hope you know you\'re surrounded by people who love you." -' + classes[4]))
c5.pack(side=LEFT)

#c6 = Button(frame2, text="Home Screen", borderless=1, bg='white', command=lambda: HomeScreen(''))
#c6.pack(side=RIGHT)

# ==================================================================================================================================

mainloop()