import os
os.environ['OPENBLAS_NUM_THREADS'] = '1' # This avoids crashes on the math cluster from using to many resources
import pandas as pd
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
from bs4 import BeautifulSoup
from matplotlib import pyplot as plt
from matplotlib.animation import FuncAnimation
from matplotlib import animation
import time
import csv
from celluloid import Camera
from matplotlib.collections import PatchCollection
from scipy import spatial, ndimage
import copy
import networkx as nx
matplotlib.use("TkAgg")

class experiment:
    """Holds tracks of the experiment, as well as any calculated observables.
    :name Name of the experiment
    :tracks Array of numpy arrays containing the times and coordinates of each particle
    :tmax number of timesteps
    :x_max Biggest x coordinate found in a track
    :y_max Biggest y coordinate found in a track
    :distance_travlled Array with one entry for every track, contains the total distance a
        particle has travelled on the track
    :tortuosity Array with one entry for every track, contains the total normalised
        difference between distance_travelled and the total displacement
    :rng Random number generator used to create histogram normalisaiton by generating particles with
        uniform density
    :uniform_histogram Stores histogram of radial density of uniform particles. Used
        for normalisation in plot_radial_density
    """
    def __init__(self,name,seed=None):
        self.periodic = False #The simulation as periodic, the experimental data is not
        self.name = name
        self.timeStep = 1 #time passed between each step of the track
        self.tracks = [] #tracks[particle][step][t,x,y]
        self.x_max = 0
        self.y_max = 0
        self.max_distance = 0 #Maximum distance between particles based on x_max, y_max
        self.reference_histogram = np.array([None])
        self.color = [] #color[particle][step]['color']
        self.seed=seed
        self._measurementTimes=None
        if seed==None:
            self.rng = np.random.default_rng()
        else:
            self.rng = np.random.default_rng(self.seed)

    def read_xml(self,fileName, min_length):
        """Reads the xml file in fileName and saves the tracks in it into fileName
        :fileName Path to the file
        :min_length All tracks shorter than that are excluded
        """
        tic = time.perf_counter()

        #Read xml file into beautiful soup:
        with open(fileName, 'r') as f:
            data = f.read()
        Bs_data = BeautifulSoup(data, "xml")

        self.periodic = False #This .xml file only exists for experiments, which are not periodic

        #keep track of the biggest x and y values we find

        #Iterate through all tracks in the file
        for track in Bs_data.find_all('particle'):
            self.tracks.append([])
            #create numpy arrays to contain the track
            # nSpots = int(track['nSpots'])
            # if nSpots >=min_length:# filter short tracks
            #     self.tracks.append(np.zeros(shape=(nSpots,3)))
            #     n=0
                #Iterate through each spot detected for the particle
            nSpots = 0
            for spot in track.find_all('detection'):
                nSpots +=1
                self.tracks[-1].append([int(spot['t']),float(spot['x']),float(spot['y'])])
                # self.tracks[-1][n][0]=int(spot['t'])
                # self.tracks[-1][n][1]=float(spot['x'])
                # self.tracks[-1][n][2]=float(spot['y'])
                self.x_max = max(self.x_max,float(spot['x']))
                self.y_max = max(self.y_max,float(spot['y']))

        self.set_max_distance()
        toc = time.perf_counter()

        print(f"Read xml file in {toc - tic:f} seconds")

    def read_csv(self, fileName):
        """Reads the simulation data from a .csv file named fileName and saves the data in experiment.
        """
        tic = time.perf_counter()
        with open(fileName, mode="r") as file:
            csvFile = csv.reader(file)
            lineCount = 0
            for lines in csvFile: #Go through every timestep
                nParticles = int((len(lines)-1)/3)
                if (lineCount==0): #Initialise the tracks
                    self.tracks = [[] for i in range(nParticles)]
                    self.color =  [[] for i in range(nParticles)]
                t = float(lines[0])
                for particleIdx in range(nParticles):
                    self.color[particleIdx].append(lines[1+3*particleIdx])
                    self.tracks[particleIdx].append([t,
                                                    float(lines[1+3*particleIdx+1]), #x value
                                                    float(lines[1+3*particleIdx+2])])#y value

                lineCount +=1
        toc = time.perf_counter()
        self.periodic = True #This .csv file only exists for simulations, which are always periodic
        print(f"Read .csv track file in {toc - tic:f} seconds")

    def read_parameter_file(self,file):
        params = {}
        """Reads the parameter .txt files used to initialise the C simulation
        """
        with open(file, 'r') as reader:
            for line in reader.readlines():
                #if line=="\n": break  #Stop when encountering an empty line
                idx1 = line.find(" ")
                idx2 = line.find(": ")
                if(idx2==-1): break #we reached the end of the parameter list
                name = line[:idx1]
                value = line[idx2+2:]
                params[name] = float(value)
                #print(name+" = %f"%params[name])
        self.timeStep = float(params["measurementInterval"])
        self.x_max = float(params["Length"])
        self.y_max = float(params["Length"])
        self.set_max_distance()
        return params

    def set_max_distance(self):
        if(self.periodic==False):
            self.max_distance = np.sqrt(self.x_max**2+self.y_max**2)*1.01 #Some wiggle room in case a point is exactly at max distane
        elif(self.periodic==True):
            self.max_distance = np.sqrt((self.x_max**2+self.y_max**2)/4)*1.01

    def find_t_max(self):
        tMax = 0
        for track in self.tracks:
            tMax = max(tMax, track[-1][0])
        return tMax

    def distance(self, r):
        """Calculates the length of the tangent vector r taking periodicity into account
        """
        if self.periodic == False:
            return np.linalg.norm(np.array(r))
        elif self.periodic == True:
            r[0] = min(abs(r[0]),abs(abs(r[0])-self.x_max)) #abs(r[0]) % self.x_max
            r[1] = min(abs(r[1]),abs(abs(r[1])-self.y_max)) #abs(r[1]) % self.y_max
            return np.linalg.norm(np.array(r))
        else:
            assert(False), "Invalid value for periodicity"

    def MSD(self):
        """Calculates the mean squared displacement. Accounts for periodic boundary conditions in the simulation.
        Assumes displacement between steps doesn't exceed half the box.
        """
        MSD = np.zeros(len(self.tracks[0]))
        time = np.zeros(len(self.tracks[0]))
        for track in self.tracks:
            origin = np.array(track[0][1:]) #keeps track of where particle started
            offset = np.zeros(2) #keeps track of how many times the periodic boundary was crossed
            oldPosition = origin
            for tIdx in range(1,len(track)):
                newPosition = np.array(track[tIdx][1:])
                displacement = newPosition-oldPosition
                #check if particle passed boundary
                if(displacement[0]>self.x_max/2): #crossed lower x boundary
                    offset[0]-= self.x_max
                elif(displacement[0] < -self.x_max/2): #crossed upper x boundary
                    offset[0] += self.x_max
                if(displacement[1]>self.y_max/2): #crossed lower y boundary
                    offset[1]-= self.y_max
                elif(displacement[1] < -self.y_max/2): #crossed upper y boundary
                    offset[1] += self.y_max

                #Update MSD
                MSD[tIdx] += np.linalg.norm(newPosition+offset-origin)**2
                time[tIdx] = track[tIdx][0]

                oldPosition = newPosition

        return MSD/len(self.tracks), time

    def TAMSD(self,delta_t,min_length):
        """Returns the average TAMSD as well as all individual TAMSDs for all tracks in the experiment
         whose length exceed min_length. WARNING: Not suited for periodic boundary conditions (doesn't undo wraparound)
        :delta_t time interval (in timesteps, not minutes) for which the TAMASD will be computed
        :min_length minimum number of timesteps a track must have to be used for the average
        """
        assert delta_t < min_length, "The minimum track length must exceed the time average interval"
        nTracks = 0 # counts how many tracks are included/ have the minimum length
        average = 0
        individual_TAMSD = []
        for track in self.tracks:
            track_duration = len(track)
            total = 0
            if track_duration >= min_length:
                nTracks += 1
                #Calculate TAMSD for the track
                for idx in range(track_duration-delta_t):
                    distance = track[idx][1:]-track[idx+delta_t][1:]
                    total += np.dot(distance,distance)
                TAMSD= total/(track_duration-delta_t)
                individual_TAMSD.append(TAMSD)
                average += TAMSD
        assert nTracks > 0, 'None of the tracks have the minimum length'
        average = average/nTracks

        return average, individual_TAMSD

    def plot_TAMSD(self, tmax, min_length,axes):
        """Calculates the TAMSD as a function of the time intervall delta_t for all tracks
        in the experiment that are longer than min_length.
        :tmax sets the biggest delta_t that is considered (in timesteps, not minutes). Has to
            be an integer
        :min_length is used to filter all tracks that are shorter
        :axes is the pyplot object into which the graph is drawn
        """
        delta_t_vec = np.linspace(1, tmax, tmax, dtype = int)
        avr_TAMSD = np.zeros(tmax)
        individual_TAMSD = []
        for delta_t in delta_t_vec:
            avr_TAMSD[delta_t-1], TAMSDs = self.TAMSD(delta_t,min_length)
            individual_TAMSD.append(TAMSDs)

        #Plot individual tracks
        individual_TAMSD = np.transpose(np.array(individual_TAMSD))
        for vec in individual_TAMSD:
            axes.loglog(delta_t_vec,vec,linewidth=0.1)
        axes.set_xlabel(r"$\Delta$ t")
        axes.set_ylabel("TAMSD")
        axes.loglog(delta_t_vec,avr_TAMSD,label='Average', linewidth=2)

        #Plot dashed lines for comparison

        #Ballistic motion, starting on the left
        x0 = avr_TAMSD[0]
        x1 = avr_TAMSD[0]*(tmax)**2
        axes.loglog([delta_t_vec[0],delta_t_vec[-1]],[x0,x1],color='black',
                    linestyle='dashed', label='Ballistic motion')

        #Normal diffusive motion,
        y0 = avr_TAMSD[0]
        y1 = avr_TAMSD[0]*(tmax)
        axes.loglog([delta_t_vec[0],delta_t_vec[-1]],[y0,y1],color='blue',
                    linestyle='dashed', label='Diffusive motion')

    def calculate_tortuosity_dun_method(self, delta_t, min_length):
        """calculates distance travelled and
        tortuosity = (distance_travelled-displacement)/distance_travlled for every track and
        saves the values in two arrays, whose indices correspond to those of the tracks
        :delta_t Size of time interval for averaging (Dun method)
        :min_length Minimum size the track must have
        """
        assert delta_t < min_length, "Can't average unless delta_t<min_length"
        assert (delta_t/self.timeStep==int(delta_t/self.timeStep)), "delta_a has to be a mutiple of the time steps"
        nTracks = len(self.tracks)
        self.distance_travelled = np.zeros(nTracks)
        self.tortuosity = np.zeros(nTracks)
        n=0 #track index
        for track in self.tracks:
            track_duration = int(len(track)*self.timeStep) #almost correct, some of the measured tracks have gaps in them
            if track_duration >= min_length:
                tortuosity = 0
                n_intervals = int(track_duration//delta_t)
                delta_idx = int(delta_t/self.timeStep)
                for intervalIdx in range(n_intervals):
                    #Calculate Tortuosity in every interval
                    straight_line_distance = self.distance(np.array(track[intervalIdx*delta_idx][1:])-np.array(track[(intervalIdx+1)*delta_idx-1][1:]))
                    displacement = 0
                    for spot_idx in range(intervalIdx*delta_idx,(intervalIdx+1)*delta_idx-1):
                        displacement += self.distance(np.array(track[spot_idx+1][1:])-np.array(track[spot_idx][1:]))
                    tortuosity += (displacement-straight_line_distance)/displacement
                tortuosity = tortuosity/n_intervals

                self.distance_travelled[n]=displacement
                self.tortuosity[n] = tortuosity
                n+=1
        assert(n>0),"No tracks fulfill the requirements"
        #Extract non-zero entries
        self.distance_travelled = self.distance_travelled[self.tortuosity!=0]
        self.tortuosity = self.tortuosity[self.tortuosity!=0]

    def calculate_tortuosity(self, min_length): #Old version without dun method
        """calculates distance travelled and
        tortuosity = (distance_travelled-displacement)/distance_travlled for every track and
        saves the values in two arrays, whose indices correspond to those of the tracks
        """
        nTracks = len(self.tracks)
        self.distance_travelled = np.zeros(nTracks)
        self.tortuosity = np.zeros(nTracks)
        n=0 #track index
        for track in self.tracks:
            if len(track)>= min_length:
                displacment = self.distance(np.array(track[0][1:])-np.array(track[-1][1:]))
                distance_travelled = 0
                for idx in range(len(track)-1):
                    distance_travelled += self.distance(np.array(track[idx+1][1:])-np.array(track[idx][1:]))
                tortuosity = (distance_travelled-displacment)/distance_travelled

                self.distance_travelled[n]=distance_travelled
                self.tortuosity[n] = tortuosity
                n+=1
        assert(n>0),"No tracks fulfill the requirements"
        #Extract non-zero entries
        self.distance_travelled = self.distance_travelled[self.tortuosity!=0]
        self.tortuosity = self.tortuosity[self.tortuosity!=0]

    def plot_tortuosity(self,axes, min_length, label, color, n_bins=None, method='Normal', delta_t=None):
        """Plots tortuosity either by considering the entire track at once, or by splitting
        the track into intervals of length delta_t and averaging over those (Dun method).
        """
        if method=='Normal':
            self.calculate_tortuosity(min_length)
        elif method == 'Dun':
            self.calculate_tortuosity_dun_method(delta_t, min_length)
        else:
            raise Exception("Unknown method for computing tortuosity")
        #axes.scatter(self.distance_travelled,self.tortuosity, s=0.4)
        if n_bins == None:
            axes.hist(self.tortuosity, density=True, alpha=0.5, label=label, color=color)
        else:
            axes.hist(self.tortuosity, density=True, alpha=0.5, label=label, color=color, bins=n_bins)
        axes.set_xlabel(r'Tortuosity = (L-I)/L')
        axes.set_ylabel(r'Probability Density')
        axes.legend()

    def generate_reference_histogram(self, n_particles, n_bins):
        '''Generates a radial density histogram for uniform distributed particles in a box of
        size self.x_max self.y_max. This serves as a normalisation for the measured density.
        :n_particles Number of particles that are generated
        :n_bins Number of bins to be used. The bins are evenly distributed between 0 and max_distance
        '''
        #Check if we already created the distogram with this number of particles
        if (all(self.reference_histogram==None ) or len(self.reference_histogram)!= n_bins):
            #Generate points
            x_values = self.rng.random((n_particles))*self.x_max
            y_values = self.rng.random((n_particles))*self.y_max
            points = np.transpose(np.array([x_values,y_values]))

            if(self.periodic==False): #Faster Computation for non-periodic data
                distances = spatial.distance.pdist(points)
            elif(self.periodic==True): #Manual compuatation for periodic data
                distances = np.zeros(int(n_particles*(n_particles-1)/2))
                cnt = 0
                for i in range(n_particles):
                    for j in range(i+1,n_particles):
                        distances[cnt]=self.distance(points[i]-points[j])
                        cnt+=1

            self.reference_histogram = ndimage.histogram(distances, 0, self.max_distance,n_bins)

        return self.reference_histogram

    def plot_radial_density(self,axes, t ,n_bins, label, color, no_plot=False, cutoff_percentange = 10,
                            n_reference_points = 10000):
        """Goes through all tracks to find spots in the time slice t and finds all distances
        between them. The distances are assigned to the n_bin histogram bins.
        :axes Histogram will be plotted here
        :t Time at which the correlation function will taken
        :n_bins number of bins in the histogram
        :no_plot Does not plot on axes
        :cutoff_percentage The percentage of bins that will be cutoff at the end.
            The tail end
        :n_reference_points Number of points used for the reference histogramm. High values
            tax performance, but smoothen the reference density
        """
        #Prepare histogram
        bins = np.linspace(0, self.max_distance, n_bins+1)[:-1] #left edge of every bin
        histogram = np.zeros(n_bins)
        n_points = 0

        #Search all tracks for points
        points = []
        for track in self.tracks:
            tIdx, = np.where(np.array(track)[:,0]==t)

            if len(tIdx)>0: #Does the tracked particle exist at t?
                tIdx = tIdx[0] #Want first (and only) occurence
                new_point = track[tIdx][1:]
                n_points += 1
                #Compute all distances to the new point and place them in the histogram
                for old_point in points:
                    distance = self.distance(np.array(new_point)-np.array(old_point))
                    bin_idx = int(np.floor(distance/self.max_distance*n_bins))
                    histogram[bin_idx]+=1
                points.append(new_point)
        assert len(points)>0, 'No points found at this time.'

        #Normalise the density
        n_distances = (n_points-1)*(n_points)/2
        n_reference_distances = (n_reference_points-1)*(n_reference_points)/2

        tic = time.perf_counter()
        self.reference_histogram = self.generate_reference_histogram(n_reference_points,n_bins)
        toc = time.perf_counter()
        print(f"Reference Histogram with {n_reference_points:.0f} particles was computed in {toc - tic:f} seconds")

        #Multiply by the ratio of the numbers of distances between points in each histogram
        reference_density = (n_distances/n_reference_distances)*self.reference_histogram
        cutoff = int(np.floor(n_bins*cutoff_percentange/100)) #Cutoff tail end of the histogarm to avoid dividing by 0
        radial_density = np.divide(histogram[:-cutoff],reference_density[:-cutoff])
        # radial_density = histogram[:-cutoff]

        #Plot Histogram
        if no_plot==False:
            bin_midpoints = bins+(bins[1]-bins[0])/2
            axes.plot(bin_midpoints[:-cutoff], radial_density, label=label, color = color)
            axes.set_xlabel(r'Distance')
            axes.set_ylabel(r'Radial distribution function')
            axes.legend()

        return histogram, bins, points

    def get_measurementTimes(self):
        if self._measurementTimes==None:
            self._measurementTimes=[self.tracks[0][idx][0] for idx in range(len(self.tracks[0]))]
        return self._measurementTimes

    measurementTimes = property(get_measurementTimes)

##################################### End of experiment class ############################################

def plot_mixed_particle_radial_density(axes, experiments, t ,n_bins, n_reference_points, no_plot=False, cutoff_percentage=0):
    """Calculates a histogram of the the distances between the points in the two experiments, i.e. cross radial distribution
    function. Then plots them to axes.
    :axes Figure on which the plot is drawn
    :experiments List containing two experiment structs
    :t Time slice at which the radial distribution function will be conputed
    :n_bins Number of bins in the histogram
    :n_reference_points Number of points used in the computation of the reference histogram used to normalise the density.
        Comutatiois skipped if such a histogram is already safed in experiment[0]
    :no_plot If true no plot is drawn
    :cutoff_percentage Percentage of the histogram that won't be shown in the plot,
        useful when the tail end is high fluctuation
    """
    #Barricades
    assert(len(experiments)==2), "Expect exactly two experiments"
    assert(experiments[0].periodic == experiments[1].periodic), "Mixing periodic and non-periodic data"

    max_distance = max(experiments[0].max_distance,experiments[1].max_distance)
    bins = np.linspace(0, max_distance, n_bins+1)[:-1] #left edge of every bin
    histogram = np.zeros(n_bins)
    n_points = 0

    position_array = []
    for exp in experiments:
        points = []
        for track in exp.tracks:
            tIdx, = np.where(np.array(track)[:,0]==t)

            if len(tIdx)>0: #Does the tracked particle exist at t?
                tIdx = tIdx[0] #Only want first (and only) occurence
                points.append(track[tIdx][1:])
        position_array.append(points) #has spots from both experiments
    points1 = np.array(position_array[0])
    points2 = np.array(position_array[1])

    if(experiments[0].periodic==False): #Faster Computation for non-periodic data
        distances = spatial.distance.cdist(points1,points2)
    elif(experiments[0].periodic==True): #Manual compuatation for periodic data
        distances = np.zeros(len(points1)*len(points2))
        cnt = 0
        for pointA in points1:
            for pointB in points2:
                distances[cnt]=experiments[0].distance(pointA-pointB)
                cnt+=1

    histogram = ndimage.histogram(distances, 0, max_distance,n_bins)

    reference_histogram = experiments[0].generate_reference_histogram(n_reference_points,n_bins)
    #Multiply by the ratio of the numbers of links between points in each histogram
    n_distances = len(points1)*len(points2)
    n_reference_distances = n_reference_points*(n_reference_points-1)/2
    reference_histogram = (n_distances/n_reference_distances)*reference_histogram
    cutoff = int(np.floor(n_bins*cutoff_percentage/100)) #Cutoff tail end of the histogarm to avoid dividing by 0
    radial_density = np.divide(histogram[:-cutoff],reference_histogram[:-cutoff])

    #Plot Histogram
    if no_plot==False:
        bin_midpoints = bins+(bins[1]/2)
        axes.plot(bin_midpoints[:-cutoff], radial_density, label= 'Red-green RDF')
        axes.legend()
        axes.set_xlabel(r'Distance')
        axes.set_ylabel(r'Radial distribution function')

def load_simulation_data(PATH, trackIdx=""):
    #Load the data
    green = experiment(PATH)
    green.read_csv(f"{PATH}_tracks{trackIdx}.csv")
    params = green.read_parameter_file(PATH)
    nGreenParticles = int(params["nGreenParticles"])
    nRedParticles = int(params["nRedParticles"])

    #Don't exclude any tracks
    min_length = 0 #Tracks shorter than that are excluded

    #split up red and green particles (the simulation has green particles first and red particles second in its list)
    red = copy.deepcopy(green)
    green.tracks = green.tracks[:nGreenParticles]
    green.color = green.color[:nGreenParticles]
    red.tracks = red.tracks[nGreenParticles:nGreenParticles+nRedParticles]
    red.color = red.color[nGreenParticles:nGreenParticles+nRedParticles]

    return green, red, params

def get_positions_and_particle_types(experiments, time):
    """Extracts positions and particle types from experiment.
    0 is red, 1 is green and greenPlus"""
    points=[]
    colorIdx=[]
    for expIdx, exp in enumerate(experiments):
            for pIdx, track in enumerate(exp.tracks):
                    tIndices, = np.where(np.array(track)[:,0]==time)

                    if len(tIndices)>0: #Does the tracked particle exist at t?
                            tIdx = tIndices[0] #first occurence
                            colorIdx.append(0 if exp.color[pIdx][tIdx]==" red" else 1)
                            points.append(np.array(track[tIdx][1:]))
    return points, colorIdx

def get_distance_based_neighbourhood_graph(points, color, experiment, neighbourDistance):
    # Creates graph whose connected components are the clusters, defined by two particles being within neighbourhood distance
    #Initialise the graph
    G =  nx.Graph()
    #Add nodes
    for nodeIdx in range(len(points)):
        G.add_node(nodeIdx)
    #Add edges
    for pIdx1 in range(len(points)):
        for pIdx2 in range(pIdx1):
            dist = experiment.distance((points[pIdx1]-points[pIdx2]))
            if(dist < neighbourDistance):
                if(color[pIdx1]==color[pIdx2]):
                    G.add_edge(pIdx1, pIdx2)
                    G.add_edge(pIdx2, pIdx1)
    return G

def get_voronoi_based_neighbourhood_graph(points, color, experiment):
    """
    Creates graph whose connected components are the clusters, defined by two particles being within neighbourhood distance
    """
    # Create Voronoi tesselation
    embeddedCoordinates = [[point[0], point[1], 0.5] for point in points]
    tess = Container(embeddedCoordinates, limits=(experiment.x_max, experiment.y_max,1), periodic=(True, True, False))
    #Initialise the graph
    G =  nx.Graph()
    #Add nodes
    for nodeIdx in range(len(points)):
        G.add_node(nodeIdx)
    for idx, point in enumerate(tess):
        for neighbourIdx in point.neighbors():
            if(neighbourIdx>=0):
                if(color[idx]==color[neighbourIdx]):
                    G.add_edge(idx, neighbourIdx)
    return G

def calculate_mixing_index(experiments, t, neighbourDistance):
    """Calculates the mixing index of the two cell types.
    :experiments List containing two experiments with the tracks
    :t Time slice at which the mixing index function will be conputed
    :neighbourDistance How far cells can be apart and still count as
    neighbours. Giving None results in using Voronoi neighbourhood.
    """
    #Barricades
    assert(len(experiments)==2), "Expect exactly two experiments"
    assert(experiments[0].periodic == experiments[1].periodic), "Mixing periodic and non-periodic data"

    points, colorIdx = get_positions_and_particle_types(experiments, t)

    # Get colorblind neighbourhood graph
    if neighbourDistance==None:
        G = get_voronoi_based_neighbourhood_graph(points, np.zeros(len(points)), experiments[0])
    else:
        G = get_distance_based_neighbourhood_graph(points, np.zeros(len(points)), experiments[0], neighbourDistance)

    # Find how many same-type and cross-type neighbours each particle has
    sameTypeCnt = np.zeros(len(points))
    crossTypeCnt = np.zeros(len(points))

    for idx1 in range(len(points)):
        for idx2 in range(idx1):
            if G.has_edge(idx1, idx2): #Are they neighbours
                if colorIdx[idx1]==colorIdx[idx2]: #Same type?
                    sameTypeCnt[idx1] += 1
                    sameTypeCnt[idx2] += 1
                elif colorIdx[idx1] != colorIdx[idx2]: # Cross type
                    crossTypeCnt[idx1] += 1
                    crossTypeCnt[idx2] += 1

    #Discard 0 values and calculate mixing index for each particle (For Voronoi there should be no 0 values)
    neighbourCnt = sameTypeCnt+crossTypeCnt
    sameTypeCnt = np.array([sameTypeCnt[idx] for idx in range(len(neighbourCnt)) if neighbourCnt[idx]>0])
    crossTypeCnt = np.array([crossTypeCnt[idx] for idx in range(len(neighbourCnt)) if neighbourCnt[idx]>0])
    neighbourCnt = np.array([neighbourCnt[idx] for idx in range(len(neighbourCnt)) if neighbourCnt[idx]>0])
    colorIdx = [colorIdx[idx] for idx in range(len(neighbourCnt)) if neighbourCnt[idx]>0]

    mixing = (sameTypeCnt)/(neighbourCnt)
    mixingGreen = np.array([mixing[idx] for idx in range(len(mixing)) if colorIdx[idx]==1])
    mixingRed = np.array([mixing[idx] for idx in range(len(mixing)) if colorIdx[idx]==0])

    #Average
    mixingIdx = np.sum(mixing)/len(mixing)
    mixingIdxGreen = np.sum(mixingGreen)/len(mixingGreen)
    mixingIdxRed = np.sum(mixingRed)/len(mixingRed)

    return mixingIdx, mixingIdxGreen, mixingIdxRed

def write_avr_mixing_index(parameters, mixingIdx, FILE_PATH):
    """Appends the mixing index with it's parameters to FILE_PATH (creates new file if none is there).
    OUTDATED: Use write_mixing_index_to_file instead, it considers multiple realisations and time points.
    """
    file = open(FILE_PATH, "a") #append mode
    for name in parameters.keys():
        file.write(name+", %f, "%parameters[name])
    file.write("mixingIdx, %f \n"%mixingIdx)
    file.close()

def read_avr_mixing_index(FILE_PATH):
    """Reads all mixing indices and parameter values at FILE_PATH. Assumes format from write_mixing_index
    and two parameter values.
    OUTDATED: Use read_mixing_index_file instead, it works on different files that consider multiple realisations and time points.
    """
    names = []
    xValues = []
    yValues = []
    mixingIndices = []
    namesEmpty = True
    with open(FILE_PATH, 'r') as reader:
        for line in reader.readlines():
            idx1 = line.find(", ")
            idx2 = line.find(", ",idx1+1)
            idx3 = line.find(", ",idx2+1)
            idx4 = line.find(", ",idx3+1)
            idx5 = line.find(", ",idx4+1)
            idx6 = line.find(" ",idx5+2)
            if(namesEmpty):
                names.append(line[:idx1])
                names.append(line[idx2+2:idx3])
                namesEmpty=False
            xValues.append(float(line[idx1+2:idx2]))
            yValues.append(float(line[idx3+2:idx4]))
            mixingIndices.append(float(line[idx5+2:idx6]))
    return xValues, yValues, mixingIndices, names

def max_cluster_size(experiments, t, neighbourDistance = 1.05):
    # Get points
    points=[]
    color=[]
    for expIdx, exp in enumerate(experiments):
            for pIdx, track in enumerate(exp.tracks):
                    tIdx, = np.where(np.array(track)[:,0]==t)

                    if len(tIdx)>0: #Does the tracked particle exist at t?
                            tIdx = tIdx[0] #first occurence
                            color.append(expIdx)
                            points.append(np.array(track[tIdx][1:]))

    G = get_distance_based_neighbourhood_graph(points, color, experiments[0], neighbourDistance)

    #get connected components
    connectedComponentsSize = [len(comp) for comp in nx.connected_components(G)]
    return max(connectedComponentsSize)

def plot_cluster_histogram(axes, clusterSizes, label ,color, nBins=None):

    # G = get_distance_based_neighbourhood_graph(points, colorList, experiments[0], neighbourDistance)

    # #get connected components
    # connectedComponentsSize = [len(comp) for comp in nx.connected_components(G)]

    maxSize = max(clusterSizes)
    if (nBins==None):
        nBins = maxSize

    # Use non-equal bin sizes, such that they look equal on log scale.
    logbins = np.logspace(np.log10(1),np.log10(maxSize),num=nBins)

    # axes.hist(clusterSizes, bins=logbins, label = label, color=color, alpha=0.5)
    axes.hist(clusterSizes, bins=nBins, label = label, color=color, alpha=0.4)
    axes.set_xscale('log')
    axes.set_yscale('log')
    axes.set_xlabel("Cluster size")
    axes.set_ylabel("Number of clusters")

    # Return histogram
    return ndimage.histogram(clusterSizes, 1, maxSize, nBins)

def write_clusters_to_file(output_file, experiments, time, neighbourDistance, newRealisation=False):
    """Finds neighbourhood graph and saves the cluster distribution by appending output_file.
    Also records the time. If newRealisation=True it will create a line to indicate that.
    Assumes colorIdx = 0 is green, colorIdx=1 is red. Files will be of the form
    ## Realisation 2 (neighbour distance = 1.05)
    # t 0
    nClustersSize1 nGreenClustersSize1 nRedClustersSize1
    nClustersSize2 nGreenClustersSize2 nRedClustersSize2
    ...
    """
    # Find neigbourhood graph
    points, colorIdx = get_positions_and_particle_types(experiments, time)
    if neighbourDistance==None:
        graph = get_voronoi_based_neighbourhood_graph(points, colorIdx, experiments[0])
    else:
        graph = get_distance_based_neighbourhood_graph(points, colorIdx, experiments[0], neighbourDistance)

    # Write clustering to file
    with open(output_file, 'a') as f:
            if newRealisation!=False:
                f.write(f"## Realisation {newRealisation} (neighbour distance = {neighbourDistance})\n")
            f.write('# t %f\n' % time)

            connectedComponents = [comp for comp in nx.connected_components(graph)]
            clusterSizes = [len(comp) for comp in connectedComponents]
            clusterColor = [colorIdx[comp.pop()] for comp in connectedComponents] # .pop removes item from set
            # for size in range(1, max(clusterSizes)+1):
            #     f.write("%d \n" % clusterSizes.count(size))
            for clusterIdx in range(len(clusterSizes)):
                color = "red" if clusterColor[clusterIdx]==0 else "green"
                f.write(f"{clusterSizes[clusterIdx]} {color}\n")

def write_mixing_index_to_file(output_file, experiments, time, neighbourDistance, newRealisation=False):
    """Saves the cluster distribution save in the graph by appending the output_file.
    Also records the time. If newRealisation=True it will create a line to indicate that.
    """
    with open(output_file, 'a') as f:
            if newRealisation!=False:
                f.write(f"## Realisation {newRealisation} (neighbour distance = {neighbourDistance})\n")
            f.write('# t %f\n' % time)

            mixingIdx, mixingIdxGreen, mixingIdxRed = calculate_mixing_index(experiments, time, neighbourDistance)

            f.write(f"{mixingIdx} {mixingIdxGreen} {mixingIdxRed}\n")

def write_data_for_all_times(observable, output_file, greenData, redData, neighbourDistance, fileIdx):
    """Calculate the cluster distributions for the realisation described by green_data and red_data
    and writes it to output_file. If neighbourDistance=None, it will find the clusters
    using the Voronoi-based neighbourhood measure, otherwise it will use the distance-based method.
    "observable" can be either "mixingIdx" or "clustering": This determines which observable is calculated
    and written into the file.
    """
    newRealisation=fileIdx
    for time in greenData.measurementTimes:
        # Calculate clustering
        if observable == "clustering":
            write_clusters_to_file(output_file, [greenData, redData], time, neighbourDistance, newRealisation=newRealisation)
        # Calculate mixing index
        elif observable == "mixingIdx":
            write_mixing_index_to_file(output_file, [greenData, redData], time, neighbourDistance, newRealisation=newRealisation)
        else:
            raise ValueError("Unknown observable type; Can't write data.")

        newRealisation=False
    print("Data written for one realisation")

def write_data_for_all_realisations(observable, parameterFile, outputFile, neighbourDistance):
    """Goes through all tracks belonging to parameterFile and writes their cluster sizes into
    output for every time.
    "observable" can be either "mixingIdx" or "clustering": This determines which observable is calculated
    and written into the file.
    """

    outOfFiles = False
    fileIdx=0

    while(not outOfFiles):
        try: # See if file with this index exists
            if fileIdx==0: # See if unnumbered track file exists
                try:
                    green, red, params = load_simulation_data(parameterFile)
                except:
                    fileIdx += 1
            if fileIdx > 0: # numbered track file
                green, red, params = load_simulation_data(parameterFile, trackIdx=f"_{fileIdx}")
        except:
            outOfFiles = True
        if not outOfFiles:
            write_data_for_all_times(observable, outputFile, green, red, neighbourDistance, fileIdx)
            fileIdx += 1

def read_clustering_file(inputFile):
    """Reads out a file that contains cluster sizes for multiple time steps and realisations.
    Adds the numbers up over the different realisations and returns the accumulated numbers.
    """
    with open(inputFile, mode="r") as file:
        csvFile = csv.reader(file, delimiter=" ")

        measurementTimes = []
        clusterSizes = [] #clusterSizes[timeIdx][clusterIdx]
        clusterSizesGreen = [] #Clusters of only green particles
        clusterSizesRed = [] # Clusters of only red particles
        timeIdx = -1

        for line in csvFile:
            if line[0] =="##": #Next realisation?
                timeIdx = -1
            elif line[0]=="#": #Next time step?
                timeIdx += 1
                if len(measurementTimes) == timeIdx: # Never had this time step before?
                    measurementTimes.append(float(line[2]))
                    clusterSizes.append([])
                    clusterSizesGreen.append([])
                    clusterSizesRed.append([])
                assert(measurementTimes[timeIdx]==float(line[2]))

            else: # This line contains a cluster size
                clusterSizes[timeIdx].append(int(line[0]))
                if line[1]=="green":
                    clusterSizesGreen[timeIdx].append(int(line[0]))
                elif line[1]=="red":
                    clusterSizesRed[timeIdx].append(int(line[0]))
                else:
                    raise ValueError(f"Found unknown particle type in file {inputFile}")

    return clusterSizes, clusterSizesGreen, clusterSizesRed, measurementTimes

def read_mixing_index_file(inputFile):
    """Reads out a file that contains cluster sizes for multiple time steps and realisations.
    Averages the indeices over all realisations
    """
    nRealisations = 0
    measurementTimes = []
    mixingIdx = []
    mixingIdxGreen = []
    mixingIdxRed = []
    with open(inputFile, mode="r") as file:
        csvFile = csv.reader(file, delimiter=" ")
        for line in csvFile:
            if line[0] =="##": #Next realisation?
                timeIdx = -1 #Will be incremented before first use
                nRealisations += 1
            elif line[0]=="#": #Next time step?
                timeIdx += 1
                if len(measurementTimes) == timeIdx: # Never had this time step before?
                    measurementTimes.append(float(line[2]))
                    mixingIdx.append(0)
                    mixingIdxGreen.append(0)
                    mixingIdxRed.append(0)
            else: # This line contains a mixing index
                assert(len(measurementTimes)==len(mixingIdx))
                mixingIdx[timeIdx] += float(line[0])
                mixingIdxGreen[timeIdx] += float(line[1])
                mixingIdxRed[timeIdx] += float(line[2])

    # Average
    mixingIdx = np.array(mixingIdx)/nRealisations
    mixingIdxGreen = np.array(mixingIdxGreen)/nRealisations
    mixingIdxRed = np.array(mixingIdxRed)/nRealisations

    return mixingIdx, mixingIdxGreen, mixingIdxRed, measurementTimes

########################################### Animation ########################################################

def readTracksFile(filename):
    """This is similar to read_csv in analysis library, but has a different output structure and is
    only used here"""
    measurementTimes = []
    positionMeasurements = []
    colorMeasurements = []
    with open(filename, mode="r") as file:
        csvFile = csv.reader(file)
        for lines in csvFile:
            measurementTimes.append(float(lines[0]))
            positionMeasurements.append([])
            colorMeasurements.append([])
            nParticles = int((len(lines)-1)/3)
            for particleIdx in range(nParticles):
                colorMeasurements[-1].append(lines[1+3*particleIdx])
                positionMeasurements[-1].append([float(lines[1+3*particleIdx+1]), #x value
                                              float(lines[1+3*particleIdx+2])])   #y value

    return measurementTimes, colorMeasurements, np.array(positionMeasurements)

def getColors(colorMeasurement): ##ToDo: Get right colors
    c = []
    for color in colorMeasurement:
        if color==" red":
            c.append('red')
        elif color==" green":
            c.append('green')
        elif color==" greenPlus":
            c.append('green')
        else:
            assert False, "Invalid color value"
    return c

def animate_tracks(PATH):
    #Get parameters
    sim = experiment("test")
    params = sim.read_parameter_file(PATH)
    xLength = params["Length"]
    yLength = params["Length"]
    R = 0.5 #WARNING: Radius of the cells since we set r0=1
    transparency = 0.7 #transparency of circles

    # Get tracks
    try: # Single realisation
        measurementTimes, colorMeasurements, positionMeasurements = readTracksFile(PATH+"_tracks.csv")
    except: # Multiple realisations
        measurementTimes, colorMeasurements, positionMeasurements = readTracksFile(PATH+"_tracks_1.csv")
    positionMeasurements = np.array(positionMeasurements)

    positionMeasurements = np.array(positionMeasurements)

    #Set up figure
    fig,ax = plt.subplots()
    plt.axis([0, xLength, 0, yLength])
    plt.gca().set_aspect('equal', adjustable='box') #Makes both axis have some scaling while keeping the set limits

    #Record the animation
    camera = Camera(fig)
    for frameIdx in range(len(measurementTimes)):
        c = getColors(colorMeasurements[frameIdx])
        for particleIdx in range(len(positionMeasurements[0])):
            circle = plt.Circle((positionMeasurements[frameIdx][particleIdx][0],
                                        positionMeasurements[frameIdx][particleIdx][1]),
                                        radius=R, linewidth=0, color = c[particleIdx], alpha = transparency)
            plt.gca().add_artist(circle)
        plt.gca().set_title("Time = %f"%measurementTimes[frameIdx])
        camera.snap()
        #Delete all artists before next frame
        for artist in plt.gca().lines + plt.gca().collections:
            artist.remove()
    animation = camera.animate()
    animation.save(PATH+".mov")

def animate_tracks_same_col(PATH):
    #Get parameters
    sim = experiment("test")
    params = sim.read_parameter_file(PATH)
    xLength = params["Length"]
    yLength = params["Length"]
    exponentialExpRate = params["exponentialExpRate"]
    stepLimit = params["stepLimit"]
    stepDuration = params["stepDuration"]

    R = 0.5 #WARNING: Radius of the cells since we set r0=1
    transparency = 0.7 #transparency of circles

    # Get tracks
    try: # Single realisation
        measurementTimes, colorMeasurements, positionMeasurements = readTracksFile(PATH+"_tracks.csv")
    except: # Multiple realisations
        measurementTimes, colorMeasurements, positionMeasurements = readTracksFile(PATH+"_tracks_1.csv")
    positionMeasurements = np.array(positionMeasurements)

    positionMeasurements = np.array(positionMeasurements)

    #Set up figure
    fig,ax = plt.subplots()
    # plt.axis([0, xLength, 0, yLength])
    # plt.axis([0, xLength*np.exp(stepLimit*stepDuration*exponentialExpRate), 0, yLength*np.exp(stepLimit*stepDuration*exponentialExpRate)])
    plt.gca().set_aspect('equal', adjustable='box') #Makes both axis have some scaling while keeping the set limits

    #Record the animation
    camera = Camera(fig)
    for frameIdx in range(len(measurementTimes)):
        c = getColors(colorMeasurements[frameIdx])
        for particleIdx in range(len(positionMeasurements[0])):
            circle = plt.Circle((positionMeasurements[frameIdx][particleIdx][0],
                                        positionMeasurements[frameIdx][particleIdx][1]),
                                        radius=R, linewidth=0, color = 'black', alpha = transparency)
            plt.gca().add_artist(circle)
        plt.gca().set_title("Time = %f"%measurementTimes[frameIdx])
        plt.axis([0, xLength*np.exp(measurementTimes[frameIdx]*exponentialExpRate), 0, yLength*np.exp(measurementTimes[frameIdx]*exponentialExpRate)])

        camera.snap()
        #Delete all artists before next frame
        for artist in plt.gca().lines + plt.gca().collections:
            artist.remove()
    animation = camera.animate()
    animation.save(PATH+".mov")

def final_snapshot(PATH):
    #Get parameters
    sim = experiment("test")
    params = sim.read_parameter_file(PATH)
    xLength =params["Length"]
    yLength =params["Length"]
    R = 0.5 #WARNING: Radius of the cells since we set r0=1
    transparency = 0.7 #transparency of circles

    #Get tracks
    try: # Single realisation
        measurementTimes, colorMeasurements, positionMeasurements = readTracksFile(PATH+"_tracks.csv")
    except: # Multiple realisations
        measurementTimes, colorMeasurements, positionMeasurements = readTracksFile(PATH+"_tracks_1.csv")
    positionMeasurements = np.array(positionMeasurements)

    #Set up figure
    fig,ax = plt.subplots()
    plt.axis([0, xLength, 0, yLength])
    plt.gca().set_aspect('equal', adjustable='box') #Makes both axis have some scaling while keeping the set limits

    frameIdx = -1 # last frame
    c = getColors(colorMeasurements[frameIdx])
    for particleIdx in range(len(positionMeasurements[0])):
        circle = plt.Circle((positionMeasurements[frameIdx][particleIdx][0],
                            positionMeasurements[frameIdx][particleIdx][1]),
                            radius=R, linewidth=0, color = c[particleIdx], alpha = transparency)
        plt.gca().add_artist(circle)
    plt.gca().set_title("Time = %f"%measurementTimes[frameIdx])
    plt.savefig(PATH+"_final_frame.png")

# from numba import jit
# @jit(nopython=True)
# def generate_reference_histogram(n_particles, n_bins, x_max,y_max, seed=None):
#         '''Generates a radial density histogram for uniform distributed particles in a box of
#         size x_max y_max. This serves as a normalisation for the measured density.
#         :n_particles Number of particles that are generated
#         :n_bins Number of bins to be used. The bins are evenly distributed between 0 and max_distance
#             where max_distance is calculated from x_max and y_max
#         :x_max, y_max Dimensions of the rectangle for which we generate the density
#         :rng Random number generator to be used
#         '''

#         #Numba compatible r andom number genration
#         if seed!=None:
#             np.random.seed(seed)

#         #Generate all points
#         max_distance = np.sqrt(x_max**2+y_max**2)
#         x_values = np.random.random((n_particles))*x_max
#         y_values = np.random.random((n_particles))*y_max
#         points = np.stack((x_values,y_values),axis=1)

#         # distances = spatial.distance.pdist(points)
#         # reference_histogram = ndimage.histogram(distances, 0, max_distance,n_bins)

#         #Prepare histogram
#         bins = np.linspace(0, max_distance, n_bins+1)[:-1] #left edge of every bin
#         histogram = np.zeros(n_bins)

#         #calculate distances between all points
#         for n in range(n_particles):
#             for m in range(n):
#                 distance = self.distance(points[n]-points[m])
#                 bin_idx = int(np.floor(distance/max_distance*n_bins))
#                 histogram[bin_idx]+=1

#         reference_histogram = histogram

#         return reference_histogram

Códigos Materia Activa/código viejo (malo)/analysis_library.py


import networkx as nx
import matplotlib.pyplot as plt
import random
from randomwalker import Simulation
import numpy as np
import scipy
from readdata import set_G
import pandas as pd
import seaborn as sns
from collections import OrderedDict
from readdata import data_settings

def set_watts_strogatz(n, k, p): # Create a Watts Strogatz Network and assign signs to its edges.
    # G.edges[0, 1]['sign']
    edge_labels = {}
    edge_new = {}
    node_labels = {}
    G = nx.watts_strogatz_graph(n, k, p)

    for i,x,attr in G.edges(data=True):
        val = random.uniform(0,1)
        if val <= 0.5:
            G.add_edge(i,x, sign = '+')

        else:
            G.add_edge(i,x,sign = '-')

        edge_labels[i,x] = attr['sign']

    nx.set_node_attributes(G,1,'r')
    # print('Edge Attributes:')
    # print(nx.get_edge_attributes(G,'sign'))
    edge_attr = nx.get_edge_attributes(G,'sign')
    # print(f'edgelabels: {edge_labels}')
    return G, edge_attr

def comparison_dict(seed_node,rd_val,graph_nodes): # SRWR Scores with respect to the seed node.
    comparison = {}

    for idx, val in enumerate(graph_nodes):
        comparison[seed_node,val] = rd_val[idx]

    for a,b in comparison:
        if comparison[a,b] > 0:
            comparison[a,b] = '+'
        else:
            comparison[a,b] = '-'

    return comparison

def accuracy(seed_node,rd_val,graph_nodes,edge_attributes): # SRWR sign prediction comparison to real data signs.
    count = 0
    true = 0

    comparison = comparison_dict(seed_node,rd_val,graph_nodes)
    for a,b in edge_attributes:
            if a == seed_node:
                count+=1
                if edge_attributes[a,b] == comparison[a,b]:
                    true+=1
    accuracy_percentage = (true/count) * 100

    return accuracy_percentage

"""
Part 1: Working with a synthetic Watts-Strogatz network to check if SRWR works.
"""
#
# graph = set_watts_strogatz(8,5,0.05)
# simulation = Simulation(graph[0],seed_node=1,c=0.15,beta=0.1,gamma=0.1,epsilon=0.1)
#
#
# print(f"Initial State: {nx.get_edge_attributes(simulation.G,'sign')}")
#
# simulation.remove_edge(simulation.seed_node,0.8)
#
# simulation.normalize()
# iterate = simulation.iterate(simulation.normalize()[0],simulation.normalize()[1],simulation.seed_node)
# rd = iterate[1] - iterate[0]
#
# edge_attr = nx.get_edge_attributes(simulation.G,'sign')
# print(f'Accuracy Percentage: {accuracy(simulation.seed_node,rd,simulation.nodes,edge_attr)}%')
# print(edge_attr)
#
# pos = nx.spring_layout(simulation.G)
# nx.draw_networkx(simulation.G,pos,with_labels=True)
# nx.draw_networkx_edge_labels(simulation.G,pos,edge_labels=edge_attr,font_color='red',font_weight='bold')
# plt.show()

"""
Part 2: Use a real network BitcoinAlpha for SRWR score link prediction.
"""

graph_network = set_G()
clustering_coefficient = nx.clustering(graph_network[0],4)
# print(f'Clustering Coefficient: {clustering_coefficient}')
def run(beta, gamma, epsilon,step=0): # Runs the Simulation.
    if step == 1:
        print(f'Seed Node: {graph_network[2]}')
    simulate = Simulation(graph_network[0],seed_node=graph_network[2],c=0.15,beta=beta,gamma=gamma,epsilon=epsilon)
    edge_attributes = nx.get_edge_attributes(simulate.G, 'sign')
    if step == 1:
        simulate.remove_edge(simulate.seed_node,0.1)
    simulate.normalize()
    iterate = simulate.iterate(simulate.normalize()[0],simulate.normalize()[1],graph_network[2])
    rd = iterate[1] - iterate[0]
    print(f'Accuracy Percentage: {accuracy(graph_network[2],rd,simulate.nodes,edge_attributes)}%')
    return accuracy(graph_network[2],rd,simulate.nodes,edge_attributes)

parameter_values = np.arange(1,11,1)
accuracies = np.zeros((10,10))
step_iter = 1
for index,value in enumerate(parameter_values): # X = beta Y = Gamma. Looks for the best beta and gamma values
    for index2,value2 in enumerate(parameter_values):
        print(f'beta,gamma: {value,value2}')
        accuracies[index,index2] = run(value,value2,0.1,1)
        step_iter = 0

df = pd.DataFrame(accuracies)
ax = sns.heatmap(df, annot = True)
ax.set(xlabel='Beta',ylabel='Gamma')
plt.savefig('accuracies_parameter.png')

param = np.where(accuracies == np.amax(accuracies))
print(f'PARAM: {param}')

run(param[0][0],param[1][0],0.1,1) # Runs the simulation again with the most optimal beta and gamma values to get highest accuracy.

# data_settings() # For plots


import os
import logging
import logging.config

from cfg import Config

LOG_BASE_DIR = Config.APP_LOG_PATH
APP_NAME = Config.APP_NAME

LOG_APP_NAME = Config.LOG_APP_NAME
LOG_DB_NAME = Config.LOG_DB_NAME
LOG_ER_NAME = Config.LOG_ER_NAME
LOGGER_LEVEL = Config.LOGGER_LEVEL

app_log_path = os.path.join(LOG_BASE_DIR, APP_NAME)
if not os.path.exists(app_log_path):
    os.makedirs(app_log_path)

LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'standard': {
            'format': '[%(levelname)s][%(asctime)s][%(process)d][%(threadName)s:%(thread)d][%(filename)s:%(lineno)d][%(funcName)s][%(message)s]'
        },
        'simple': {
            'format': '[%(levelname)s][%(asctime)s][%(filename)s:%(lineno)d]%(message)s'
        },
        'collect': {
            'format': '%(message)s'
        }
    },
    'handlers': {
        # 终端日志
        'console': {
            'level': LOGGER_LEVEL,
            'class': 'logging.StreamHandler',
            'formatter': 'standard'
        },
        # 应用文件日志，收集info及以上的日志
        'default': {
            'level': LOGGER_LEVEL,
            'class': 'logging.handlers.TimedRotatingFileHandler',
            'filename': os.path.join(app_log_path, LOG_APP_NAME),  # 日志文件
            'when': 'D',
            'backupCount': 100,
            'formatter': 'standard',
            'encoding': 'utf-8',
        },
        # 应用文件日志，收集错误及以上的日志
        'error': {
            'level': 'ERROR',
            'class': 'logging.handlers.RotatingFileHandler',  # 保存到文件，自动切
            'filename': os.path.join(app_log_path, LOG_ER_NAME),  # 日志文件
            'maxBytes': 1024 * 1024 * 500,  # 日志大小 5M
            'backupCount': 100,
            'formatter': 'standard',
            'encoding': 'utf-8',
        },
        # 数据库层日志
        'db': {
            'level': LOGGER_LEVEL,
            'class': 'logging.handlers.RotatingFileHandler',  # 保存到文件，自动切
            'filename': os.path.join(app_log_path, LOG_DB_NAME),
            'maxBytes': 1024 * 1024 * 5,  # 日志大小 5M
            'backupCount': 5,
            'formatter': 'standard',
            'encoding': "utf-8"
        }
    },
    'loggers': {
        # logging.getLogger(__name__)拿到的logger配置
        '': {
            'handlers': ['default', 'console', 'error'],
            'level': LOGGER_LEVEL,
            'propagate': True,
        },
        # logging.getLogger('db')拿到的logger配置
        'db': {
            'handlers': ['db'],
            'level': LOGGER_LEVEL,
        },
    },
}

logging.config.dictConfig(LOGGING)
logger = logging.getLogger(__name__)
db_logger = logging.getLogger("db")

if __name__ == '__main__':
    logger.info('应用日志。。。')
    logger.error('应用出错日志。。。')
    db_logger.error('数据库访问日志')
    logger.debug('debug --- ')


import pandas as pd
from pandas.io import gbq
import numpy as np
import matplotlib.pyplot as plt
from sklearn.preprocessing import MinMaxScaler
from tensoflow import keras
from keras.models import Sequential
from keras.layers import LSTM
from keras.layers import Dense
import math
from sklearn.metrics import mean_squared_error

class future_ratio:

  #this is the url_form
  def __init__(self , base_url):
    self.base_url = base_url
    data = gbq.read_gbq(self.base_url , project_id='herbalites')
    self.df = data.reset_index()['amount']

  #prepare our Train and Test Data and change them into dataset
  def prepare_data(self):
      self.scaler = MinMaxScaler(feature_range = (0,1))
      self.df = self.scaler.fit_transform(np.array(self.df).reshape(-1,1))
      training_size = int(len(self.df) * 0.67)
      test_size = len(self.df) - training_size
      self.Train_data , self.Test_data = self.df[0:training_size , :] , self.df[training_size: len(self.df) , :1]
      X_train , Y_train = self.creat_dataset(self.Train_data , 100)
      X_test  , Y_test  = self.creat_dataset(self.Test_data , 100)
      X_train = X_train.reshape(X_train.shape[0] , X_train.shape[1] , 1)
      X_test  = X_test.reshape(X_test.shape[0] , X_test.shape[1] , 1)
      return X_train , X_test , Y_test , Y_train

  #function that create the dataset , i call it in the last part "prepare_data"
  @staticmethod
  def creat_dataset(data_set , time_step = 1):
    data_x = []
    data_y = []
    for i in range(len(data_set)-time_step-1):
      data_x.append(data_set[i : (i+time_step) ,0])
      data_y.append(data_set[i + (time_step) , 0])
    return np.array(data_x) , np.array(data_y)

  #now we gonna make and train our model
  def train_model(self):
    X_train , X_test , Y_test , Y_train = self.prepare_data()
    self.model = Sequential()
    self.model.add(LSTM(50 , return_sequences = True , input_shape = (100 , 1)))
    self.model.add(LSTM(50 , return_sequences = True))
    self.model.add(LSTM(50))
    self.model.add(Dense(1))
    self.model.compile(optimizer='adam',
                  loss = 'mean_squared_error',
                  metrics = ['accuracy'])
    self.model.fit(X_train , Y_train , validation_data = (X_test , Y_test) , epochs = 100)
    train_product = self.model.predict(X_train)
    test_product  = self.model.predict(X_test)
    train_product = self.scaler.inverse_transform(train_product)
    test_product  = self.scaler.inverse_transform(test_product)
    x_input = self.Test_data[len(self.Test_data) - 100 :].reshape(1 , -1)
    temp_input =  list(x_input)[0].tolist()
    return x_input , temp_input

  #after training and product our model , we cam use the following function to product future time sires
  def predect_future(self):
    x_input , temp_input = self.train_model()
    lat_liste  = []
    n_steps = 100
    i = 0
    for i in range(30):
      if len(temp_input) > 100:
        x_input = np.array(temp_input[1:])
        x_input = x_input.reshape(1 , -1)
        x_input = x_input.reshape((1 , n_steps , -1))
        y_input   = self.model.predict(x_input , verbose = 0)
        temp_input.extend(y_input[0].tolist())
        temp_input = temp_input[1:]
        lat_liste.extend(y_input.tolist())
      else :
        x_input = x_input.reshape((1, n_steps,1))
        y_input   = self.model.predict(x_input, verbose=0)
        temp_input.extend(y_input[0].tolist())
        lat_liste.extend(y_input.tolist())
    return lat_liste

  #and now after having the next 30 data , we can just show it in a graph
  def show_graph(self):
    next_future = self.predect_future()
    new  = np.arange(1,101)
    pred = np.arange(101,131)
    plt.figure(figsize = (18,6))
    plt.plot(new,self.scaler.inverse_transform(self.df[900:]))
    plt.plot(pred,self.scaler.inverse_transform(next_future))
    plt.show()

test = future_ratio("""SELECT * FROM `herbalites.braintree_herbaly_herbaly.transactions` LIMIT 1000""")
test.show_graph()


from __future__ import annotations

import asyncio
from copy import deepcopy
import sys
from math import inf
from threading import Thread
from typing import NewType, Union

import pbcoin.config as conf
from pbcoin.block import Block
from pbcoin.db import DB
from pbcoin.mempool import Mempool
from pbcoin.cli_handler import CliServer
from pbcoin.utils.netbase import Addr
from pbcoin.network import Node
from pbcoin.process_handler import ProcessingHandler
from pbcoin.wallet import Wallet
from pbcoin.blockchain import BlockChain
from pbcoin.mine import Mine
from pbcoin.logger import getLogger
import pbcoin.core as core

logging = getLogger(__name__)

def create_core():
    """Create the core objects"""
    core.WALLET = Wallet()
    core.BLOCK_CHAIN = BlockChain([])
    core.MEMPOOL = Mempool()
    core.MINER = Mine(core.BLOCK_CHAIN, core.WALLET, core.MEMPOOL, core.NETWORK)
    core.DATABASE = DB()
    core.DATABASE.init()

inf_type = NewType("inf_type", float)

async def mine_starter(how_many: Union[inf_type, int]=inf):
    """Set up and start mining

        how_many: it determines how many blocks should be mined.If it doesn't pass
        then it is valued with infinity which means mining forever.
    """
    if conf.settings.glob.mining:
        core.BLOCK_CHAIN.is_full_node = conf.settings.glob.full_node
        core.BLOCK_CHAIN.cache = conf.settings.glob.cache * 1000  # to bytes
        if how_many == inf:
            while True:
                await core.MINER.mine()
                Block.update_outputs(deepcopy(core.MINER.setup_block), core.ALL_OUTPUTS)
        else:
            for _ in range(how_many):
                await core.MINER.mine()
                core.MINER.setup_block.update_outputs(core.ALL_OUTPUTS)

async def setup_network(has_cli, has_socket_network):
    """Set up network for connect other nodes and cli"""
    logging.info(f"your public key is generated: {core.WALLET.public_key}")
    handlers = []
    if has_socket_network:
        addr = Addr(conf.settings.network.ip,
                    conf.settings.network.port,
                    pub_key=core.WALLET.public_key)  # TODO: make a valid public key
        proc_handler = ProcessingHandler(core.BLOCK_CHAIN,
                                         core.ALL_OUTPUTS,
                                         core.WALLET,
                                         core.MEMPOOL)
        core.NETWORK = Node(addr, proc_handler, None)
        await core.NETWORK.start_up(conf.settings.network.seeds)
        core.MINER.node = core.NETWORK
        handlers.append(core.NETWORK.listen())
    if has_cli:
        cli_server = CliServer(conf.settings.network.socket_path)
        handlers.append(cli_server.start())
    loop = asyncio.get_event_loop()
    all_net = await asyncio.gather(*handlers)
    loop.run_until_complete(all_net)

def run(raise_runtime_error=True, how_many_mine: Union[inf_type, int]=inf, reset=True) -> None:
    """Run and start the node with option that gets

        Note: this function doesn't configure app

        Parameters
        ----------
            raise_runtime_error: bool = True
                if this is True then exit and raise threads runtime error
                for example KeyboardInterrupt
            how_many_mine:  Union[inf_type, int] = inf
                it determines how many blocks should be mined.If it doesn't pass
                then it is valued with infinity which means mining forever.
    """
    if conf.settings.logger.do_logging:
        # Clear logfile
        with open(conf.settings.logger.log_filename, "w") as _:
            pass
    if reset:
        create_core()
    getLogger("asyncio", False)  # for asyncio logging
    try:
        net_thread = None
        mine_thread = None
        # network thread
        if conf.settings.glob.network:
            net_thread = Thread(target=asyncio.run,
                                args=(setup_network(conf.settings.network.cli,
                                                    conf.settings.network.socket_network),),
                                kwargs={"debug": conf.settings.glob.debug},
                                name="network",
                                daemon=True)
            net_thread.start()
            logging.debug(f"socket is made in {conf.settings.network.socket_path}")
        # mine thread
        if conf.settings.glob.mining:
            mine_thread = Thread(target=asyncio.run,
                                 args=(mine_starter(how_many_mine),),
                                 kwargs={"debug": conf.settings.glob.debug},
                                 name="mining", daemon=True)
            mine_thread.start()

        if raise_runtime_error:
            while True:
                if net_thread is not None and net_thread.is_alive():
                    net_thread.join(0.01)
                if mine_thread is not None and mine_thread.is_alive():
                    mine_thread.join(0.01)
        else:
            if net_thread is not None:
                net_thread.join()
            if mine_thread is not None:
                mine_thread.join()

    except KeyboardInterrupt:
        sys.exit(1)

    except Exception:
        logging.critical("app was suddenly stopped", exc_info=sys.exc_info())


import sys
sys.path.append('./src')

# Spark imports
from pyspark.sql import SparkSession
from common.settings import ENTITY_ORBIS_ID

# Debug
def enrich_df_on(df, enrichement_df_list):
    enriched_df = df
    for enrichement_df in enrichement_df_list:
        enriched_df = enriched_df \
            .join(enrichement_df, [ENTITY_ORBIS_ID], how="left")
    return enriched_df

def main():
    """
    Usage: Bulk
    """

    print("ðŸš€ Starting up analysis.")

    spark = SparkSession \
        .builder \
        .appName("etl19_adapter_orbis") \
        .config("spark.sql.legacy.parquet.datetimeRebaseModeInWrite", "CORRECTED") \
        .getOrCreate()

    # Read datasets from infocamere bulk
    imprese_con_dati_periodici_df = spark.read.parquet("/data/imprese_con_dati_periodici_df.nodes.parquet")

    persone_giuridiche_da_censire_df = spark.read.parquet("/data/persone_giuridiche_da_censire.parquet")
    titolari_da_censire_df = spark.read.parquet("/data/titolari_da_censire_df.parquet")
    partecipanti_a_eventi_da_censire_df = spark.read.parquet("/data/partecipanti_a_eventi_da_censire_df.parquet")

    # Read datasets from orbis bulk financials
    orbis_key_financials_df = spark.read.parquet("/data/orbis_key_financials_df.nodes.parquet")
    orbis_industry_financials_df = spark.read.parquet("/data/orbis_industry_financials_df.nodes.parquet")
    orbis_bank_financials_df = spark.read.parquet("/data/orbis_bank_financials_df.nodes.parquet")
    financials_dataframes = [orbis_key_financials_df, orbis_industry_financials_df, orbis_bank_financials_df]

    # Merge datasets with financials

    imprese_con_dati_periodici_with_financials_df = enrich_df_on(imprese_con_dati_periodici_df, financials_dataframes)

    persone_giuridiche_da_censire_with_financials_df = enrich_df_on(persone_giuridiche_da_censire_df, financials_dataframes)
    titolari_da_censire_with_financials_df = enrich_df_on(titolari_da_censire_df, financials_dataframes)
    partecipanti_a_eventi_with_financials_df = enrich_df_on(partecipanti_a_eventi_da_censire_df, financials_dataframes)

    # Write datasets
    imprese_con_dati_periodici_with_financials_df.write.mode('overwrite').parquet("/data/imprese_con_dati_periodici_with_financials_df.nodes.parquet")
    persone_giuridiche_da_censire_with_financials_df.write.mode('overwrite').parquet("/data/persone_giuridiche_da_censire_with_financials_df.nodes.parquet")
    titolari_da_censire_with_financials_df.write.mode('overwrite').parquet("/data/titolari_da_censire_with_financials_df.nodes.parquet")
    partecipanti_a_eventi_with_financials_df.write.mode('overwrite').parquet("/data/partecipanti_a_eventi_with_financials_df.nodes.parquet")
    spark.stop()

if __name__ == "__main__":
    main()


import pandas as pd
import gensim.downloader as api
from wrapper.ScikitWrapper import ScikitWrapper
from utils.EmbeddingsUtils import EmbeddingsUtils
from sklearn.linear_model import LogisticRegressionCV
from sklearn import svm
from gensim import models

class Trainer:
    """
    This class trains the machine learning models
    """
    def run(self):
        """
        Trains the classifier using logistic regression and saves the models
        """
        print("Loading the emebeddings...")
        google_repository = models.KeyedVectors.load_word2vec_format(
            '../data/pre_trained_models/GoogleNews-vectors-negative300.bin.gz', binary=True)
        wiki_repository = api.load("glove-wiki-gigaword-100")

        google_utils = EmbeddingsUtils(google_repository, google_repository.vector_size)
        wiki_utils = EmbeddingsUtils(wiki_repository, wiki_repository.vector_size)
        google_scikit_wrapper = ScikitWrapper(LogisticRegressionCV(class_weight='balanced',
                                                                   multi_class='multinomial',
                                                                   solver='lbfgs',
                                                                   max_iter=2000))

        wiki_scikit_wrapper = ScikitWrapper(svm.SVC(decision_function_shape='ovo'))

        print("Trains the models...")
        self.__train_helper(google_utils, google_scikit_wrapper, 'google')
        self.__train_helper(wiki_utils, wiki_scikit_wrapper, 'wiki')

    def __train_helper(self, utils, scikit_wrapper, repo_name):

        dataframe = pd.read_csv('../data/classifier_data.csv')
        embeddings = []
        classes = []

        for index, row in dataframe.iterrows():
            if repo_name == 'wiki':
                row['string'] = row['string'].lower()
            embedding = utils.embedding_finder(row['string'])
            if embedding is not None:
                embeddings.append(embedding)
                classes.append(row['class'])

        scikit_wrapper.model_trainer(embeddings, classes)
        print(f'{repo_name} model trained with accuracy: {scikit_wrapper.model_scorer(embeddings, classes)}')
        scikit_wrapper.save_ml_model(repo_name)

if __name__ == '__main__':
    trainer = Trainer()
    trainer.run()


import numpy as np
from sklearn.cluster import KMeans
""" Inspiration from https://github.com/SimplisticCode/ODA-ML/blob/master/NearestSubclassCentroid.py """

class NearestSubclassCentroid:

    def __init__(self):
        self.k_means_models = []

    def fit(self, x_train, y_train, n_sub_classes):
        """ Find the number of labels https://numpy.org/doc/stable/reference/generated/numpy.unique.html """
        lbls = np.unique(y_train)

        for i in range(np.min(lbls), np.max(lbls)):
            indx = np.where(y_train == i)
            self.k_means_models.append((i, KMeans(n_sub_classes).fit(x_train[indx]).cluster_centers_))

    def predict(self, x):
        if len(self.k_means_models) <= 0:
            print("You need to call predict first")
        else:
            y_pred = np.full([len(x), ], None)
            for x_idx, x_val in enumerate(x):
                distances = []
                for idx, (i, centroids) in enumerate(self.k_means_models):
                    for sub_k in range(len(centroids)):
                        distances.append((np.linalg.norm(x_val - centroids[sub_k]), i))

                """ Sort Distances to Centroids
                https://www.programiz.com/python-programming/methods/list/sort """
                sortedDistances = sorted(distances, key=lambda y: y[0])
                """ Take the first objects label """
                y_pred[x_idx] = int(sortedDistances[0][1])
        return y_pred.astype(int)


import numpy as np
import pygame
import glob
import os
import neat
from sklearn.model_selection import train_test_split
from rocketNeat import GameManager, Player, Wall

pygame.init()
game = GameManager()

output_dir = 'model_output/rocketNEAT/'
if not os.path.exists(output_dir):
    os.makedirs(output_dir)

#the state is rocket y coordinate, next wall y coordinate, and next wall x coordinate
state_size = 3
#the actions are go up, do nothing, and go down.
action_size = 3

def softmax(x):
    e_x = np.exp(x - np.max(x))
    return e_x / e_x.sum(axis=0)

def load_checkpoint(path, config):
    if os.path.exists(path):
        files = glob.glob('{}*'.format(path))
        if len(files) == 0:
            return neat.Population(config)
        latest_file = max(files, key = os.path.getctime)
        print("weights loaded")
        return neat.Checkpointer.restore_checkpoint(latest_file)

def getState(index):
        states = game.get_states()
        state = states[index]
        state = np.array(state)/100
        def scale(X, x_min, x_max):
            nom = (X-X.min(axis=0))*(x_max-x_min)
            denom = X.max(axis=0) - X.min(axis=0)
            return x_min + nom/denom
        return state

def eval_genomes(genomes, config):
    nets = []

    for genome_id, genome in genomes:
        genome.fitness = 0.0
        net = neat.nn.FeedForwardNetwork.create(genome, config)
        nets.append(net)

    done = False
    scores = []
    game.start(len(genomes))
    while not done:
        actions = []
        for index, rocket in enumerate(game.population):
            action_dist = nets[index].activate(getState(index))
            action_dist = softmax(action_dist)
            action = np.argmax(action_dist)
            actions.append(action)

        game.step(actions)
        game.render()
        done = game.stopLoop
        if done:
            for index, rocket in enumerate(game.population):
                scores.append(rocket.score)

    for genome_id, genome in genomes:
        genome.fitness = scores[0]
        scores.pop(0)

def run(config_file):
    config = neat.config.Config(neat.DefaultGenome, neat.DefaultReproduction,
                         neat.DefaultSpeciesSet, neat.DefaultStagnation,
                         config_file)

    p = load_checkpoint(output_dir, config)

    p.add_reporter(neat.StdOutReporter(True))
    stats = neat.StatisticsReporter()
    p.add_reporter(stats)
    p.add_reporter(neat.Checkpointer(5))

    winner = p.run(eval_genomes, 200)
    print('\nBest genome:\n{!s}'.format(winner))

if __name__ == '__main__':
    local_dir = os.path.dirname(__file__)
    config_path = os.path.join(local_dir, 'config-feedforward.txt')
    run(config_path)


from tkinter.ttk import Notebook

class Autoresized_Notebook(Notebook):
  def __init__(self, master=None, **kw):

    Notebook.__init__(self, master, **kw)
    self.bind("<<NotebookTabChanged>>", self._on_tab_changed)

  def _on_tab_changed(self,event):
    event.widget.update_idletasks()

    tab = event.widget.nametowidget(event.widget.select())
    event.widget.configure(height=tab.winfo_reqheight())

if __name__== "__main__":
    from tkinter import Frame, Tk
    root = Tk()

    notebook = Autoresized_Notebook(root)
    notebook.add(Frame(notebook, width=400, height=200, name="a"),text="TAB 1")
    notebook.add(Frame(notebook, width=400, height=300, name="b"),text="TAB 2")
    notebook.add(Frame(notebook, width=400, height=100, name="c"),text="TAB 3")

    notebook.pack()

    root.mainloop()


from api import MovieClient
from pprint import pprint

def main():
    input_loop()

def input_loop():
    while True:
        val = input("What kind of search would you like to perform? "
                    "Options are [k]eyword, [d]irector, or [i]mdb code. Alternatively, you may type 'x' to exit.\n>>> ")
        if not val:
            raise ValueError("Must specify a search type.")
        if val == 'x':
            return
        if val not in ['k', 'd', 'i']:
            raise ValueError("Invalid input.")

        try:
            if val == 'k':
                keyword = input("Please enter keyword\n>>> ").strip()
                get_movies_by_keyword(keyword)
            elif val == 'd':
                pass
            elif val == 'i':
                pass
        except ValueError as ve:
            print(ve)
            continue

def get_movies_by_keyword(keyword):
    svc = MovieClient()
    response = svc.find_by_keyword(keyword)
    movies = response.json().get('hits')

    print(f"Keyword {keyword} returned {len(movies)} results.")
    for idx, movie in enumerate(movies, 1):
        print(f"{idx}. {movie.get('title')} - {movie.get('year')}, directed by {movie.get('director')}")

    selected = get_movie_selection(movies)

    try:
        selected_movie = movies[selected - 1]
        pprint(selected_movie)
    except ValueError as ve:
        print(ve)
    get_movies_by_keyword(keyword)

def get_movie_selection(movies):
    selected = input(f"Please select one you'd like to see more info about, or enter 'x' to exit this search\n>>>")
    if selected == 'x':
        main()
    if not selected:
        raise ValueError("Must specify a number")
    elif not selected.isdigit():
        raise ValueError("Input must be a number.")
    elif int(selected) - 1 not in range(0, len(movies)):
        raise ValueError("Number must be within range of movies")
    else:
        return int(selected)
    # try:
    #     selected_movie = movies[selected - 1]
    #     pprint(selected_movie)
    #     get_movies_by_keyword(keyword)
    # except ValueError as ve:
    #     print(ve)

def get_movies_by_director(director):
    svc = MovieClient()

def get_movie_by_imdb_code(imdb_code):
    svc = MovieClient()

if __name__ == '__main__':
    main()


paths = []
edges = {}

def search(a, b, path):
    global paths
    global edges
    for idx in range(len(edges[a])):
        if edges[a][idx] == b:
            path.append(edges[a][idx])
            paths.append(path)
        elif edges[a][idx].isupper() or (edges[a][idx].islower() and edges[a][idx] not in path):
            newPath = path.copy()
            newPath.append(edges[a][idx])
            search(edges[a][idx],b,newPath)

lines = []
with open('inputs/day12.txt') as f:
    lines = f.readlines()

for line in lines:
    edge = line.strip().split('-')
    if edge[0] == 'start':
        if 'start' not in edges:
            edges['start'] = []
        if edge[1] not in edges['start']:
            edges['start'].append(edge[1])
    elif edge[1] == 'start':
        if 'start' not in edges:
            edges['start'] = []
        if edge[0] not in edges['start']:
            edges['start'].append(edge[0])
    elif edge[0] == 'end':
        if edge[1] not in edges:
            edges[edge[1]] = []
        if 'end' not in edges[edge[1]]:
            edges[edge[1]].append('end')
    elif edge[1] == 'end':
        if edge[0] not in edges:
            edges[edge[0]] = []
        if 'end' not in edges[edge[0]]:
            edges[edge[0]].append('end')
    else:
        if edge[0] not in edges:
            edges[edge[0]] = []
        if edge[1] not in edges[edge[0]]:
            edges[edge[0]].append(edge[1])
        if edge[1] not in edges:
            edges[edge[1]] = []
        if edge[0] not in edges[edge[1]]:
            edges[edge[1]].append(edge[0])

search('start','end', ['start'])

print(len(paths))


import re
from typing import Any, Dict, List

def parse_dumpsys_package_for_details(output: str) -> Dict[str, Any]:
    """
    Parse one entry of a dumpsys package information
    """
    details = {
        "uid": "",
        "version_name": "",
        "version_code": "",
        "timestamp": "",
        "first_install_time": "",
        "last_update_time": "",
        "permissions": [],
        "requested_permissions": [],
    }

    in_install_permissions = False
    in_runtime_permissions = False
    in_declared_permissions = False
    in_requested_permissions = True
    for line in output.splitlines():
        if in_install_permissions:
            if line.startswith(" " * 4) and not line.startswith(" " * 6):
                in_install_permissions = False
            else:
                lineinfo = line.strip().split(":")
                permission = lineinfo[0]
                granted = None
                if "granted=" in lineinfo[1]:
                    granted = "granted=true" in lineinfo[1]

                details["permissions"].append(
                    {"name": permission, "granted": granted, "type": "install"}
                )

        if in_runtime_permissions:
            if not line.startswith(" " * 8):
                in_runtime_permissions = False
            else:
                lineinfo = line.strip().split(":")
                permission = lineinfo[0]
                granted = None
                if "granted=" in lineinfo[1]:
                    granted = "granted=true" in lineinfo[1]

                details["permissions"].append(
                    {"name": permission, "granted": granted, "type": "runtime"}
                )

        if in_declared_permissions:
            if not line.startswith(" " * 6):
                in_declared_permissions = False
            else:
                permission = line.strip().split(":")[0]
                details["permissions"].append({"name": permission, "type": "declared"})
        if in_requested_permissions:
            if not line.startswith(" " * 6):
                in_requested_permissions = False
            else:
                details["requested_permissions"].append(line.strip())

        if line.strip().startswith("userId="):
            details["uid"] = line.split("=")[1].strip()
        elif line.strip().startswith("versionName="):
            details["version_name"] = line.split("=")[1].strip()
        elif line.strip().startswith("versionCode="):
            details["version_code"] = line.split("=", 1)[1].strip()
        elif line.strip().startswith("timeStamp="):
            details["timestamp"] = line.split("=")[1].strip()
        elif line.strip().startswith("firstInstallTime="):
            details["first_install_time"] = line.split("=")[1].strip()
        elif line.strip().startswith("lastUpdateTime="):
            details["last_update_time"] = line.split("=")[1].strip()
        elif line.strip() == "install permissions:":
            in_install_permissions = True
        elif line.strip() == "runtime permissions:":
            in_runtime_permissions = True
        elif line.strip() == "declared permissions:":
            in_declared_permissions = True
        elif line.strip() == "requested permissions:":
            in_requested_permissions = True

    return details

def parse_dumpsys_packages(output: str) -> List[Dict[str, Any]]:
    """
    Parse the dumpsys package service data
    """
    pkg_rxp = re.compile(r"  Package \[(.+?)\].*")

    results = []
    package_name = None
    package = {}
    lines = []
    for line in output.splitlines():
        if line.startswith("  Package ["):
            if len(lines) > 0:
                details = parse_dumpsys_package_for_details("\n".join(lines))
                package.update(details)
                results.append(package)
                lines = []
                package = {}

            matches = pkg_rxp.findall(line)
            if not matches:
                continue

            package_name = matches[0]
            package["package_name"] = package_name
            continue

        if not package_name:
            continue

        lines.append(line)

    if len(lines) > 0:
        details = parse_dumpsys_package_for_details("\n".join(lines))
        package.update(details)
        results.append(package)

    return results


"""Added new fields to user table

Revision ID: 0de77b29f731
Revises: 7392fbc36f06
Create Date: 2023-01-03 17:32:29.474076

"""
import sqlalchemy as sa
from alembic import op

# revision identifiers, used by Alembic.
revision = "0de77b29f731"
down_revision = "7392fbc36f06"
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table("user", schema=None) as batch_op:
        batch_op.add_column(
            sa.Column("first_name", sa.String(length=200), nullable=False)
        )
        batch_op.add_column(
            sa.Column("last_name", sa.String(length=200), nullable=False)
        )
        batch_op.add_column(
            sa.Column("username", sa.String(length=200), nullable=False)
        )
        batch_op.create_unique_constraint(None, ["username"])

    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table("user", schema=None) as batch_op:
        batch_op.drop_constraint(None, type_="unique")
        batch_op.drop_column("username")
        batch_op.drop_column("last_name")
        batch_op.drop_column("first_name")

    # ### end Alembic commands ###


from bs4 import BeautifulSoup as bs
import datetime as dt
from splinter import Browser
import pandas as pd
from webdriver_manager.chrome import ChromeDriverManager

def scrape_info():
    executable_path = {'executable_path': ChromeDriverManager().install()}
    browser = Browser('chrome', **executable_path, headless=False)

    title, paragraph = mars_news(browser)

    data = {
        "title": title,
        "paragraph": paragraph,
        "featured_image": featured_image(browser),
        "hemispheres": hemispheres(browser),
        "last_modified": dt.datetime.now()
        }

    browser.quit()
    return data

def mars_news(browser):
    url = "https://mars.nasa.gov/news/?page=0&per_page=40&order=publish_date+desc%2Ccreated_at+desc&search=&category=19%2C165%2C184%2C204&blank_scope=Latest"
    browser.visit(url)

    browser.is_element_present_by_css('div.list_text', wait_time=1)

    html = browser.html
    news_soup = bs(html, 'html.parser')

    try:
        slide_elem = news_soup.select_one('div.list_text')
        title = slide_elem.find('div', class_="content_title").get_text()
        paragraph = slide_elem.find('div', class_='article_teaser_body').get_text()
    except AttributeError:
        return None, None

    return title, paragraph

def featured_image(browser):
    url = 'https://spaceimages-mars.com'
    browser.visit(url)

    full_image_elem = browser.find_by_tag('button')[1]
    full_image_elem.click()

    html = browser.html
    img_soup = bs(html, 'html.parser')

    try:
        img_url_rel = img_soup.find('img', class_='fancybox-image').get('src')

    except AttributeError:
        return None

    img_url = f'https://spaceimages-mars.com/{img_url_rel}'

    return img_url

def mars_facts():
    facts_url = 'http://space-facts.com/mars/'

    mars_facts = pd.read_html(facts_url)
    mars_df = mars_facts[0]
    mars_df.columns = ['Description','Value']
    mars_df.set_index('Description', inplace=True)
    facts = mars_df.to_html()
    return facts

def hemispheres(browser):
    url = 'https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars'

    browser.visit(url + 'index.html')

    next_page_url = []
    image_title = []
    base_url = 'https://astrogeology.usgs.gov'

    html = browser.html
    soup = bs(html, 'html.parser')
    results_3 = soup.find_all('div', class_='description')

    number = 0
    for result in results_3:
        href=result.find('a')['href']
        img_title = result.a.find('h3')
        img_title_text = img_title.text
        image_title.append(img_title_text)
        next_page = base_url + href
        next_page_url.append(next_page)
        number = number+1

    images_list = []

    for x in next_page_url:
        browser.visit(x)
        html = browser.html
        img_soup = bs(html, 'html.parser')
        image = img_soup.find('img', class_='wide-image')['src']
        hem_image = base_url = image
        images_list.append(hem_image)

    hemisphere_url = []

    valles = {"title": image_title[3], "img_url": images_list[3]}
    cerberus = {"title": image_title[0], "img_url": images_list[0]}
    schiaparelli = {"title": image_title[1], "img_url": images_list[1]}
    syrtis = {"title": image_title[2], "img_url": images_list[2]}

    hemisphere_image_urls = [valles, cerberus, schiaparelli, syrtis]

    return hemisphere_image_urls

def scrape_hemisphere(html_text):
    hemi_soup = bs(html_text, "html.parser")

    try:
        title_elem = hemi_soup.find("h2", class_="title").get_text()
        sample_elem = hemi_soup.find("a", text="Sample").get("href")

    except AttributeError:
        title_elem = None
        sample_elem = None

    hemispheres = {
        "title": title_elem,
        "img_url": sample_elem
    }

    return hemispheres

if __name__ == "__main__":
    scrape_info()


# iris dataset으로 군집화

import pandas as pd
import matplotlib.pyplot as plt
plt.rc('font', family='malgun gothic')
from sklearn.datasets import load_iris
from scipy.spatial.distance import pdist, squareform
from scipy.cluster.hierarchy import linkage, dendrogram

iris=load_iris()
iris_df=pd.DataFrame(iris.data, columns=iris.feature_names)
print(iris_df.head(3))
print('--------거리제기-------')
# dist_vec=pdist(iris_df.loc[0:4,['sepal length (cm)','sepal width (cm)']], metric='euclidean')
dist_vec=pdist(iris_df.loc[:,['sepal length (cm)','sepal width (cm)']], metric='euclidean')
print('dist_vec',dist_vec)
print()

# 보기편하게 squareform에 넣어볼게요
row_dist=pd.DataFrame(squareform(dist_vec))
print(row_dist)

# 우리의 찐 목적은 거리를 보는게 아니라 군집화 하려는거야
# 계층적 군집분석(비지도학습)
row_clusters=linkage(dist_vec, method='complete')
print('row_clusters \n',row_clusters)

df=pd.DataFrame(row_clusters, columns=['군집id1', '군집id2','거리','멤버수'])
print(df)

# dendrogram으로 row_clusters를 시각화
low_dend=dendrogram(row_clusters)
plt.ylabel('유클리드 거리')
plt.show()


###define card deck####
import random
def deck():
	cards = []
	rank_cards = '2 3 4 5 6 7 8 9 10 Joker Queen king Ace'.split()
	face_cards = 'spade heart diamond club'.split()
	for rank_card in range(13):
		for face_card in range(4):
			card_string = rank_cards[rank_card] + "of" + face_cards[face_card]
			cards.append(card_string)
	return cards

#print(deck())

###deal the card####2 cards each randomly###
def deal():
	test_cards = deck()
#	print(test_cards)
	player_cards = []
	dealer_cards = []
#	random.shuffle(test_cards)
	while len(player_cards) < 2:
		random.shuffle(test_cards)
		for card in test_cards:
			print(card)
			player_cards.append(card)
#			test_cards.remove(card)
#	for card in test_cards:
#		random.shuffle(test_cards)
#		while len(dealer_cards) < 2:
#			dealer_cards.append(card)
	print(player_cards)
#	print(dealer_cards)
deal()

deal()

###play the game###deal or stay###

####check for winner###card count more than 21###


def encode(key, msg):
    encodedMsg = ""
    for i in range(len(msg)):
        if(ord(msg[i]) + int(key) > 126): # if the encoded character exceeds ascii limits (we will just loop around)
            encodedMsg += (chr(ord(msg[i]) + int(key) - 126 + 32))
        else:
            encodedMsg += (chr(ord(msg[i]) + int(key)))
    print("Message: " + encodedMsg)
    print("Key: " + key)

def decode(key, msg):
    decodedMsg = ""
    for i in range(len(msg)):
        if(ord(msg[i]) - int(key) < 32):
            decodedMsg += (chr(ord(msg[i]) - int(key) + 126 - 32))
        else:
            decodedMsg += (chr(ord(msg[i]) - int(key)))
    print("Message: " + decodedMsg)
    print("Key: " + key)

userIn = ""
while(userIn.casefold() != "q" and userIn.casefold() != "exit" and userIn.casefold() != "quit"): # infinite loop until quit
    userIn = input("Encode (e) or Decode (d)?\n> ")
    if(userIn.casefold() == "e" or userIn.casefold() == "encode"): # casefold ignores case
        userMsg = input("Enter a message\n> ")
        userKey = input("Enter a key (Int)\n> ")
        encode(userKey, userMsg)
    elif(userIn.casefold() == "d" or userIn.casefold() == "decode"):
        userMsg = input("Enter an encrypted message\n> ")
        userKey = input("Enter a key (Int)\n> ")
        decode(userKey, userMsg)


from pygame import mixer as mix
import random
import pyfiglet

# ASCII Text Art
print(pyfiglet.figlet_format("Music Player", font = "bulbhead" ))
print(pyfiglet.figlet_format("Created By @FaLLenGuY", font = "digital" ))
print("-"*70,"\n")

## initialize the pygame mixer
mix.init()

##initialize song list
songs=[]

## user Song list
while True:
        print("Add Songs ... ( Enter path of the file !! )")
        songs.append(input()+".mp3")
        add=input("ADD more songs! Y/N ")
        if add=='y':
                pass
        elif add=='n':
                break
        else:
                print("invalid Choice ")

## Sbuffle function to Add shuffle feature in Music Player
def shuffle():
        choice = input ("Do you want to shuffle the list? y/n ")
        if choice=='y':
                return random.shuffle(songs)            ##random shuffle of songs
        elif choice =='n':
                pass

## A Fucntion to start playing playlist
def play():
        shuffle()                                       ##shuffle call

        print("Playing",songs)
        index = 0
        while True:
                if index<len(songs):
                        mix.music.load(songs[index])
                        mix.music.set_volume(1)
                        mix.music.play()
                        index += 1
                else:
                        print("End of the Playlist")
                        break

                while True:
                        query = input("  Press 'P' to pause and Press 'R' to resume \n  Press 'N' for next song \n  Press 'E' exit player \n>>> ")

                        if (query == 'p'):
                                mix.music.pause()
                        elif (query =='r'):
                                mix.music.unpause()
                        elif (query =='n'):
                                mix.music.stop()
                                break
                        if (query == 'e'):
                                quit()

play()

Pygame/Music Player/Music Player 1.2 .py


dados = [
    [],  # Nome [0]
    [],  # Notas [1]
    []   # Média [2]
]
v = 0  # Vetor
i = 0  # Índice
e = 0  # Elemento
while True:
    dados[0].append(str(input('Nome: ')))  # Adiciona nome na lista de nomes [0]
    dados[1].append([])  # Cria uma nova lista em [1] para receber duas notas
    dados[1][v].append(float(input('1ª nota: ')))  # Acessa o vetor interno da lista 'notas' e adiciona as duas notas
    dados[1][v].append(float(input('2ª nota: ')))
    v += 1  # Representa o vetor dentro de notas [1]
    r = ' '
    while r not in 'sn':
        r = str(input('Deseja continuar? [S/N] ')).strip().lower()[0]
    if r == 'n':
        break

for v in dados[1]:  # v representa vetores internos de dados[1], onde cada um contém duas notas
    dados[2].append(sum(v) / 2)  # Soma notas e adiciona no vetor média [2]

print('-=' * 30)
print(f'{"Nº":<5} {"Nome":<7} {"Média"}')
print('-' * 30)
for i, e in enumerate(dados[0]):  # 'i' representa índice, 'e' representa elemento
    print(f'{i:<5} {e:<7} {dados[2][i]:.1f}')
print('-' * 30)

while True:  # Usário escolhe aluno para ver notas
    r = int(input('Você deseja ver a nota de qual aluno? (-1 para encerrar): '))
    if r == -1:
        break
    print('-' * 35)
    print(f'As notas do(a) {dados[0][r]} são {dados[1][r]}.')
    print('-' * 35)