import copy
import heapq
import metrics
import multiprocessing.pool as mpool
import os
import random
import shutil
import time
import math

width = 200
height = 16

options = [
    "-",  # an empty space
    "X",  # a solid wall
    "?",  # a question mark block with a coin
    "M",  # a question mark block with a mushroom
    "B",  # a breakable block
    "o",  # a coin
    "|",  # a pipe segment
    "T",  # a pipe top
    "E",  # an enemy
    #"f",  # a flag, do not generate
    #"v",  # a flagpole, do not generate
    #"m"  # mario's start position, do not generate
]

# The level as a grid of tiles

class Individual_Grid(object):
    __slots__ = ["genome", "_fitness"]

    def __init__(self, genome):
        self.genome = copy.deepcopy(genome)
        self._fitness = None

    # Update this individual's estimate of its fitness.
    # This can be expensive so we do it once and then cache the result.
    def calculate_fitness(self):
        measurements = metrics.metrics(self.to_level())
        # Print out the possible measurements or look at the implementation of metrics.py for other keys:
        # print(measurements.keys())
        # Default fitness function: Just some arbitrary combination of a few criteria.  Is it good?  Who knows?
        # STUDENT Modify this, and possibly add more metrics.  You can replace this with whatever code you like.
        coefficients = dict(
            meaningfulJumpVariance=0.6,
            negativeSpace=0.5,
            pathPercentage=0.5,
            emptyPercentage=0.6,
            linearity=-0.5,
            solvability=2.0
        )
        self._fitness = sum(map(lambda m: coefficients[m] * measurements[m],
                                coefficients))
        return self

    # Return the cached fitness value or calculate it as needed.
    def fitness(self):
        if self._fitness is None:
            self.calculate_fitness()
        return self._fitness

    # Mutate a genome into a new genome.  Note that this is a _genome_, not an individual!
    def mutate(self, genome):
        # STUDENT implement a mutation operator, also consider not mutating this individual
        # STUDENT also consider weighting the different tile types so it's not uniformly random
        # STUDENT consider putting more constraints on this to prevent pipes in the air, etc
        new_genome = copy.deepcopy(genome)

        # mutate for each tile: pit, pipe, blocks (coin, breakable, Mushroom), enemies, coins
        left = 10
        right = width - 10

        ground = list(range(left, right)) # don't modify the first and last ten tiles

        for i in range(left, right): # refill pits
            new_genome[height - 1][i] = "X"

        counter = 0 # counter keeps track of position for ground index

        # ---- PITS ----------------------------------------------------------------------------------

        # THIS LOOP GENERATES PITS
        while counter < len(ground):
            temp = ground[counter]
            if random.random() < temp * .1: # chance to create pit is based on progress
                tileNum = ground[counter]
                pitDistance = random.randrange(2, 6) # Pits are anywhere from 2 to 5 tiles long
                if tileNum + pitDistance > ground[-1]: # check if length is longer than ground
                    pitDistance = counter - ground[-1]
                for tiles in range(pitDistance): # Create a pit and remove it from ground
                    temp = ground.pop(counter)
                    new_genome[height - 1][temp] = "-"
                counter += 5 # at least five spaces to next pit
            else:
                counter += 1

        counter = 0

        # ---- WALLS ----------------------------------------------------------------------------------

        """
        # THIS LOOP GENERATES WALLS
        while counter < len(ground):
            if random.random() < .25:
                tileNum = ground[counter]
                wallDistance = random.randrange(1, 5)
                for currentDistance in range(wallDistance):
                    if new_genome[height - 1][tileNum + currentDistance] != "-": # Check if there's not a pit
                        wallHeight = random.randrange(2, 5)
                        for currentHeight in range(wallHeight):
                            new_genome[height - (1 + currentHeight)][temp] = "X"
                counter += 5 # at least five spaces to next wall
            else:
                counter += 1
        """

        # ---- PIPES ----------------------------------------------------------------------------------

        # THIS LOOP GENERATES PIPES IN THE TOP-HALF AND ERASES TOPLESS PIPES
        x = left
        while x < right:

            previous_pipe_not_found = True

            # Check that there isn't already a pipe
            for y in range(4, 1, -1):
                if new_genome[y][x - 1] == "T":
                    previous_pipe_not_found = False
                    break

            # This loop erases topless pipes
            for y in range(4, -1, -1):
                if new_genome[y][x] == "T":
                    if previous_pipe_not_found:
                        break
                    else:
                        new_genome[y][x] = "-"
                elif new_genome[y][x] == "|":
                    new_genome[y][x] = "-"

            if previous_pipe_not_found:

                # Check for pipe head in top-half
                pipe_head_not_found = True
                for y in range(4, 1, -1): # precedence given to taller pipes
                    if new_genome[y][x] == "T": # If pipe head found
                        pipe_head_not_found = False
                        for y2 in range(0, y + 1): # Remove all tiles next to it
                            new_genome[y2][x + 1] = "-"
                        for y2 in range(0, y): # generate pipe body
                            new_genome[y2][x] = "|"
                        break

                # If pipe head not found in top-half
                if pipe_head_not_found:
                    if random.random() < .05: # 5% chance to generate pipe head
                        chosenHeight = random.randrange(2, 5)
                        new_genome[chosenHeight][x] = "T"
                        for y2 in range(0, chosenHeight + 1): # remove tiles near pipe
                            new_genome[y2][x + 1] = "-"
                        for y2 in range(0, chosenHeight):
                            new_genome[y2][x] = "|"

            x += 1

        # THIS LOOP GENERATES PIPES IN THE BOTTOM-HALF AND ERASES TOPLESS PIPES
        x = left
        while x < right:

            previous_pipe_not_found = True

            # Check that there isn't already a pipe
            for y in range(height - 5, height - 2):
                if new_genome[y][x - 1] == "T":
                    previous_pipe_not_found = False
                    break

            # This loop erases topless pipes
            for y in range(height - 5, height - 1):
                if new_genome[y][x] == "T":
                    if previous_pipe_not_found:
                        break
                    else:
                        new_genome[y][x] = "-"
                elif new_genome[y][x] == "|":
                    new_genome[y][x] = "-"

            if previous_pipe_not_found:

                # Check for pipe head in bottom-half
                pipe_head_not_found = True
                for y in range(height - 5, height - 2):
                    if new_genome[y][x] == "T":
                        pipe_head_not_found = False
                        x += 1
                        for y2 in range(y, height): # Remove all tiles next to it
                            new_genome[y2][x] = "-"
                        for y2 in range(y + 1, height):
                            new_genome[y2][x - 1] = "|"
                        break

                if pipe_head_not_found:
                    if random.random() < .05: # 5% chance to generate pipe head
                        chosenHeight = random.randrange(height - 5, height - 1)
                        new_genome[chosenHeight][x] = "T"
                        x += 1
                        for y2 in range(chosenHeight, height): # remove tiles near pipe
                            new_genome[y2][x] = "-"
                        for y2 in range(chosenHeight + 1, height):
                            new_genome[y2][x - 1] = "|"

            x += 1

        # ---- BLOCKS --------------------------------------------------------------------------------

        x = left
        previous = -35
        mushroomTimer = 3

        while x < len(ground):

            if ground[x] - previous > 30 and random.random() > .5: # at least 30 blocks of space between blocks
                generator = []
                previous = ground[x]
                blockAmount = random.randrange(2, 5)

                for blockNum in range(blockAmount): # for each block, overwrite if the space is empty or has a coin

                    hit = 0
                    for heightAdjustment in range(height - 5, height):
                        hit = heightAdjustment - (height - 5) # check if low enough to make second layer
                        if new_genome[heightAdjustment][ground[x] + blockNum] == "X" or new_genome[heightAdjustment - 5][ground[x] + blockNum] == "T":
                            if new_genome[heightAdjustment - 4][ground[x] + blockNum] == "-" or new_genome[heightAdjustment - 4][ground[x] + blockNum] == "o":
                                generator.append((heightAdjustment - 4, ground[x] + blockNum))

                if random.random() > .25 and hit > 4: # 75% chance to generate a second layer of blocks

                    for blockNum in range(blockAmount): # repeat for second layer
                        generator.append((height - (5 + hit), ground[x] + blockNum))

                for item in generator:
                    if new_genome[item[0]][item[1]] != "-": # if there's already a block there
                        new_genome[item[0]][item[1]] = "-"
                    else:
                        if mushroomTimer >= 3: # generate Mushroom if timer is high enough
                            mushroomTimer = 1
                            new_genome[item[0]][item[1]] = "M"
                        elif random.random() > .5: # 50% chance to generate coin block
                            new_genome[item[0]][item[1]] = "?"
                        else: # 50% chance to generate break block
                            new_genome[item[0]][item[1]] = "B"

                mushroomTimer += 1
                x += 3 # don't generate blocks for at least another 3 tiles

            x += 1

        # ---- ENEMIES -------------------------------------------------------------------------------

        x = left
        while x < len(ground):
            if random.random() < (ground[x] * .005):
                for y in range(5, height):
                    if new_genome[y][ground[x]] == "E":
                        if random.random() > .5:
                            new_genome[y-1][ground[x]] = "-"
                    if new_genome[y][ground[x]] == "X" or new_genome[y][ground[x]] == "T" or new_genome[y][ground[x]] == "M" \
                    or new_genome[y][ground[x]] == "?" or new_genome[y][ground[x]] == "B":
                        new_genome[y - 1][ground[x]] = "E" # place enemy above
                x += 3 # don't generate enemies for at least another 3 tiles
            x += 1

        # ---- COINS ---------------------------------------------------------------------------------

        x = left
        while x < right:
            for y in range (5, height - 2):
                if new_genome[y][x] == "-":
                    if random.random() >= .97: # 3% chance to generate coins
                        new_genome[y][x] = "o"
                elif new_genome[y][x] == "o":
                    if random.random() >= .97: # 3% chance to remove instead
                        new_genome[y][x] = "-"
            x += 1

        return new_genome

    # Create zero or more children from self and other
    def generate_children(self, other):
        new_genome = copy.deepcopy(self.genome)
        # Leaving first and last columns alone...
        # do crossover with other
        left = 1          # leftmost column of the level
        right = width - 1 # rightmost column of the level
        dadChance = 0
        dadChance = .58 if self._fitness > other._fitness else .42
        for y in range(height): # for each tile in height
            for x in range(left, right): # for each tile from (left, right)
                # STUDENT Which one should you take?  Self, or other?  Why?
                # STUDENT consider putting more constraints on this to prevent pipes in the air, etc
                # compare fitness of both genomes
                new_genome[y][x] = self.genome[y][x] if random.random() < dadChance else other.genome[y][x]

        # do mutation; note we're returning a one-element tuple here
        new_genome = self.mutate(new_genome)
        return (Individual_Grid(new_genome), )

    # Turn the genome into a level string (easy for this genome)
    def to_level(self):
        return self.genome

    # These both start with every floor tile filled with Xs
    # STUDENT Feel free to change these
    @classmethod
    def empty_individual(cls):
        g = [["-" for col in range(width)] for row in range(height)]
        g[15][:] = ["X"] * width
        g[14][0] = "m"
        g[7][-1] = "v"
        for col in range(8, 14):
            g[col][-1] = "f"
        for col in range(14, 16):
            g[col][-1] = "X"
        return cls(g)

    @classmethod
    def random_individual(cls):
        # STUDENT consider putting more constraints on this to prevent pipes in the air, etc
        # STUDENT also consider weighting the different tile types so it's not uniformly random
        g = [random.choices(options, k=width) for row in range(height)]
        g[15][:] = ["X"] * width
        g[14][0] = "m"
        g[7][-1] = "v"
        g[8:14][-1] = ["f"] * 6
        g[14:16][-1] = ["X", "X"]
        return cls(g)

def offset_by_upto(val, variance, min=None, max=None):
    val += random.normalvariate(0, variance**0.5)
    if min is not None and val < min:
        val = min
    if max is not None and val > max:
        val = max
    return int(val)

def clip(lo, val, hi):
    if val < lo:
        return lo
    if val > hi:
        return hi
    return val

# Inspired by https://www.researchgate.net/profile/Philippe_Pasquier/publication/220867545_Towards_a_Generic_Framework_for_Automated_Video_Game_Level_Creation/links/0912f510ac2bed57d1000000.pdf

class Individual_DE(object):
    # Calculating the level isn't cheap either so we cache it too.
    __slots__ = ["genome", "_fitness", "_level"]

    # Genome is a heapq of design elements sorted by X, then type, then other parameters
    def __init__(self, genome):
        self.genome = list(genome)
        heapq.heapify(self.genome)
        self._fitness = None
        self._level = None

    # Calculate and cache fitness
    def calculate_fitness(self):
        measurements = metrics.metrics(self.to_level())
        # Default fitness function: Just some arbitrary combination of a few criteria.  Is it good?  Who knows?
        # STUDENT Add more metrics?
        # STUDENT Improve this with any code you like
        coefficients = dict(
            meaningfulJumpVariance=0.5,
            negativeSpace=0.6,
            pathPercentage=0.5,
            emptyPercentage=0.6,
            linearity=-0.5,
            solvability=2.0
        )
        penalties = 0
        # STUDENT For example, too many stairs are unaesthetic.  Let's penalize that
        if len(list(filter(lambda de: de[1] == "6_stairs", self.genome))) > 5:
            penalties -= 2
        # STUDENT If you go for the FI-2POP extra credit, you can put constraint calculation in here too and cache it in a new entry in __slots__.
        self._fitness = sum(map(lambda m: coefficients[m] * measurements[m],
                                coefficients)) + penalties
        return self

    def fitness(self):
        if self._fitness is None:
            self.calculate_fitness()
        return self._fitness

    def mutate(self, new_genome):
        # STUDENT How does this work?  Explain it in your writeup.
        # STUDENT consider putting more constraints on this, to prevent generating weird things
        if random.random() < 0.1 and len(new_genome) > 0:
            to_change = random.randint(0, len(new_genome) - 1)
            de = new_genome[to_change]
            new_de = de
            x = de[0]
            de_type = de[1]
            choice = random.random()
            if de_type == "4_block":
                y = de[2]
                breakable = de[3]
                if choice < 0.33:
                    x = offset_by_upto(x, width / 8, min=1, max=width - 2)
                elif choice < 0.66:
                    y = offset_by_upto(y, height / 2, min=0, max=height - 1)
                else:
                    breakable = not de[3]
                new_de = (x, de_type, y, breakable)
            elif de_type == "5_qblock":
                y = de[2]
                has_powerup = de[3]  # boolean
                if choice < 0.33:
                    x = offset_by_upto(x, width / 8, min=1, max=width - 2)
                elif choice < 0.66:
                    y = offset_by_upto(y, height / 2, min=0, max=height - 1)
                else:
                    has_powerup = not de[3]
                new_de = (x, de_type, y, has_powerup)
            elif de_type == "3_coin":
                y = de[2]
                if choice < 0.5:
                    x = offset_by_upto(x, width / 8, min=1, max=width - 2)
                else:
                    y = offset_by_upto(y, height / 2, min=0, max=height - 1)
                new_de = (x, de_type, y)
            elif de_type == "7_pipe":
                h = de[2]
                if choice < 0.5:
                    x = offset_by_upto(x, width / 8, min=1, max=width - 2)
                else:
                    h = offset_by_upto(h, 2, min=2, max=height - 4)
                new_de = (x, de_type, h)
            elif de_type == "0_hole":
                w = de[2]
                if choice < 0.5:
                    x = offset_by_upto(x, width / 8, min=1, max=width - 2)
                else:
                    w = offset_by_upto(w, 4, min=1, max=width - 2)
                new_de = (x, de_type, w)
            elif de_type == "6_stairs":
                h = de[2]
                dx = de[3]  # -1 or 1
                if choice < 0.33:
                    x = offset_by_upto(x, width / 8, min=1, max=width - 2)
                elif choice < 0.66:
                    h = offset_by_upto(h, 8, min=1, max=height - 4)
                else:
                    dx = -dx
                new_de = (x, de_type, h, dx)
            elif de_type == "1_platform":
                w = de[2]
                y = de[3]
                madeof = de[4]  # from "?", "X", "B"
                if choice < 0.25:
                    x = offset_by_upto(x, width / 8, min=1, max=width - 2)
                elif choice < 0.5:
                    w = offset_by_upto(w, 8, min=1, max=width - 2)
                elif choice < 0.75:
                    y = offset_by_upto(y, height, min=0, max=height - 1)
                else:
                    madeof = random.choice(["?", "X", "B"])
                new_de = (x, de_type, w, y, madeof)
            elif de_type == "2_enemy":
                pass
            new_genome.pop(to_change)
            heapq.heappush(new_genome, new_de)
        return new_genome

    def generate_children(self, other):
        # STUDENT How does this work?  Explain it in your writeup.
        pa = random.randint(0, len(self.genome) - 1) if len(self.genome) > 0 else 0
        pb = random.randint(0, len(other.genome) - 1) if len(other.genome) > 0 else 0
        a_part = self.genome[:pa] if len(self.genome) > 0 else []
        b_part = other.genome[pb:] if len(other.genome) > 0 else []
        ga = a_part + b_part
        b_part = other.genome[:pb] if len(other.genome) > 0 else []
        a_part = self.genome[pa:] if len(self.genome) > 0 else []
        gb = b_part + a_part
        # do mutation
        return Individual_DE(self.mutate(ga)), Individual_DE(self.mutate(gb))

    # Apply the DEs to a base level.
    def to_level(self):
        if self._level is None:
            base = Individual_Grid.empty_individual().to_level()
            for de in sorted(self.genome, key=lambda de: (de[1], de[0], de)):
                # de: x, type, ...
                x = de[0]
                de_type = de[1]
                if de_type == "4_block":
                    y = de[2]
                    breakable = de[3]
                    base[y][x] = "B" if breakable else "X"
                elif de_type == "5_qblock":
                    y = de[2]
                    has_powerup = de[3]  # boolean
                    base[y][x] = "M" if has_powerup else "?"
                elif de_type == "3_coin":
                    y = de[2]
                    base[y][x] = "o"
                elif de_type == "7_pipe":
                    h = de[2]
                    base[height - h - 1][x] = "T"
                    for y in range(height - h, height):
                        base[y][x] = "|"
                elif de_type == "0_hole":
                    w = de[2]
                    for x2 in range(w):
                        base[height - 1][clip(1, x + x2, width - 2)] = "-"
                elif de_type == "6_stairs":
                    h = de[2]
                    dx = de[3]  # -1 or 1
                    for x2 in range(1, h + 1):
                        for y in range(x2 if dx == 1 else h - x2):
                            base[clip(0, height - y - 1, height - 1)][clip(1, x + x2, width - 2)] = "X"
                elif de_type == "1_platform":
                    w = de[2]
                    h = de[3]
                    madeof = de[4]  # from "?", "X", "B"
                    for x2 in range(w):
                        base[clip(0, height - h - 1, height - 1)][clip(1, x + x2, width - 2)] = madeof
                elif de_type == "2_enemy":
                    base[height - 2][x] = "E"
            self._level = base
        return self._level

    @classmethod
    def empty_individual(_cls):
        # STUDENT Maybe enhance this
        g = []
        return Individual_DE(g)

    @classmethod
    def random_individual(_cls):
        # STUDENT Maybe enhance this
        elt_count = random.randint(8, 128)
        g = [random.choice([
            (random.randint(1, width - 2), "0_hole", random.randint(1, 8)),
            (random.randint(1, width - 2), "1_platform", random.randint(1, 8), random.randint(0, height - 1), random.choice(["?", "X", "B"])),
            (random.randint(1, width - 2), "2_enemy"),
            (random.randint(1, width - 2), "3_coin", random.randint(0, height - 1)),
            (random.randint(1, width - 2), "4_block", random.randint(0, height - 1), random.choice([True, False])),
            (random.randint(1, width - 2), "5_qblock", random.randint(0, height - 1), random.choice([True, False])),
            (random.randint(1, width - 2), "6_stairs", random.randint(1, height - 4), random.choice([-1, 1])),
            (random.randint(1, width - 2), "7_pipe", random.randint(2, height - 4))
        ]) for i in range(elt_count)]
        return Individual_DE(g)

Individual = Individual_Grid

def generate_successors(population):
    # STUDENT Design and implement this
    # Hint: Call generate_children() on some individuals and fill up results.
    # 5% Elitist, 10% Random - 25% Elite (.025), 75% Roulette (.075)
    results = []            # returned list for genomes selected to carry over to next population
    elitePopulation = []    # list of genomes selected through elitist select; 5% of pop_limit
    roulettePopulation = [] # list of genomes selected through roulette wheel select; 10% of pop_limit, has half of elitePopulation
    pop_limit = len(population)
    sortedPopulation = sorted(population, key=lambda level: level._fitness * -1) # sort on fitness (most to least)

    for num in range(int(.05 * pop_limit)): # 5%, (weirdly) rounded to whole number (see doc)
        temp = sortedPopulation.pop(num)
        elitePopulation.append(temp) # append to elitePopulation while removing from sortedPopulation
        results.append(temp)

    elitePick = random.randrange(0, pop_limit - 1) # pick a random number from 0 to length of elitePopulation

    # if elitePick is even, append odd and vice-versa
    if elitePick % 2 != 0:
        for item in range(0, len(elitePopulation), 2):
            roulettePopulation.append(elitePopulation[item])
    else:
        for item in range(1, len(elitePopulation), 2):
            roulettePopulation.append(elitePopulation[item])

    # select via roulette to fill roulettePopulation
    fitnessTotal = 0 # get total fitness of population
    for item in sortedPopulation:
        fitnessTotal += abs(item._fitness)

    for num in range(int(.075 * len(sortedPopulation))): # get 7.5% random
        rouletteSelect = random.randrange(0, int(fitnessTotal)) # rouletteSelect is a number between 0 and total fitness
        for item in range(len(sortedPopulation)):
            rouletteSelect -= abs(sortedPopulation[item]._fitness) # keep subtracting from the random roulette select number until it hits 0
            if (rouletteSelect <= 0): # if selected
                fitnessTotal -= sortedPopulation[item]._fitness # remove item from pool and append it
                roulettePopulation.append(sortedPopulation.pop(item))
                break
        if rouletteSelect > 0: # in case the selected number is more than the fitness, append last
            roulettePopulation.append(sortedPopulation.pop())

    for i in range(pop_limit - len(results)):
        child = random.choice(elitePopulation).generate_children(random.choice(roulettePopulation))
        results.append(child[0])

    # print("TESTING: ", len(results), results)
    return results

def ga():
    # STUDENT Feel free to play with this parameter
    pop_limit = 64
    # Code to parallelize some computations
    batches = os.cpu_count()
    if pop_limit % batches != 0:
        print("It's ideal if pop_limit divides evenly into " + str(batches) + " batches.")
    batch_size = int(math.ceil(pop_limit / batches))
    with mpool.Pool(processes=os.cpu_count()) as pool:
        init_time = time.time()
        # STUDENT (Optional) change population initialization
        """ Before
        population = [Individual.random_individual() if random.random() < 0.9
                      else Individual.empty_individual()
                      for _g in range(pop_limit)]
        """
        population = [Individual.empty_individual() for _g in range(pop_limit)]

        # But leave this line alone; we have to reassign to population because we get a new population that has more cached stuff in it.
        population = pool.map(Individual.calculate_fitness,
                              population,
                              batch_size)
        init_done = time.time()
        print("Created and calculated initial population statistics in:", init_done - init_time, "seconds")
        generation = 0
        start = time.time()
        now = start
        print("Use ctrl-c to terminate this loop manually.")
        try:
            runs = 0
            # while runs == 0:
            while True:
                now = time.time()
                # Print out statistics
                if generation > 0:
                    best = max(population, key=Individual.fitness)
                    print("Generation:", str(generation))
                    print("Max fitness:", str(best.fitness()))
                    print("Average generation time:", (now - start) / generation)
                    print("Net time:", now - start)
                    with open("levels/last.txt", 'w') as f:
                        for row in best.to_level():
                            f.write("".join(row) + "\n")
                generation += 1
                # STUDENT Determine stopping condition
                stop_condition = False
                if stop_condition:
                    break
                # STUDENT Also consider using FI-2POP as in the Sorenson & Pasquier paper
                gentime = time.time()
                next_population = generate_successors(population)
                gendone = time.time()
                print("Generated successors in:", gendone - gentime, "seconds")
                # Calculate fitness in batches in parallel
                next_population = pool.map(Individual.calculate_fitness,
                                           next_population,
                                           batch_size)
                popdone = time.time()
                print("Calculated fitnesses in:", popdone - gendone, "seconds")
                population = next_population
                # runs += 1
        except KeyboardInterrupt:
            pass
    return population

if __name__ == "__main__":
    final_gen = sorted(ga(), key=Individual.fitness, reverse=True)
    best = final_gen[0]
    print("Best fitness: " + str(best.fitness()))
    now = time.strftime("%m_%d_%H_%M_%S")
    # STUDENT You can change this if you want to blast out the whole generation, or ten random samples, or...
    for k in range(0, 10):
        with open("levels/" + now + "_" + str(k) + ".txt", 'w') as f:
            for row in final_gen[k].to_level():
                f.write("".join(row) + "\n")


import os
import urllib.request
import pandas as pd
from delphi.epidata.acquisition.mobility.database import database

url = "https://www.gstatic.com/covid19/mobility/Global_Mobility_Report.csv"
directory = 'Google_Mobility'

def get_mobility_data():
    """Download Google Mobility data in CSV format
    """
    # create directory if it doesn't exist
    if not os.path.exists(directory) and directory != '':
        os.makedirs(directory)
    else:
        files = os.listdir(directory)
        for file in files:
            os.remove(directory + '/' + file)

    urllib.request.urlretrieve(url, os.path.join(directory, "google_mobility_data_US.csv"))

def build_report():
    """Build cleaned Google report
    """
    mobilityData = pd.read_csv(os.path.join(directory, "google_mobility_data_US.csv"), low_memory=False)
    mobilityData.columns = mobilityData.columns.str.replace(
        r'_percent_change_from_baseline', '')
    mobilityData.columns = mobilityData.columns.str.replace(r'_', ' ')
    mobilityData = mobilityData.rename(columns={'country region': 'country'})
    mobilityData = mobilityData[(mobilityData['country'] == "United States")]
    mobilityData = mobilityData.rename(
        columns={
            'sub region 1': 'state',
            'sub region 2': 'county',
            'census fips code': 'fips code'})
    mobilityData = mobilityData.loc[:,
                   ['state',
                    'county',
                    'fips code',
                    'date',
                    'retail and recreation',
                    'grocery and pharmacy',
                    'parks',
                    'transit stations',
                    'workplaces',
                    'residential']]

    mobilityData.dropna(subset=['county'], how='all', inplace=True)
    mobilityData.fillna(0, inplace=True)
    mobilityData['fips code'] = mobilityData['fips code'].astype(str).replace('\.0', '', regex=True)
    mobilityData['fips code'] = mobilityData['fips code'].str.rjust(5, '0')
    return mobilityData

def split_df(df, size):
    """Split df and insert into database.
    """
    db = database.Database()
    for i in range(0, df.shape[0], size):
        start = i
        end = min(i + size - 1, df.shape[0])
        subset = df.iloc[start:end]
        db.insert_google_data(subset)

if __name__ == '__main__':
    # get Google data
    get_mobility_data()
    # build reports
    mobilityData_US = build_report()
    # Insert into database
    split_df(mobilityData_US, 100000)


#
# Training Deep Neural Networks on Noisy Labels with Bootstrapping
# http://www-personal.umich.edu/~reedscot/bootstrap.pdf
#

import torch
from torch.nn import Module
import torch.nn.functional as F

class SoftBootstrappingLoss(Module):
    """
    ``Loss(t, p) = - (beta * t + (1 - beta) * p) * log(p)``

    Args:
        beta (float): bootstrap parameter. Default, 0.95
        reduce (bool): computes mean of the loss. Default, True.
        as_pseudo_label (bool): Stop gradient propagation for the term ``(1 - beta) * p``.
            Can be interpreted as pseudo-label.
    """
    def __init__(self, beta=0.95, reduce=True, as_pseudo_label=True):
        super(SoftBootstrappingLoss, self).__init__()
        self.beta = beta
        self.reduce = reduce
        self.as_pseudo_label = as_pseudo_label

    def forward(self, y_pred, y):
        # cross_entropy = - t * log(p)
        beta_xentropy = self.beta * F.cross_entropy(y_pred, y, reduction='none')

        y_pred_a = y_pred.detach() if self.as_pseudo_label else y_pred
        # second term = - (1 - beta) * p * log(p)
        bootstrap = - (1.0 - self.beta) * torch.sum(F.softmax(y_pred_a, dim=1) * F.log_softmax(y_pred, dim=1), dim=1)

        if self.reduce:
            return torch.mean(beta_xentropy + bootstrap)
        return beta_xentropy + bootstrap

class HardBootstrappingLoss(Module):
    """
    ``Loss(t, p) = - (beta * t + (1 - beta) * z) * log(p)``
    where ``z = argmax(p)``

    Args:
        beta (float): bootstrap parameter. Default, 0.95
        reduce (bool): computes mean of the loss. Default, True.

    """
    def __init__(self, beta=0.8, reduce=True):
        super(HardBootstrappingLoss, self).__init__()
        self.beta = beta
        self.reduce = reduce

    def forward(self, y_pred, y):
        # cross_entropy = - t * log(p)
        beta_xentropy = self.beta * F.cross_entropy(y_pred, y, reduction='none')

        # z = argmax(p)
        z = F.softmax(y_pred.detach(), dim=1).argmax(dim=1)
        z = z.view(-1, 1)
        bootstrap = F.log_softmax(y_pred, dim=1).gather(1, z).view(-1)
        # second term = (1 - beta) * z * log(p)
        bootstrap = - (1.0 - self.beta) * bootstrap

        if self.reduce:
            return torch.mean(beta_xentropy + bootstrap)
        return beta_xentropy + bootstrap


"""
auto registration the python module that under the rest folder python files
"""
import importlib
import os

from component import flask_blueprint

rest_dir_name = 'rest'
ignore_dir_list = ['__pycache__']
file_tag = '.py'

def confirm_file_tag():
    if not os.path.exists('setup.py'):
        flask_blueprint.file_tag = '.pyc'

def auto_registration_flask_blueprint(blueprint_path, module_name):
    from __init__ import app
    for item in os.listdir(blueprint_path):
        if item in ignore_dir_list:
            continue
        current_level_path = os.path.join(blueprint_path, item)
        cur_module_name = module_name + '.' + item
        if os.path.isdir(current_level_path):
            auto_registration_flask_blueprint(current_level_path, cur_module_name)
            continue
        if not cur_module_name.endswith(flask_blueprint.file_tag):
            continue
        cur_module_name = cur_module_name[:(0 - len(flask_blueprint.file_tag))]
        target_dynamic_module = importlib.import_module(cur_module_name)
        if hasattr(target_dynamic_module, 'app'):
            app.register_blueprint(target_dynamic_module.app)

def do_init():
    confirm_file_tag()
    auto_registration_flask_blueprint(rest_dir_name, rest_dir_name)


import re
import time

# from selenium import webdriver
from selenium import webdriver
from bs4 import BeautifulSoup
from lmf.dbv2 import db_write
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.common.exceptions import NoSuchElementException,StaleElementReferenceException
from selenium.common.exceptions import WebDriverException
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

#初始化
driver=webdriver.Chrome()
# driver.maximize_window()

# driver.set_page_load_timeout(3)
# try:
driver.get('http://www.ccggzy.gov.cn/qxxxgk/003002/003002001/CountyPurhcaseNotice.html')
# except:
    # driver.execute_script('javascript:window.stop()')

#第一个等待
locator=(By.XPATH,'//tbody[@id="showList"]/tr[1]/td[2]/a')
WebDriverWait(driver,10).until(EC.presence_of_element_located(locator))
url=driver.current_url
PAGE=[]
CC_TEXT=[]
#
# for i in range(1,17):
#     if i != 1:
#         val = driver.find_element_by_xpath('//tbody[@id="showList"]/tr[1]/td[2]/a').text
#         driver.find_element_by_xpath('(//div[@class="ewb-screen-list"])[1]/span[{}]'.format(i)).click()
#         #
#         locator = (By.XPATH, '//tbody[@id="showList"]/tr[1]/td[2]/a[not(contains(string(),"%s"))]'%val)
#         WebDriverWait(driver, 10).until(EC.presence_of_element_located(locator))
#         locator = (By.XPATH, '//span[@class="pg_maxpagenum"]')
#         WebDriverWait(driver, 10).until(EC.presence_of_element_located(locator))
#         time.sleep(0.5)
#     page = driver.find_element_by_xpath('//span[@class="pg_maxpagenum"]').text.strip()
#     cc_text=driver.find_element_by_xpath('//span[@class="ewb-screen-name current"]').text
#
#     print(page)
#     total_ = int(re.findall('/(\d+)', page)[0])
#     PAGE.append(total_)
#     CC_TEXT.append(cc_text)
# print(PAGE)
# print(CC_TEXT)
# print(sum(PAGE))
#
#
# #获取总页数
# page=driver.find_element_by_xpath('//span[@class="pg_maxpagenum"]').text
# print(page)
# total=re.findall('/(\d+)',page)[0]
# total=int(total)
# print(total)

data=[]

html = driver.page_source
soup = BeautifulSoup(html, 'lxml')
div=soup.find('tbody',id='showList')
trs=div.find_all('tr')

for tr in trs:

    tds=tr.find_all('td')
    href=tds[1].a['href']
    name=tds[1].a.get_text().strip()

    if name.startswith('【'):
        gg_type=re.findall(r'^【(.+?)】',name)[0]
        name=name.split('】',maxsplit=1)[1]
    else:
        gg_type=None

    ggstart_time=tds[2].get_text()

    if 'http' in href:
        href=href
    else:
        href='http://www.ccggzy.gov.cn'+href

    tmp = [gg_type,name, ggstart_time, href]

    print(tmp)

driver.quit()


# -*- coding: utf-8 -*-

import os
import shutil
import sys
import unittest
from tempfile import TemporaryDirectory

import pytest

from mltk import *
from mltk.utils import json_loads
from tests.helpers import set_environ_context, get_file_content, zip_snapshot

@config_params(undefined_fields=True)
class _YourConfig(Config):
    max_epoch = 100
    learning_rate = 0.5

    class train(Config):
        batch_size = 64

class ExperimentTestCase(unittest.TestCase):

    def test_construct(self):
        # test auto select script name and output dir
        exp = Experiment(_YourConfig)
        script_name = os.path.splitext(
            os.path.basename(sys.modules['__main__'].__file__))[0]
        self.assertEqual(exp.script_name, script_name)
        self.assertEqual(
            exp.output_dir, os.path.abspath(f'./results/{script_name}'))
        self.assertIsNone(exp.id)
        self.assertIsNone(exp.client)

        # test select output dir according to mlrunner env
        with TemporaryDirectory() as temp_dir:
            with set_environ_context(MLSTORAGE_OUTPUT_DIR=temp_dir):
                exp = Experiment(_YourConfig)
                self.assertEqual(exp.output_dir, temp_dir)

        # test specifying script_name
        exp = Experiment(_YourConfig, script_name='abc')
        self.assertEqual(exp.script_name, 'abc')
        self.assertEqual(exp.output_dir, os.path.abspath(f'./results/abc'))

        # test specifying the output dir
        with TemporaryDirectory() as temp_dir:
            exp = Experiment(_YourConfig, output_dir=temp_dir)
            self.assertEqual(exp.output_dir, temp_dir)

        # test config
        self.assertIsInstance(Experiment(_YourConfig).config, _YourConfig)
        config = _YourConfig()
        self.assertIs(Experiment(config).config, config)

        with pytest.raises(TypeError,
                           match='`config_or_cls` is neither a Config class, '
                                 'nor a Config instance: <class \'object\'>'):
            _ = Experiment(object)

        with pytest.raises(TypeError,
                           match='`config_or_cls` is neither a Config class, '
                                 'nor a Config instance: <object .*>'):
            _ = Experiment(object())

        # test result
        self.assertDictEqual(Experiment(_YourConfig).results, {})

        # test args
        self.assertTupleEqual(Experiment(_YourConfig).args, tuple(sys.argv[1:]))
        self.assertIsNone(Experiment(_YourConfig, args=None).args)

        args = ('--output-dir=abc', '--max_epoch=123')
        self.assertTupleEqual(Experiment(_YourConfig, args=args).args, args)

        # test specifying the environment variable of MLStorage server
        with set_environ_context(MLSTORAGE_SERVER_URI='http://localhost:8080',
                                 MLSTORAGE_EXPERIMENT_ID='your_id'):
            e = Experiment(_YourConfig)
            self.assertEqual(e.id, 'your_id')
            self.assertIsInstance(e.client, MLStorageClient)
            self.assertEqual(e.client.uri, 'http://localhost:8080')

    def test_events(self):
        with TemporaryDirectory() as temp_dir:
            event_track = []

            exp = Experiment(_YourConfig, output_dir=temp_dir, args=None)
            exp.on_enter.do(lambda: event_track.append('on enter'))
            exp.on_exit.do(lambda: event_track.append('on exit'))

            event_track.append('before enter')
            with exp:
                event_track.append('after enter')
                event_track.append('before exit')
            event_track.append('after exit')

            self.assertListEqual(event_track, [
                'before enter', 'on enter', 'after enter',
                'before exit', 'on exit', 'after exit',
            ])

    def test_experiment(self):
        with TemporaryDirectory() as temp_dir:
            output_dir = os.path.join(temp_dir, 'output')

            # test new output dir
            with Experiment(_YourConfig, output_dir=output_dir,
                            args=('--max_epoch=200', '--train.batch_size=128')
                            ) as exp:
                self.assertEqual(exp.output_dir, output_dir)
                self.assertTrue(os.path.isdir(exp.output_dir))

                # the config should be correctly loaded from CLI argument,
                # and the values should be saved into "/config.json"
                self.assertEqual(exp.config, _YourConfig(
                    max_epoch=200, train=_YourConfig.train(batch_size=128)
                ))
                self.assertDictEqual(
                    json_loads(get_file_content(
                        os.path.join(exp.output_dir, 'config.json'))),
                    {'max_epoch': 200, 'learning_rate': 0.5,
                     'train.batch_size': 128}
                )
                self.assertDictEqual(
                    json_loads(get_file_content(
                        os.path.join(exp.output_dir, 'config.defaults.json'))),
                    {'max_epoch': 100, 'learning_rate': 0.5,
                     'train.batch_size': 64}
                )

                # there should have not been "/result.json"
                self.assertDictEqual(exp.results, {})
                self.assertFalse(os.path.exists(
                    os.path.join(output_dir, 'result.json')))

                # after we `update_results`, there should be "/result.json"
                exp.update_results({'loss': 1, 'acc': 2})
                self.assertDictEqual(exp.results, {'loss': 1, 'acc': 2})
                self.assertDictEqual(
                    json_loads(get_file_content(
                        os.path.join(exp.output_dir, 'result.json'))),
                    {'loss': 1, 'acc': 2}
                )

                # further modify `config` and `results`, and they should be
                # saved after exiting the context
                exp.config.max_epoch = 300
                exp.config.max_step = 10000
                exp.results['acc'] = 3
                exp.results['time'] = 4

                # test to take absolute path
                self.assertEqual(exp.abspath('a/b/c.txt'),
                                 os.path.join(output_dir, 'a/b/c.txt'))

                # test to make directory
                self.assertFalse(os.path.exists(os.path.join(output_dir, 'a')))
                self.assertEqual(exp.make_dirs('a/b'),
                                 os.path.join(output_dir, 'a/b'))
                self.assertTrue(os.path.isdir(os.path.join(output_dir, 'a/b')))
                exp.make_dirs('a/b')

                with pytest.raises(IOError, match='.* exists'):
                    exp.make_dirs('a/b', exist_ok=False)

                # test to make parents
                self.assertFalse(os.path.exists(os.path.join(output_dir, 'b')))
                self.assertEqual(exp.make_parent('b/c.txt'),
                                 os.path.join(output_dir, 'b/c.txt'))
                self.assertTrue(os.path.isdir(os.path.join(output_dir, 'b')))

                # test to open file
                with pytest.raises(IOError, match='No such file or directory'):
                    _ = exp.open_file('c/d.txt', 'wb', make_parent=False)

                with pytest.raises(IOError, match='No such file or directory'):
                    _ = exp.open_file('c/d.txt', 'rb')

                self.assertFalse(os.path.exists(os.path.join(output_dir, 'c')))

                with exp.open_file('c/d.txt', 'wb') as f:
                    f.write(b'hello, world!')
                self.assertEqual(
                    get_file_content(os.path.join(output_dir, 'c/d.txt')),
                    b'hello, world!'
                )

                with exp.open_file('c/d.txt', 'wb', encoding='gbk') as f:
                    f.write('你好，世界')
                self.assertEqual(
                    get_file_content(os.path.join(output_dir, 'c/d.txt')),
                    b'\xc4\xe3\xba\xc3\xa3\xac\xca\xc0\xbd\xe7'
                )

                with exp.open_file('c/d.txt', 'ab') as f:
                    f.write(b'\nhi, world!')

                with exp.open_file('c/d.txt', 'rb', 'gbk') as f:
                    cnt = f.read()
                self.assertIsInstance(cnt, str)
                self.assertEqual(cnt, '你好，世界\nhi, world!')

                # test to put & get file content
                exp.put_file_content(
                    'c/nested/e.txt', '你好，世界', encoding='gbk')
                exp.put_file_content(
                    'c/nested/e.txt', b'\nhello, world!', append=True)
                self.assertEqual(exp.get_file_content('c/nested/e.txt'),
                                 b'\xc4\xe3\xba\xc3\xa3\xac\xca\xc0\xbd\xe7'
                                 b'\nhello, world!')

                # test make_archive error
                with pytest.raises(IOError, match='Not a directory'):
                    _ = exp.make_archive('non-exist')

                # test make_archive without deleting source directory
                zip_file = os.path.join(output_dir, 'cc.zip')
                self.assertEqual(
                    exp.make_archive('c', zip_file, delete_source=False),
                    zip_file
                )
                self.assertTrue(os.path.isdir(os.path.join(output_dir, 'c')))
                self.assertDictEqual(zip_snapshot(zip_file), {
                    'd.txt': b'\xc4\xe3\xba\xc3\xa3\xac\xca\xc0\xbd\xe7'
                             b'\nhi, world!',
                    'nested': {
                        'e.txt': b'\xc4\xe3\xba\xc3\xa3\xac\xca\xc0\xbd\xe7'
                                 b'\nhello, world!'
                    }
                })

                # test make archive with deleting source directory,
                # and merging the original archive files
                old_zip_file = zip_file
                zip_file = os.path.join(output_dir, 'c.zip')
                shutil.move(old_zip_file, zip_file)

                os.remove(os.path.join(output_dir, 'c/nested/e.txt'))
                exp.put_file_content('c/d.txt', b'd.txt')
                exp.put_file_content('c/nested/f.txt', b'f.txt')

                self.assertEqual(exp.make_archive('c'), zip_file)
                self.assertFalse(os.path.exists(os.path.join(output_dir, 'c')))
                self.assertDictEqual(zip_snapshot(zip_file), {
                    'd.txt': b'd.txt',
                    'nested': {
                        'e.txt': b'\xc4\xe3\xba\xc3\xa3\xac\xca\xc0\xbd\xe7'
                                 b'\nhello, world!',
                        'f.txt': b'f.txt',
                    },
                })

                # test make archive on exit
                exp.put_file_content('c/nested/f.txt', b'overrided f.txt')
                exp.put_file_content('c/nested/g.txt', b'g.txt')
                exp.make_archive_on_exit('c')

            # check whether the config and results have been saved
            self.assertDictEqual(
                json_loads(get_file_content(
                    os.path.join(exp.output_dir, 'config.json'))),
                {'max_epoch': 300, 'max_step': 10000, 'learning_rate': 0.5,
                 'train.batch_size': 128}
            )
            self.assertDictEqual(
                json_loads(get_file_content(
                    os.path.join(exp.output_dir, 'result.json'))),
                {'loss': 1, 'acc': 3, 'time': 4}
            )

            # check whether the archive has been made on exit
            zip_file = os.path.join(output_dir, 'c.zip')
            self.assertFalse(os.path.exists(os.path.join(output_dir, 'c')))
            self.assertDictEqual(zip_snapshot(zip_file), {
                'd.txt': b'd.txt',
                'nested': {
                    'e.txt': b'\xc4\xe3\xba\xc3\xa3\xac\xca\xc0\xbd\xe7'
                             b'\nhello, world!',
                    'f.txt': b'overrided f.txt',
                    'g.txt': b'g.txt',
                },
            })

            # test restore from the previous output dir
            # (and also parse `output_dir` from CLI arguments)
            exp = Experiment(_YourConfig, args=['--output-dir=' + output_dir])
            self.assertNotEqual(exp.output_dir, output_dir)
            with exp:
                self.assertEqual(exp.output_dir, output_dir)
                self.assertEqual(exp.config, _YourConfig(
                    max_epoch=300, max_step=10000, train=_YourConfig.train(
                        batch_size=128
                    )
                ))


# FLOW NINJA
# Copyright 2016-2017 Thomas E. Barchyn

# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
# associated documentation files (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge, publish, distribute,
# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software
# is furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in all copies or
# substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

# flow ninja: the purpose of this is to manage collections of vector estimates and perform
#             assimilations of local flow conditions.

# NOTE: this does not handle 3D flows presently, this only handles 2D flows (az and mag),
#       but . . the flows can have 3D context (e.g., a z coordinate). When there is a 3D
#       flow sensor available, I will update this to work in 3D.

import os
import sys
import copy
from math import *
import numpy as np
import pandas as pd
from threading import Lock

from weight_engine import *
from particle_ninja import *

# import number ninja (you need to specify this yourself!)
if os.name == 'posix':
    try:
        sys.path.append (os.path.abspath ('../packages/'))
    except:
        try:
            sys.path.append (os.path.abspath ('./packages/'))
        except:
            pass

else:
    sys.path.append ('C://Users//tom//Dropbox//python')

from number_ninja import *

class flow_ninja:
    def __init__ (self, weight_engine_in = None, model = 'invdist', space_wt = 0.75, space_zerodist = 100.0,
                  time_zerodist = 3000.0, space_exponent = 0.2, time_exponent = 0.2, is_wind = False):
        '''
        constructor creates the number ninjas to handle az and mag data

        weight_engine_in = pre-defined weight engine, if unsupplied, takes the weight engine
                           parameters defined in other arguments
        model = the model to use for flow weighting
        space_wt = space weight for weight engine, the time weight is 1 - space_wt
        space_zerodist = the distance whereby space influence is zero
        time_zerodist = the distance whereby time influence is zero
        space_exponent = the space weighting exponent
        time_exponent = the time weighting exponent
        is_wind = config to denote this is a wind measurement (and thus the direction of flow
               is input as the direction the flow is coming from, not the direction that it
               is going!). This will be stored internally as the direction it is going (as per
               the semantic concept of flow).
        '''

        self.is_wind = is_wind
        self.az = number_ninja (n = None, is_angle = True, log_time = True, always_return = True)
        self.mag = number_ninja (n = None, is_angle = False, log_time = True, always_return = True)

        # either manually construct weight engine, or use pre-defined one
        if not weight_engine_in is None:
            self.weight_engine = weight_engine_in
        else:
            self.weight_engine = weight_engine (model, space_wt, space_zerodist, time_zerodist,
                                                space_exponent, time_exponent)

        return

    def add_data_np (self, azs, mags, xs = None, ys = None, zs = None, ts = None):
        '''
        method to add numpy arrays of vals, xs, ys, zs, ts. Note that this uses the
        add_data method incrementally to guard against issues with the data and to
        honor the pre-existing setup of the number ninja. This is by no means efficient,
        and direct assignment of the arrays is another option.

        azs = the azimuth values (not optional!)
        mags = the magnitude values (not optional!)
        xs = x positions (optional)
        ys = y positions (optional)
        zs = z positions (optional)
        ts = time positions as a datetime object (optional, defaults to now)
        '''

        # set defaults
        x = None
        y = None
        z = None
        t = None

        for i in range (0, azs.shape[0]):
            if not np.isnan (azs[i]) and not np.isnan (mags[i]):
                if not xs is None:
                    x = xs[i]
                if not ys is None:
                    y = ys[i]
                if not zs is None:
                    z = zs[i]
                if not ts is None:
                    t = ts[i]

                self.add_data (az = azs[i], mag = mags[i], x = x, y = y, z = z, t = t)

        return

    def add_data (self, az, mag, x = None, y = None, z = None, t = None):
        '''
        the method adds values to the list, with a noted x, y, z, time position

        az = the azimuth of flow (in the case of wind, the direction it is coming from) (deg)
        mag = the magnitude of flow
        x = x position (optional)
        y = y position (optional)
        z = z position (optional)
        t = time position as a datetime object (optional, defaults to now)
        '''

        # flip az if this is wind
        if self.is_wind:
            az = (180.0 + az) % 360.0

        # add data to underlying number ninjas
        self.az.add_data (az, x = x, y = y, z = z, t = t)
        self.mag.add_data (mag, x = x, y = y, z = z, t = t)
        return

    def sample (self, test_x, test_y, test_t, test_z = None):
        '''
        method to perform weighted sample of flow conditions at a given location in
        space and time. This can be used for stochastic simulations of lagrangian
        style advection (among other things).

        test_x = x location to sample from
        test_y = y location to sample from
        test_z = z location to sample from (optional)
        test_t = time location to sample from
        returns az and mag
        '''

        # get the weighted sample of the az, then use index to get mag
        az, index = self.az.spacetime_sample (test_x = test_x, test_y = test_y, test_t = test_t,
                                              weight_engine = self.weight_engine, test_z = test_z)
        mag = float (self.mag.vals [index])
        return (az, mag)

    def calc_mean (self, test_x, test_y, test_t, test_z = None, scalar_mean = True):
        '''
        method to perform weighted mean in space for a given location

        test_x = x location to sample from
        test_y = y location to sample from
        test_z = z location to sample from (optional)
        test_t = time location to sample from
        scalar_mean = calculate the scalar mean, if false, return the vector mean
        returns az and mag
        '''

        # if we are returning the scalar mean, calculate the means separately
        if scalar_mean:
            az, influence = self.az.spacetime_mean (test_x = test_x, test_y = test_y, test_t = test_t,
                                                    weight_engine = self.weight_engine, test_z = test_z, circle_ranging = True)
            mag, influence = self.mag.spacetime_mean (test_x = test_x, test_y = test_y, test_t = test_t,
                                                      weight_engine = self.weight_engine, test_z = test_z)

        else:
            # use the mag ninja to get the influence array and manually create vector mean (discard mean mag)
            temp, influence = self.mag.spacetime_mean (test_x = test_x, test_y = test_y, test_t = test_t,
                                                       weight_engine = self.weight_engine, test_z = test_z)

            # create weighted vector mean
            if influence.shape[0] > 0:
                na_mask = ~np.logical_or (np.isnan (self.az.vals), np.isnan (influence))
                mask = np.logical_and ((influence > 0.0), na_mask)
                az_masked = self.az.vals [mask]
                mag_masked = self.mag.vals [mask]
                influence_masked = influence [mask]
                az_masked_radians = az_masked * np.pi / 180.0
                xcp = mag_masked * np.sin (az_masked_radians)
                ycp = mag_masked * np.cos (az_masked_radians)
                xcp_mean = np.average (xcp, weights = influence_masked)
                ycp_mean = np.average (ycp, weights = influence_masked)
                az = degrees (atan2 (xcp_mean, ycp_mean))
                az = az % 360.0
                mag = sqrt (xcp_mean**2.0 + ycp_mean**2.0)

            else:
                az = np.nan             # address issue where there are no records
                mag = np.nan

        return (az, mag)

    def assimilate (self, locs, scalar_mean = True, az_colname = 'az', mag_colname = 'mag'):
        '''
        method to assimilate for a pandas dataframe containing coordinates in space and time.
        This is a useful method for making assimilations to view in paraview or qgis as flow glyphs

        locs = pandas dataframe containing points where we need to create mean estimates for
               must have 'x', 'y', 'z', 't', cols. Can be missing 'z' column if no z coords available.
        scalar_mean = calculate the scalar mean (instead of the vector mean) - these two numbers
                      are rather different and you may choose one vs another for a given flow
                      assimilation task
        az_colname = the aximuth column name (for appending to existing dataframes)
        mag_colname = the magnitude column name (for appending to existing dataframes)
        returns the dataframe with az and mag columns
        '''

        locs [az_colname] = np.nan
        locs [mag_colname] = np.nan

        # calculate weighted means
        for i in range (0, locs.shape[0]):
            x = locs.loc [i, 'x']
            y = locs.loc [i, 'y']
            t = locs.loc [i, 't']

            if 'z' in locs.columns:
                z = locs.loc [i, 'z']
            else:
                z = None

            # perform calculation and append to dataframe
            az, mag = self.calc_mean (test_x = x, test_y = y, test_t = t, test_z = z,
                                      scalar_mean = scalar_mean)
            locs.loc[i, az_colname] = az
            locs.loc[i, mag_colname] = mag

        return (locs)

    def repeat_sample (self, locs, az_colname = 'az', mag_colname = 'mag'):
        '''
        method to repeatedly pull samples from the flow ninja

        locs = pandas dataframe containing points where we need to create mean estimates for
               must have 'x', 'y', 'z', 't', cols. Can be missing 'z' column if no z coords available.
        az_colname = the aximuth column name (for appending to existing dataframes)
        mag_colname = the magnitude column name (for appending to existing dataframes)
        returns the dataframe with az and mag columns
        '''

        locs [az_colname] = np.nan
        locs [mag_colname] = np.nan

        # calculate the sample
        for i in range (0, locs.shape[0]):
            x = locs.loc [i, 'x']
            y = locs.loc [i, 'y']
            t = locs.loc [i, 't']

            if 'z' in locs.columns:
                z = locs.loc [i, 'z']
            else:
                z = None

            # make a sample
            az, mag = self.sample (test_x = x, test_y = y, test_t = t, test_z = z)
            locs.loc[i, az_colname] = az
            locs.loc[i, mag_colname] = mag

        return (locs)

    def timeslice (self, min_time, max_time):
        '''
        method to timeslice the number ninjas to limit the volume of numbers in the ninja

        min_time = the minimum time to keep
        max_time = the maximum time to keep
        '''

        self.az.timeslice (min_time, max_time)
        self.mag.timeslice (min_time, max_time)

        return

    def add_data_file (self, filename, az_col = 'az', mag_col = 'mag'):
        '''
        method to add data from a file. Do note that this could be optimized for huge flows
        as it presently reads in the file twice as it calls the underlying number ninja add_data_file
        method. Also note all the important gotchas from the number_ninja add_data_file method. In
        particular it is vital to note that this only appends data into the ninjas. It does not clear
        the ninjas and replace the data.

        filename = the filename to add data from
        az_col = the azimuth column (defaults to az)
        mag_col = the magnitude column (defaults to mag)
        '''
        self.az.add_data_file (filename, val_col = az_col)
        self.mag.add_data_file (filename, val_col = mag_col)
        return

    def write (self, filename):
        '''
        method to write out the record as a pandas dataframe
        filename = filename to write out to
        '''

        if self.az.vals.shape[0] > 0:
            df = pd.DataFrame (self.az.vals, columns = ['az'])
            df['mag'] = self.mag.vals

            if len (self.az.x) > 0:
                df['x'] = self.az.x
            if len (self.az.y) > 0:
                df['y'] = self.az.y
            if len (self.az.z) > 0:
                df['z'] = self.az.z
            if len (self.az.t) > 0:
                df['t'] = self.az.t

            df.to_csv (filename, index = False)

        return

    def deepcopy (self):
        '''
        method to deepcopy the individual number ninjas
        returns a copy
        '''
        selfcopy = copy.copy (self)
        selfcopy.az = self.az.deepcopy ()
        selfcopy.mag = self.mag.deepcopy ()
        return (selfcopy)


# -*- coding: utf-8 -*-
"""
Created on Sat Feb 16 11:18:30 2019

@author: student
"""

# Guessing Game
from random import randint
user_guess=int(0)
guesses_left=int(5)
out_of_guesses = False   # boolean to determine if the game can be continued for whatever reason
my_rand = randint(1,100) # random number between 1 and 100

def check(): # function checks to see if user_guess is equal to my_rand
    if user_guess == my_rand:
        out_of_guesses=True
    else:
        guesses_left-=1

while out_of_guesses== False:
    user_guess=input("Enter a guess between 1 and 100: ")
    if (user_guess<my_rand):
        check()
        print("Your guess is too low you have" + guesses_left + "guesses left.")
    elif (user_guess>my_rand):
        check()
        print("Your guess is too high you have" + guesses_left + "guesses left.")
    else:
        check()
        print("Great job you did it, and in " + guesses_left + "guesses, maybe " + guesses_left + " and " + my_rand + "are your lucky numbers.")


import bpy
from math import *

def intersects(obj1, obj2):
    obj1_x = obj1.location.x
    obj1_y = obj1.location.y
    obj1_z = obj1.location.z

    obj2_x = obj2.location.x
    obj2_y = obj2.location.y
    obj2_z = obj2.location.z

    obj1_dim_x = obj1.dimensions.x
    obj1_dim_y = obj1.dimensions.y
    obj1_dim_z = obj1.dimensions.z

    obj2_dim_x = obj2.dimensions.x
    obj2_dim_y = obj2.dimensions.y
    obj2_dim_z = obj2.dimensions.z

    th = 0.05*(((obj1_dim_x+obj1_dim_y + obj1_dim_z)/3 + (obj2_dim_x+obj2_dim_y + obj2_dim_z)/3)/2)

    if (fabs(obj1_x - obj2_x)) >= ((obj1_dim_x+obj2_dim_x)/2 + th) or (fabs(obj1_y - obj2_y)) >= ((obj1_dim_y+obj2_dim_y)/2 + th) or (fabs(obj1_z - obj2_z)) >= ((obj1_dim_z+obj2_dim_z)/2 + th):
        return False
    else:
        return True

g = bpy.data.groups["Group.001"]
count = 0
num = 0
for i in g.objects:
    num+=1
    for j in g.objects:
        if i == j:
            continue
        count+=intersects(i, j)

print(count)
print(num)
print((count/(num)))


import baseTest
from AllNeo_Code.Feature.Count import Count
from logging import Logger

class testDB(baseTest):
    def testConnection(self):
        ret_Var = False
        try:
            if(self.testSession):
                print("CONNECTION SUCCESSFUL!")
                ret_Var = True

        except:
            Logger.exception("EXCEPTION OCCURED.")

        #obj = con.getSession()
        """
        con.getNodeCount("Roni") # n : Node, Label, Node outBound, inbound, No of properties,
        con.getNodeCount("Roni","-d")
        con.getLabelCount()
        con.getLableCount("-d")
        con.getNodeCountByLabel("Student")
        con.getNodeCountByLabel("Student","-d")
        con.getAllroperties()
        con.getAllroperties("-d")
        con.getPropertiesByNode("Node")
        con.getPropertiesByNode("Node","-d")
        """

        #1. Count
        #2. Count With Properties

    def testAllCount(self):
        countAll = Count.getAllNodeCount(self.testSession)
        print(countAll)

testObj = testDB()

#testObj.testLogger()
testObj.testConnection()
testObj.testAllCount()


import pymunk
from Engine.loading import load_model, cast_model
from game_class import BaseEngine

NAME = __name__.split('.')[-1]
MODEL = load_model('Components\\Models\\%s' % (NAME,))

CS = None

class Engine(BaseEngine):
    max_health = 30
    size_inc = 1
    engine_force = 5000000
    max_vel = 250
    max_fps = 18

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.i_body = pymunk.Body()
        self.shape = pymunk.Circle(self.body, self.RADIUS)
        self.shape.density = 1

    @classmethod
    def init_class(cls):
        cls._frames, cls.IMAGE_SHIFT = cast_model(MODEL, CS, cls.size_inc)
        cls.precalculate_shape()
        cls.calculate_poly_shape()

    @classmethod
    def precalculate_shape(cls):
        radius = 18

        cls.RADIUS = radius * cls.size_inc

    @classmethod
    def calculate_poly_shape(cls):
        img_poly_left = []
        poly_left = [tuple(e[n] - CS[n] for n in range(2)) for e in img_poly_left]
        poly_right = [(e[0], -e[1]) for e in poly_left[::-1]]
        cls.POLY_SHAPE = [(e[0] * cls.size_inc, e[1] * cls.size_inc) for e in poly_left + poly_right]

Engine.init_class()


from Crypto.PublicKey import RSA
from Crypto.Cipher import PKCS1_OAEP
import zlib
import base64
import os
import os.path
from os.path import isfile, join
from os import listdir

def en_get(file):
    fd = open(file, "rb")
    unencrypted_blob = fd.read()
    fd.close()
    encrypted_blob = encrypt_blob(unencrypted_blob)
    fd = open(file+".enc", "wb")
    fd.write(encrypted_blob)
    fd.close()
    os.remove(file)

def de_get(file):
    fd = open(file+".enc", "rb")
    encrypted_blob = fd.read()
    fd.close()
    #Write the decrypted contents to a file
    fd = open(file, "wb")
    fd.write(decrypt_blob(encrypted_blob))
    fd.close()
    os.remove(file+".enc")

def dee_get(file):
    fd = open(file, "rb")
    encrypted_blob = fd.read()
    fd.close()
    #Write the decrypted contents to a file
    fd = open(file, "wb")
    fd.write(decrypt_blob(encrypted_blob))
    fd.close()
    #os.remove(file)

def getAllFiles():
    dir_path = os.path.dirname(os.path.realpath(__file__))
    dirs = []
    for dirName, subdirList, fileList in os.walk(dir_path):
        for fname in fileList:
           if (fname != 'locker.py' and fname != 'public_key.pem' and fname != 'private_key.pem'):
                dirs.append(dirName + "\\" + fname)
    return dirs

#Our Decryption Function
def decrypt_blob(encrypted_blob):

    #Import the Private Key and use for decryption using PKCS1_OAEP
    rsakey = RSA.importKey(private_key)
    rsakey = PKCS1_OAEP.new(rsakey)

    #Base 64 decode the data
    encrypted_blob = base64.b64decode(encrypted_blob)

    #In determining the chunk size, determine the private key length used in bytes.
    #The data will be in decrypted in chunks
    chunk_size = 512
    offset = 0
    decrypted = ("").encode()

    #keep loop going as long as we have chunks to decrypt
    while offset < len(encrypted_blob):
        #The chunk
        chunk = encrypted_blob[offset: offset + chunk_size]

        #Append the decrypted chunk to the overall decrypted file
        decrypted += rsakey.decrypt(chunk)

        #Increase the offset by chunk size
        offset += chunk_size

    #return the decompressed decrypted data
    return zlib.decompress(decrypted)

#Our Encryption Function
def encrypt_blob(blob):
    #Import the Public Key and use for encryption using PKCS1_OAEP
    rsa_key = RSA.importKey(public_key)
    rsa_key = PKCS1_OAEP.new(rsa_key)

    #compress the data first
    blob = zlib.compress(blob)

    #In determining the chunk size, determine the private key length used in bytes
    #and subtract 42 bytes (when using PKCS1_OAEP). The data will be in encrypted
    #in chunks
    chunk_size = 470
    offset = 0
    end_loop = False
    encrypted =  ("").encode()

    while not end_loop:
        #The chunk
        chunk = blob[offset:offset + chunk_size]

        #If the data chunk is less then the chunk size, then we need to add
        #padding with " ". This indicates the we reached the end of the file
        #so we end loop here
        if len(chunk) % chunk_size != 0:
            end_loop = True
            chunk += (" " * (chunk_size - len(chunk))).encode()

        #Append the encrypted chunk to the overall encrypted file
        encrypted += rsa_key.encrypt(chunk)

        #Increase the offset by chunk size
        offset += chunk_size

    #Base 64 encode the encrypted file
    return base64.b64encode(encrypted)
if os.path.isfile('public_key.pem')==False:

#Generate a public/ private key pair using 4096 bits key length (512 bytes)
    new_key = RSA.generate(4096, e=65537)

    #The private key in PEM format
    private_key = new_key.exportKey("PEM")

    #The public key in PEM Format
    public_key = new_key.publickey().exportKey("PEM")

    fd = open("private_key.pem", "wb")
    fd.write(private_key)
    fd.close()

    fd = open("public_key.pem", "wb")
    fd.write(public_key)
    fd.close()
fd = open("public_key.pem", "rb")
public_key = fd.read()
fd.close()
#Use the private key for decryption
fd = open("private_key.pem", "rb")
private_key = fd.read()
fd.close()

while True:
    choice = int(input(
    "1. Press '1' to encrypt file.\n2. Press '2' to decrypt file.\n3.\n2. Press '3' to encrypt all file.\n3.\n2. Press '4' to decrypt all file.\n3. Press '5' to exit.\n"))
    if choice == 1:
        file=str(input("Enter name of file to encrypt: "))
        en_get(file)
    elif choice == 2:
        file=str(input("Enter name of file to decrypt: "))
        de_get(file)

    elif choice == 3:
        dirs = getAllFiles()
        for file_name in dirs:
            en_get(file_name)
    elif choice == 4:
        dirs = getAllFiles()
        for file_name in dirs:
            dee_get(file_name)

    elif choice == 5:
        break
    else:
        print("Please select a valid option!")


"""
Given a list of non negative integers, arrange them such that they form the largest number.

Example 1:

Input: [10,2]
Output: "210"
Example 2:

Input: [3,30,34,5,9]
Output: "9534330"
"""

class Solution:
    def largestNumber(self, nums: List[int]) -> str:
        nums = sorted(nums, key = self.bi, reverse = True)
        if nums[0] == 0: return "0"
        return "".join([str(i) for i in nums])

    def bi(self, x):
        if x == 0: return 0
        s, k = 0, x
        while x >= 1:
            x = x / 10
            s += 1
        return k /( 10 **s -1)


#/usr/bin/python
#encoding=utf-8

import re
import sys

stage = sys.argv[1]
print(stage)
if stage == '0':
    import nltk
    read_file = './lm_train.text'
    write_file = 'LM.txt'
    label_file = './timit/phone_list.txt'

    char2int = dict()
    char2int['#']=0
    f = open(label_file, 'r')
    for line in f.readlines():
        line = line.strip().split()
        char2int[line[0]] = int(line[1])
    f.close

    sen = []
    f = open(read_file, 'r')
    wf = open(write_file, 'w')
    dict_map = '#123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLM'

    for line in f.readlines():
        line = line.strip().lower().split()
        for phone in line:
            sen.append(dict_map[char2int[phone]])
        wf.writelines(' '.join(sen)+'\n')
        sen = []
    f.close()

if stage == '1':
    #import pytorch_ctc
    #labels = "#'acbedgfihkjmlonqpsrutwvyxz_"
    #pytorch_ctc.generate_lm_trie('./dict.txt', './bigram.ken','./trie', labels, 0, 28)
    dicte = '123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLM'
    f = open('./dict.txt', 'w')
    for i in range(len(dicte)):
        f.writelines(dicte[i]+'\n')
    f.writelines('<s>\n')
    f.writelines('</s>\n')
    f.writelines(' \n')
    f.close()


from django.shortcuts import render

# Create your views here.

from django.http import  HttpResponse
from django.template import loader

def productInfo(req):
    template=loader.get_template("productinfo.html")
    data= {"name": "VIVO",
           "desc": " Smart Phone",
           "price": 45555}
    return HttpResponse(template.render(data,req))

def productList(req):
    template=loader.get_template("productList.html")
    data= {

        "products": [{"name": "VIVO",
           "desc": " Smart Phone",
           "price": 45555},

         {"name": "OPPO",
               "desc": " Smart Phone",
               "price": 50000,
               "feature": ["8GB RAM", "30 MP Camera", "6 Inch Display"]},

        {"name": "Samsung",
               "desc": " Smart Phone",
               "price": 30000,
            "feature": ["4GB RAM", "20 MP Camera", "5 Inch Display"]},

         ]

    }
    return HttpResponse(template.render(data,req))


"""dropbox URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/3.2/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path, include
from rest_framework.routers import DefaultRouter
from file.views import FileViewSet, RecentFileView, StarredFileView, UpdateFileView, ShareFileView, DeleteFileView
from trash.views import TrashViewSet

router = DefaultRouter(trailing_slash=False)
router.register(r'files', FileViewSet, basename='files')
router.register(r'trash', TrashViewSet, basename='trash')

urlpatterns = [
    path('admin/', admin.site.urls),
    path(r'', include(router.urls)),
    path('users/', include('user.urls')),
    path('', include('file.urls')),
    path('', include('trash.urls')),
    path('myfile/recent', RecentFileView.as_view()),
    path('myfile/starred', StarredFileView.as_view()),
    path('myfile/update/<str:file_name>', UpdateFileView.as_view()),
    path('myfile/share', ShareFileView.as_view()),
    path('myfile/delete/<str:file_name>', DeleteFileView.as_view()),
]


"""User admin classes."""

# Django
from django.contrib import admin
from django.contrib.auth.admin import UserAdmin as BaseUserAdmin

# Models
from user.models import Profile
from django.contrib.auth.models import User

@admin.register(Profile)
class ProfileAdmin(admin.ModelAdmin):
    """Profile admin."""
    # Listado de opciones al incio de Dashboard-Admin
    list_display = ('pk','user', 'phone_number', 'website', 'picture')

    # Permite darle Click a la opcion, envia al detalle
    list_display_links = ('pk','user')

    #Permite editar el registro desde el listado
    list_editable = ('phone_number', 'website', 'picture')

    #Permite agregar buscar con los campos que deseann buscar
    search_fields = ('user__email', 'user__first_name', 'user__last_name', 'phone_number')

    #Agrega filtros por fecha de creacion y modificacion
    list_filter = ('created', 'modified', 'user__is_active', 'user__is_staff')

    #Permite cambiar el orden de los datos o agrupalas
    fieldsets = (
        ('Profile', {
            'fields': (('user', 'picture'),),
        }),
        ('Exra Info', {
            'fields': (
                ('website', 'phone_number'),
                ('biography')
            )
        })
    )

    # Esto deshabilita la opcion Modificar
    readonly_fields = ('created', 'modified', 'user')

class ProfileInline(admin.StackedInline):
    """ Prpfie in-line admin for users"""
    model = Profile
    can_delete = False
    verbose_name_plural = 'profiles'

class UserAdmin(BaseUserAdmin):
    """ Add profile admin to base user admin"""
    inlines = (ProfileInline,)
    list_display = (
        'username',
        'email',
        'first_name',
        'last_name',
        'is_active',
        'is_staff',
    )

admin.site.unregister(User)
admin.site.register(User, UserAdmin)


def get_result(A,B,probs):
    result = 0

    exps = [0 for i in range(int(A+2))]
    prob_correct = reduce(lambda x,y:x*y, probs)

    partials = [1 for i in range(A)]
    partials[0] = probs[0]
    for i,elem in enumerate(partials[1:]):
        partials[i+1] = partials[i] * probs[i+1]

    for j,elem in enumerate(exps[:A-1]):
        right_cost = 2 * (j+1) + (B-A) + 1
        wrong_cost = right_cost + B + 1
        exps[j] = partials[len(partials)-j-2] * right_cost + (1 - partials[len(partials)-j-2]) * wrong_cost

    exps[A-1] = A + B + 1
    exps[A] = (B-A+1) * prob_correct + (2*B - A + 2) * (1 - prob_correct)
    exps[A+1] = B + 2

    return min(exps)

def solve():

    file_in = file("A-small-attempt0.in")
    file_out = file("A-small-attempt0.out","w")
    num_of_cases = int(file_in.readline())

    for j in range(num_of_cases):
        line = file_in.readline()

        line = line.strip().split()

        A = int(line[0])
        B = int(line[1])

        probs = file_in.readline()
        probs = probs.strip().split()
        probs = [float(i) for i in probs]

        result = str(get_result(A,B,probs))

        file_out.write("Case #%d: %s\n" % ((j+1),result))

    file_out.close()

if __name__ == '__main__':
    solve()


# coding: utf-8

from __future__ import absolute_import
from datetime import date, datetime  # noqa: F401

from typing import List, Dict  # noqa: F401

from swagger_server.models.base_model_ import Model
from swagger_server import util

class COMMAND(Model):
    """NOTE: This class is auto generated by the swagger code generator program.

    Do not edit the class manually.
    """

    def __init__(self, command=None):  # noqa: E501
        """COMMAND - a model defined in Swagger

        :param command: The command of this COMMAND.  # noqa: E501
        :type command: str
        """
        self.swagger_types = {
            'command': str
        }

        self.attribute_map = {
            'command': 'command'
        }

        self._command = command

    @classmethod
    def from_dict(cls, dikt):
        """Returns the dict as a model

        :param dikt: A dict.
        :type: dict
        :return: The COMMAND of this COMMAND.  # noqa: E501
        :rtype: COMMAND
        """
        return util.deserialize_model(dikt, cls)

    @property
    def command(self):
        """Gets the command of this COMMAND.

        :return: The command of this COMMAND.
        :rtype: str
        """
        return self._command

    @command.setter
    def command(self, command):
        """Sets the command of this COMMAND.

        :param command: The command of this COMMAND.
        :type command: str
        """
        if command is None:
            raise ValueError("Invalid value for `command`, must not be `None`")  # noqa: E501

        self._command = command


# -*- coding: utf-8 -*-
#
# 上海，深圳证券分红计划
# http://stock.finance.qq.com/corp1/distri.php?zqdm=${code}
#
# Example:
# scrapy crawl SecuritiesDividendPlan -a years=2015
# scrapy crawl SecuritiesDividendPlan -a years=2015,2016
# scrapy crawl SecuritiesDividendPlan -a years=2015,2016 -a codes=600000
# scrapy crawl SecuritiesDividendPlan -a years=2015,2016 -a codes=600000,600036

import scrapy

import logging

from securities_china.SecuritiesDB import SecuritiesDB

logger = logging.getLogger(__name__)

class SecuritiesDividendPlan(scrapy.Spider):
    name = "SecuritiesDividendPlan"
    allowed_domains = ["stock.finance.qq.com"]
    url_tpl = "http://stock.finance.qq.com/corp1/distri.php?zqdm="

    def __init__(self, years=None, codes=None):
        self.db = SecuritiesDB()
        if (years is not None):
            self.years = years.split(",")
            logger.info("scrape securities dividend plan for years: " + years)
        else:
            self.years = None
            logger.info("scrape securities dividend plan for all years.")

        if (codes is not None):
            self.codes = codes.split(",")
            logger.info("scrape securities dividend plan for codes: " + codes)
        else:
            self.codes = None
            logger.info("scrape securities dividend plan for all codes.")

        if (self.codes is not None):
            self.start_urls = [self.url_tpl + code for code in self.codes]
        else:
            self.start_urls = [
                self.url_tpl + code[0]
                for code in
                self.db.query_securities_code_for_dividend_plan(self.years)
                .fetch_row(maxrows=0)
            ]

    def parse(self, response):
        self.db.insert_securities_dividend_plan(self.merge_rows(response))

    def merge_rows(self, response):
        import itertools
        import operator
        from decimal import getcontext, Decimal
        getcontext().prec = 4

        it = itertools.groupby(
            self.parse_rows(response), operator.itemgetter(1))
        for key, subiter in it:
            eps = div1 = div2 = div3 = Decimal(0)
            code = year = eps = ""
            for item in subiter:
                div1 = div1 + Decimal(item[3])
                div2 = div2 + Decimal(item[4])
                div3 = div3 + Decimal(item[5])
                code = item[0]
                year = item[1]
                eps = Decimal(item[2])
            record = (code, year, str(eps), str(div1), str(div2), str(div3))
            logger.info("merged: " + str(record))
            yield record

    def parse_rows(self, response):
        code = response.url.split("zqdm=")[1]
        rows = response.xpath("/html/body/div/div/table[2]/tr")
        for i in range(2, len(rows)):
            values = rows[i].xpath("td/text()").extract()
            year = values[0][:4]
            eps = values[1] if values[1] != "--" else "0.0"
            div1 = values[2] if values[2] != "--" else "0"
            div2 = values[3] if values[3] != "--" else "0"
            div3 = values[4] if values[4] != "--" else "0.0"
            if (self.years is None or year in self.years):
                record = (code, year, eps, div1, div2, div3)
                logger.info("parsed: " + str(record))
                yield record


"""launch toolbox
"""
import sys
import argparse

from sftoolboxqt import qtgui
from sftoolbox.projects import load_project_from_filepath
from sftoolboxqt.widgets import ProjectWidget

def main(args):
    """launch sf toolbox
    """
    parser = argparse.ArgumentParser()
    parser.add_argument('-p', '--project')
    parser.add_argument('--live', action='store_true', default=False)
    app = qtgui.QApplication([])

    namespace = parser.parse_args(args)
    project = None
    if namespace.project:
        project = load_project_from_filepath(namespace.project)
        project.activate()

    w = ProjectWidget(project)
    w.live_edit = namespace.live

    w.show()

    app.exec_()

if __name__ == '__main__':
    sys.exit(main(sys.argv[1:]))