'''
Создайте программу для игры в ""Крестики-нолики"" при помощи виртуального окружения и PIP
'''

import emoji

array = [['1' ,'|', '2' ,'|', '3'], ['-','-','-','-','-'],['4' ,'|', '5' ,'|', '6'], ['-','-','-','-','-'],['7' ,'|', '8' ,'|', '9'] ]
player1 = emoji.emojize(":hollow_red_circle:")
player2 = emoji.emojize(":cross_mark:")

def print_desk():
    for i in array:
        print(' '.join(list(map(str, i))))

def motion(player):
    number = str(input(f'Введите число, на которое нужно поставить {player}: '))
    flag = False
    for i in range(len(array)):
        for k in range(len(array)):
            if array[i][k] == number:
                array[i][k] = player
                flag = True
    print_desk()
    return flag

def itogi():
    if array[0][0] == array[0][2] == array[0][4] or array[2][0] == array[2][2] == array[2][4] or array[4][0] == array[4][2] == array[4][4] or array[0][0] == array[2][0] == array[4][0] or array[0][2] == array[2][2] == array[4][2] or array[0][4] == array[2][4] == array[4][4] or array[0][0] == array[2][2] == array[4][4] or array[0][4] == array[2][2] == array[4][0]:
        return False
    else:
        return True

def draw():
    flag = False
    for i in range(len(array)):
        for k in range(len(array)):
            if array[i][k] in ["1","2","3","4","5","6","7","8","9"]:
                flag = True
    return flag

print_desk()

current_player = player1
while itogi() and draw():
    if motion(current_player):
        if current_player == player1:
            current_player = player2
        else:
            current_player = player1
    else:
        print(f'Неверное число, игрок {current_player} вводит число заново')

if not draw() and itogi():
    print('Ничья - игра окончена')
elif current_player == player1:
    print("Победил игрок", player2)
else:
    print("Победил игрок", player1)


import os
from random import shuffle

import numpy as np

def get_paths(type, partition, path_relations, path_features, input2feature, type2class):

    dict = {}

    for input in input2feature:
        dict[input] = []

    dict['label'] = []

    for class_num in range(type2class[type]):

        img_list = os.listdir(os.path.join(
            path_relations, partition, str(class_num)))

        for single_img in img_list:

            img_name = single_img.split('.')[0]

            for input in input2feature:
                dict[input].append(os.path.join(path_features, type, partition, str(
                    class_num), img_name, input2feature[input]))

            dict['label'].append(class_num)

    for input in input2feature:
        assert len(dict[input]) == len(dict['label'])

    return dict

def get_data(dict_data, step, input2size):

    dict = {}
    count_feature = {}
    total_len = 0

    for key in dict_data:

        if(key != 'label'):

            dir_list = dict_data[key]
            feature_path = dir_list[step]
            feature_list = os.listdir(feature_path)
            feature_len = len(feature_list)
            dict[key] = []

            total_len += feature_len
            count_feature[key] = feature_len

            if feature_len > 0:
                for i in range(feature_len):

                    feature_tensor = np.load(
                        os.path.join(feature_path, feature_list[i]))
                    dict[key].append(np.reshape(
                        feature_tensor, [input2size[key], ]))

            else:
                dict[key] = np.zeros((1, input2size[key]))

    label = dict_data['label']

    dict['label'] = label[step]

    return dict

def shuffle_dict(self, dict):

    zipped = list(zip(*dict.values()))

    shuffle(zipped)

    shuffled_list = list(zip(*zipped))

    count = 0
    dict_shuffled = {}

    for key in dict:
        dict_shuffled[key] = shuffled_list[count]
        count += 1

    return dict_shuffled


from col import COL
from range import *
import update as update
from query import *
import utility as util
import math
from copy import deepcopy
import list as lst
from rule import RULE

def bins(cols, rowss):
    """
    Function:
        bins
    Description:
        Computes the bins (ranges of values) for a list of
        columns based on the values of each column in a list of rows.

    Input:
        cols
            a list of columns to compute the bins for.
        rowss:
            a dictionary of rows where each key represents a row label and the value is a list of values for each column.
    Output:
        out
            a list of computed bins (ranges of values) for each column.
            Each bin is represented as a list of Range objects. If the column is symbolic,
            the list will only contain Range objects. If the column is not symbolic,
            the list may contain merged ranges of adjacent bins.
    """
    out = []
    for col in cols:
        ranges = {}
        for y, rows in rowss.items():
            for row in rows:
                if (isinstance(col, COL)):
                    col = col.col
                x = row[col.at]
                if x != "?":
                    k = int(bin(col, float(x) if x != "?" else x))
                    ranges[k] = ranges[k] if k in ranges else RANGE(col.at, col.txt, float(x) if x != "?" else x)
                    update.extend(ranges[k], float(x), y)
        ranges = {key: value for key, value in sorted(ranges.items(), key=lambda x: x[1].lo)}
        newRanges = {}
        i = 0
        for key in ranges:
            newRanges[i] = ranges[key]
            i += 1
        newRangesList = []
        if hasattr(col, "isSym") and col.isSym:
            for item in newRanges.values():
                newRangesList.append(item)
        out.append(newRangesList if hasattr(col, "isSym") and col.isSym else mergeAny(newRanges))
    return out

def bin(col, x):
    """
    Function:
        bin

    Description:
        The bin function takes a column object col and a value x as
        input and returns the corresponding bin value for x based on
        the range of col. If x is "?" or col is a symbol column, then
        the function simply returns x.

    Input:
        col - A column object containing the range of values to be binned.
        x - A value to be binned.

    Output:
        The corresponding bin value for x based on the range of col.
    """
    if x=="?" or hasattr(col, "isSym"):
        return x
    tmp = (col.hi - col.lo)/(util.args.bins - 1)
    return 1 if col.hi == col.lo else math.floor(x / tmp + 0.5) * tmp

def mergeAny(ranges0):
    """
    Function:
        mergeAny

    Description:
        The mergeAny function takes a list of range objects ranges0
        as input and recursively merges adjacent ranges until there
        are no more adjacent ranges to merge. The resulting ranges
        are returned in a list.

    Input:
        ranges0 - A list of range objects.

    Output:
        A list of range objects resulting from merging adjacent ranges in ranges0.
    """
    def noGaps(t):
        for j in range(1, len(t)):
            t[j].lo = t[j-1].hi
        t[0].lo = -float("inf")
        t[-1].hi = float("inf")
        return t
    ranges1, j = [], 0
    while j < len(ranges0):
        left, right = ranges0[j], ranges0[j+1] if j + 1 < len(ranges0) else None
        if right:
            y = merge2(left.y, right.y)
            if y:
               j = j+1
               left.hi, left.y = right.hi, y
        ranges1.append(left)
        j += 1
    return noGaps(ranges0) if len(ranges1)==len(ranges0) else mergeAny(ranges1)

def merge2(col1, col2):
    """
    Function:
        merge2

    Description:
        The merge2 function takes two columns col1 and col2 as inputs,
        merges them using the merge function, and returns the merged
        column if the distance between the merged column and the individual
        columns is less than or equal to the expected distance based on their sizes.

    Input:
    col1 - A column object containing data to be merged.
    col2 - A column object containing data to be merged with col1.

    Output:
        The merged column if the distance between the merged column
        and the individual columns is less than or equal to the expected
        distance based on their sizes. Otherwise, no output is returned.
    """
    new = merge(col1, col2)
    if div(new) <= (div(col1)*col1.n + div(col2)*col2.n)/new.n:
        return new

def merge(col1, col2):
    """
    Function:
        merge

    Description:
        The merge function takes two columns col1 and col2 as inputs and returns
        a new column that contains all the data from both input columns. If col1
        has the isSym attribute set to True, it merges the data in col2 with the
        data in col1 using the has dictionary. Otherwise, it merges the data using
        the has list. Additionally, if col1 does not have the isSym attribute set to True,
        it updates the lo and hi attributes of the new column to the minimum and maximum
        values of the lo and hi attributes of the input columns, respectively.

    Input:
        col1 - A column object containing data to be merged.
        col2 - A column object containing data to be merged with col1.

    Output:
        A new column object that contains all the data from col1 and col2.
    """
    new = deepcopy(col1)
    if hasattr(col1, "isSym") and col1.isSym:
        for x, n in col2.has.items():
            add(new, x, n)
    else:
        for n in col2.has:
            add(new, n)
        new.lo = min(col1.lo, col2.lo)
        new.hi = max(col1.hi, col2.hi)
    return new

def xpln(data, best, rest):
    def v(has):
        return value(has, len(best.rows), len(rest.rows), "best")
    def score(ranges):
        rule = RULE(ranges, maxSizes)
        if rule:
            print(showRule(rule))
            bestr= selects(rule, best.rows)
            restr= selects(rule, rest.rows)
            if len(bestr) + len(restr) > 0:
                return v({"best": len(bestr), "rest": len(restr)}), rule
    tmp, maxSizes = [], {}
    for ranges in bins(data.cols.x, {"best": best.rows, "rest": rest.rows}):
        maxSizes[ranges[0].txt] = len(ranges)
        print("")
        for range in ranges:
            print(range.txt, range.lo, range.hi)
            tmp.append({"range": range, "max": len(ranges), "val": v(range.y.has)})

    rule, most = firstN(sorted(tmp, key=lambda x: x["val"], reverse=True), score)
    return rule, most

def firstN(sortedRanges, scoreFun):
    print("")
    for r in sortedRanges:
        print(r["range"].txt, r["range"].lo, r["range"].hi, round(r["val"], 2), r["range"].y.has)
    first = sortedRanges[0]["val"]
    def useful(range):
        if range["val"] > 0.05 and range["val"] > first / 10:
            return range

    sortedRanges = list(filter(useful, sortedRanges))
    most, out = -1, None

    for n in range(len(sortedRanges)):
        tmp, rule = scoreFun([r["range"] for r in sortedRanges[:n + 1]]) or (None, None)

        if tmp and tmp > most:
            out, most = rule, tmp

    return out, most

def showRule(rule):
    def pretty(range):
        return range['lo'] if range['lo'] == range['hi'] else [range['lo'], range['hi']]

    def merges(attr, ranges):
        return list(map(pretty, merge(sorted(ranges, key=lambda r: r['lo'])))), attr

    def merge(t0):
        t, j = [], 0
        while j < len(t0):
            left, right = t0[j], t0[j+1] if j+1 < len(t0) else None
            if right and left['hi'] == right['lo']:
                left['hi'] = right['hi']
                j += 1
            t.append({'lo': left['lo'], 'hi': left['hi']})
            j += 1
        return t if len(t0) == len(t) else merge(t)

    return lst.kap(rule, merges)

def selects(rule, rows):
    def disjunction(ranges, row):
        for range in ranges:
            lo = int(range['lo']) if isinstance(range['lo'], str) else range['lo']
            hi = int(range['hi']) if isinstance(range['hi'], str) else range['hi']
            at = int(range['at'])
            x = row[at]
            if x == "?":
                return True
            x = float(x)
            if lo == hi and lo == x:
                return True
            if lo <= x and x < hi:
                return True
        return False

    def conjunction(row):
        for ranges in rule.values():
            if not disjunction(ranges, row):
                return False
        return True

    return [r for r in rows if conjunction(r)]


# -*- coding: utf-8 -*-
"""
Created on Fri Feb 28 16:56:37 2020

@author: Stoephu
"""

# -*- coding: utf-8 -*-
"""
Created on Sun Dec  1 12:32:38 2019

@author: Stoephu
"""
import timeit as time
import numpy as np
from itertools import product, permutations
from itertools import groupby
import matplotlib.pyplot as plt
from collections import deque
debug = False
start = time.default_timer()
#part1
inputs = [string for string in open("data.txt")]
strings = [string.strip() for string in inputs]
dungeon = np.empty((len(strings),len(strings[0])),dtype="int")
for y in range(dungeon.shape[0]):
    for x in range(dungeon.shape[1]):
        dungeon[y,x] = ord(strings[y][x])

#setup
diffUpper = ord("a")-ord("A")
player = 64
non_keys = set(["@","#","."])
keys = set()
doors = set()
shape = dungeon.shape

for string in strings:
    for char in string:
        if (not char in non_keys) and char.islower():
            keys.add(ord(char))
        if (not char in non_keys) and char.isupper():
            doors.add(ord(char))

#node for graph class
class tile:

    def __init__(self,tileValue, coord):
        self.value = tileValue
        self.neighbours = set()
        self.coord = coord
        self.important = {}

    def addNeighbour(self,neigh):
        self.neighbours.add(neigh)

    def add_important(self,tile,distance):
        self.important[tile] = distance

#generate big graph
#convert map to graph
rand = deque()
time_needed = {}
index = np.where(dungeon==player)
orpcoord = (index[0][0],index[1][0])
dungeon[orpcoord[0]-1:orpcoord[0]+2,orpcoord[1]-1:orpcoord[1]+2] = np.array([[64,35,64],[35,35,35],[64,35,64]])
indices = np.where(dungeon == player)
pcoords = []
for i in range(len(indices[0])):
    pcoords.append((indices[0][i],indices[1][i]))

playertiles = [tile(ord("@"),pcoord) for pcoord in pcoords]
rand.extend(playertiles)
for playertile in playertiles:
    time_needed[playertile] = 0
checked = set()
checked.update(pcoords)
alltiles = {}
alldoors = set()
allkeys = set()

def get_unchecked_neighbours(current):
    #string = chr(current.value) + " => "
    y,x = current.coord
    neighbours = []
    x1 = (y,x+1)
    x2 = (y,x-1)
    x3 = (y+1,x)
    x4 = (y-1,x)
    xis = [x1,x2,x3,x4]
    for xi in xis:
        if not xi in checked:
            if xi[0] >= 0 and xi[0]< shape[0] and xi[1]>=0 and xi[1]< shape[1]:
                tilevalue = dungeon[xi[0],xi[1]]
                if  tilevalue == ord("#"):
                    pass
                else:
                    ntile = tile(tilevalue,xi)
                    if chr(tilevalue).isupper():
                        alldoors.add(ntile)
                    elif chr(tilevalue).islower():
                        allkeys.add(ntile)
                    ntile.addNeighbour(current)
                    current.addNeighbour(ntile)
                    #string += chr(ntile.value) +","
                    neighbours.append(ntile)
                    alltiles[xi] = ntile
                    time_needed[ntile] = time_needed[current] + 1
        else:
            if xi in alltiles:
                current.addNeighbour(alltiles[xi])
    for neigh in neighbours:
        checked.add(neigh.coord)
    #print(string)
    return neighbours

i = 0
while rand:
    current = rand.popleft()
    neigh = get_unchecked_neighbours(current)
    rand.extend(neigh)
    if not i%20000:
        print(len(rand))
        #print(neigh)
    i+= 1

def show_all_important(node):
    string = chr(node.value)+ " => "
    for ne in node.important:
        string += chr(ne.value) + str(node.important[ne]) + ","
    print(string)

#reduce graph to only important neighbours
print("reduce!")

def get_unchecked_neighbours_graph(node):
    neigh = node.neighbours
    a = []
    for ne in neigh:
        if not ne in checked:
            a.append(ne)
    for ne in a:
        checked.add(ne)
    return a

all_important = alldoors.union(allkeys)
all_important.update(playertiles)

for node in all_important:
    rand = deque()
    checked = set()
    time_needed = {}
    rand.append((node,0))
    checked.add(node)
    while rand:
        current, i = rand.popleft()
        neigh = get_unchecked_neighbours_graph(current)
        for ne in neigh:
            if ne in all_important:
                print(chr(node.value),"=>",chr(ne.value),i+1)
                node.add_important(ne,i+1)
                ne.add_important(node,i+1)
            else:
                rand.append((ne,i+1))
        if not i%20000:
            print(len(rand))
            #print(neigh)

#breitensuche

def get_graph_neighbours(coord):
    nodes,tilename, old = coord
    a = []
    for i in range(len(nodes)):
        node = nodes[i]
        for ne in node.important:
            if chr(ne.value).islower() and not chr(ne.value) in old:
                        l = list(old)
                        l.append(chr(ne.value))
                        new = tuple(sorted(l))
            else:
                new = old
            newnodes = list(nodes)
            newnodes[i] = ne
            newnodes = tuple(newnodes)
            necoord = (newnodes,1,new)
            if chr(ne.value).isupper() and not chr(ne.value + diffUpper) in new:
                pass
            else:
                if faster_time(necoord, time_needed[coord] + node.important[ne]):
                    a.append(necoord)
    return a

lowesttime = 10000
def faster_time(coord,time):
    global lowesttime
    faster = False
    if coord in time_needed:
        if time_needed[coord]>time:
            time_needed[coord] = time
            faster = True
    else:
        time_needed[coord] = time
        faster = True
    if len(coord[2]) == len(allkeys) and time < lowesttime:
        print("lower!")
        lowesttime = time
    return faster

def get_tree(node, tree = set(), indent = 0):
    tree.add(node)
    string = chr(node.value) + "\n"
    a = set()
    for ne in node.important:
        if not ne in tree:
            a.add(ne)
    tree.update(node.important)
    for ne in a:
            string += " "*indent + "¦" + "\n"
            string += " "*indent + "-" + str(node.important[ne]) +" " + get_tree(ne,tree, indent + 1)
    return string
for playertile in playertiles:
    print(get_tree(playertile))

rand = deque()
time_needed = {}
old  = ()
coord = (tuple(playertiles),chr(playertile.value),old)
rand.append(coord)
time_needed[coord] = 0
current = coord
i = 0
last12 =[]
while rand or len(allkeys)>len(current[2]):
    current = rand.popleft()
    neigh = get_graph_neighbours(current)
    rand.extend(neigh)
    if debug:
        print("current =", chr(current[0][0].value),chr(current[0][1].value),chr(current[0][2].value),chr(current[0][3].value))
        print("keys",current[2])
        for ne in current[0][0].important:
            print(chr(ne.value))
        print("rand:")
        #for r in rand:
        #    print(r[1],r[2])
        print("#rand",len(rand))
        print("diff keys",len(allkeys), len(current[2]))
    if not i%20000:
        print(len(rand))
        print(current[2])
        last12.append(len(current[2]))
        if len(last12)>12:
            last12.pop(0)
        print("last12",sum(last12)/12)
        print(len(allkeys), len(current[2]))
    i += 1
print("steps needed", lowesttime)
print(time.default_timer()-start)


import pyodbc

class PyDB(object):

    def __init__(self, server, database):
        self.connection = pyodbc.connect(
            r'DRIVER={ODBC DRiver 11 for SQL Server};'
            r'SERVER='+str(server)+';'
            r'DATABASE='+str(database)+';'
            r'Trusted_connection=yes;'
        )
        self.cursor = self.connection.cursor()

    def select_table(self, table):
        data_cursor = self.cursor.execute(
            'select * from {}'.format(table)
        )
        return data_cursor.fetchall()

    def call_procedure(self, procedure):
        data_cursor = self.cursor.execute(
            "{CALL"+str(procedure)+"}"
        )
        return data_cursor.fetchall()


import json
import logging
import os
import pickle
from pathlib import Path

import numpy as np
import pandas as pd
from PIL import Image
from box import Box

import mesh_utils.image_operations as image_operations
import mesh_utils.utilities as utilities
from mesh_utils import Tags
from mesh_utils.image_landmarks import a4ch_image_landmarks

class VTKSliceDataframe(object):
    """ class designed to keep track of the exported data """

    def __init__(self, opts):
        self._df = pd.DataFrame()
        self.required_keys = ['source_file', 'rotation', 'origin', 'landmarks', 'aligned', 'dest_file', 'vtk_img',
                              'model_id']
        self.vtk_id = 0
        self.df_id = 0
        self.filename = os.path.join(opts.output_dir, opts.name, "vtk_df")
        self.max_rows = 60  # save images and df after this many rows
        self.include_inverse_images = opts.include_inverse_images
        self.inverses_included = False
        self.verbose = opts.verbose

    def load_from_pck(self, filename):
        """ Load dataframe from pickle file
        :param filename: Full path to the pickle file
        """
        self._df = pickle.load(open(filename, 'rb'))
        self.vtk_id = np.max(self._df.vtk_id)

    def add_record(self, record: dict):
        """ Add a row to the dataframe. Will save dataframe if max rows is reached (avoids disk errors)
        :param record: a dictionary with each key representing a column in the dataframe
        """
        for r in self.required_keys:
            assert r in record, "record is missing required key {}".format(r)
        # assign a unique id
        record['vtk_id'] = self.vtk_id
        # assign dir column to simplify looking at where it came from
        record['dir'] = os.path.dirname(record['source_file'])
        self.vtk_id += 1
        self._df = self._df.append(record, ignore_index=True)
        # if array gets too large than it will fail so save periodically instead
        if self._df.shape[0] >= self.max_rows:
            print("reached max rows, saving and starting a new df")
            self.save_df()
            self._df = pd.DataFrame()
            self.df_id += 1
            self.inverses_included = False

    def get_df(self):
        return self._df

    def save_df(self, exts=(".pck", ".xlsx")):
        """  Save the current dataframe. Can either save as a pickle file or an excel file
        :param exts: how to save dataframe. Default saves both pickle and excel.
        """
        if self._df.shape[0] < 1:
            print("no rows in df, not saving...")
            return
        if self.include_inverse_images:
            self._add_inverse_images()  # TODO: need to handle axis name parameter here somehow
        for ext in exts:
            filename = self.filename + '_' + str(self.df_id) + ext
            if ext == '.pck':
                # export a pandas dataframe
                pickle.dump(self._df, open(filename, 'wb'))
            elif ext == '.xlsx':
                cols = list(self._df.columns)
                cols.remove('vtk_img')  # don't save image data to excel
                if self.verbose:
                    logging.info(f"Dataframe cols in excel: {cols}")
                try:
                    self._df.to_excel(filename, columns=cols)
                except:
                    logging.info("\tunable to save excel dataframe. Is file open?")
            elif ext == '.heart':
                # format to load back into this class
                pickle.dump(self, open(filename, 'wb'))
            else:
                raise ValueError("file type {} not among recognized options".format(ext))

    def recreate_vtk(self, output_dir):
        """ rewrite all vtk files in the dataframe """
        from .heart import Heart
        for i, row in self._df.iterrows():
            heart = Heart(row.source_file)
            outname = os.path.join(output_dir, str(row.unique_id))
            heart.write_vtk('', type_='PolyData', outname=outname)

    def _add_inverse_images(self, axis="axis0"):
        """ add the inverse of all images """
        logging.warning("Adding inverse images but assuming full rotation so last is dropped. See code for more.")
        # The assumption in this function is that if you are flipping images it's because you want the entire rotation
        # Therefore this function will actually overwrite the last image in each rotation since that image is actually
        # just the flipped version of the first image.
        flip_df = self._df[~self._df.has_inverse]  # only include for rows where inverse hasn't been addded
        num_rotations = flip_df.rotation_id.max()
        new_rows = list()
        for i, row in flip_df.iterrows():
            row_copy = row.copy()
            row_copy.rotation_id += num_rotations
            row_copy[axis] = utilities.check_angle(row.axis0 + np.pi)
            row_copy.vtk_img = row_copy.vtk_img.transpose(Image.FLIP_TOP_BOTTOM)  # actually flips left to right
            new_rows.append(row_copy)
        # separate out adding new_rows to avoid changing size of df during iteration
        for nr in new_rows:
            self._df = self._df.append(nr, ignore_index=True)
            # drop the one duplicate created in the middle when start gets rotated
        self._df.drop_duplicates(["model_id", "rotation_id"], inplace=True)
        # drop the last value because this one is duplicate of first
        num_rotations = self._df.rotation_id.max()
        self._df = self._df[(self._df.rotation_id != num_rotations) | self._df.has_inverse]  # don't remove previous samples
        self._df.sort_values(["model_id", "rotation_id"], inplace=True)
        self._df["has_inverse"] = True  # mark that all rows have an inverse

    def save_images(self, output_dir):
        """ save the vtk images as pngs to output_dir """
        utilities.mkdir(os.path.join(output_dir, 'slices'))
        for _, row in self._df.iterrows():
            save_name = '_'.join(['models{:d}'.format(int(row.model_id)),
                                  'v{:d}'.format(int(row.vtk_id)),
                                  row.dir + '.png'])
            slice_save_name = os.path.join(output_dir, 'slices', save_name)
            img = image_operations.process_output_image(row.vtk_img)
            image_operations.save_img(img, slice_save_name)

class PseudoDatasetDataframe(object):
    """ class designed to keep track of the exported data """
    def __init__(self, unique_id=0):
        self._df = pd.DataFrame()
        self.required_keys = ['tissue_img', 'label_img', 'vtk_id', 'img_df_num', 'source_file', 'dir']
        self.unique_id = unique_id  # can be passed in to continue numbering from a previous dataframe

    def restore_from_pickle(self, df_filename, overwrite_unique_id=False):
        """ restore a class of this from a saved dataframe"""
        df = pickle.load(open(df_filename, 'rb'))
        for r in self.required_keys:
            assert r in list(df.columns), f"required key {r} is missing from  {df_filename} with cols {df.columns}"
        self._df = df
        if overwrite_unique_id:
            num_rows = df.shape[0]
            new_ids = np.arange(self.unique_id, self.unique_id + num_rows)
            self._df.unique_id = new_ids
        self.unique_id = self._df.unique_id.max() + 1
        return self

    def add_record(self, record: dict):
        for r in self.required_keys:
            assert r in record, "record is missing required key {}".format(r)
        # assign a unique id
        if "unique_id" not in record:
            record['unique_id'] = self.unique_id
            self.unique_id += 1
        # assign dir column to simplify looking at where it came frome
        self._df = self._df.append(record, ignore_index=True)

    def get_df(self):
        return self._df

    def __len__(self):
        return self._df.shape[0]

    def save_df(self, filename):
        ext = os.path.splitext(filename)[1]
        if ext == '.pck':
            # export a pandas dataframe
            pickle.dump(self._df, open(filename, 'wb'))
        elif ext == '.xlsx':
            cols = list(self._df.columns)
            cols.remove('label_img')
            cols.remove('tissue_img')
            cols.remove('vtk_img')
            cols.remove("cone_img")
            cols.remove("normal_img")
            # don't save image data to excel
            self._df.to_excel(filename, columns=cols)
        else:
            raise ValueError("filetype {} not among recognized options".format(ext))

    def save_images_original_size(self, output_dir, debug=False):
        """ save_images, but first resize to the original size by finding the original landmarks again """
        utilities.mkdirs(output_dir)
        utilities.mkdirs(os.path.join(output_dir, "labels"))
        if debug:
            utilities.mkdirs(os.path.join(output_dir, "plotted_landmarks"))
        bbox_lims = [0, 0, 0, 0]
        images = dict()
        original_distances = dict()
        new_distances = dict()
        for _, row in self._df.iterrows():
            save_name = '_'.join(['u{:03d}'.format(int(row.unique_id)),
                                  'model{:03d}'.format(int(row.model_id)),
                                  'r{:03d}'.format(int(row.rotation_id)),
                                  'v{:03d}'.format(int(row.vtk_id)),
                                  'i{:03d}'.format(int(row.img_df_num)),
                                  Path(row.dir).parts[-1] + '.png'])
            # New landmarks are the landmarks in the image. Old landmarks are the landmarks in the model
            new_landmarks = a4ch_image_landmarks(row.label_img)
            old_landmarks = row.landmarks

            # Project the old landmarks to the 2D plane of the image. First find the plane
            dx, dy, origin = row.rotation["dir_x"], row.rotation["dir_y"], row.rotation["origin"]
            normal = np.cross(dx, dy)
            normal /= np.linalg.norm(normal)
            dx /= np.linalg.norm(dx)
            dy = np.cross(dx, normal)  # original dy not guaranteed to be orthogonal to dx
            # Our plane is now defined by dx, dy and origin. Normal is redundant.
            # origin doesn't really matter since we later remove the shift component of the affine transform
            shifted_landmarks = [utilities.project3d_to_2d(o, dx, dy, origin, normal) for o in old_landmarks]
            if debug:
                import matplotlib.pyplot as plt
                # plt.imshow(row.label_img)
                for l, name, symbol, in zip(new_landmarks, ["mv", "tv", "apex"], ["o", "+", "*"]):
                    plt.plot(l[1], l[0], symbol, color="r", label=name)
                for l, symbol, in zip(shifted_landmarks, ["o", "+", "*"]):
                    plt.plot(l[1], l[0], symbol, color="b")
                plt.legend()
                plt.savefig(os.path.join(output_dir, "plotted_landmarks", save_name))
                plt.close()

            affine = utilities.solve_affine(shifted_landmarks, new_landmarks)
            # Get only the scale component
            scale = utilities.separate_affine(affine)

            # add a margin to the image. Otherwie the features will go outside the borders.
            padded_image = utilities.add_margin(row.label_img, 10, 134, 134, 10, color=Tags["label_background"])
            new_image = padded_image.transform((400, 400), Image.AFFINE, scale.ravel()[0:6], resample=Image.NEAREST,
                                               fillcolor=Tags["label_background"])

            post_landmarks = a4ch_image_landmarks(new_image)
            original_distances[save_name] = [np.linalg.norm(shifted_landmarks[p[0]] - shifted_landmarks[p[1]]) for p in
                                             [[0, 1], [0, 2], [1, 2]]]
            new_distances[save_name] = [np.linalg.norm(post_landmarks[p[0]] - post_landmarks[p[1]]) for p in
                                        [[0, 1], [0, 2], [1, 2]]]

            # find bounding box of important regions (global for all images)
            non_background = Image.fromarray((np.array(new_image) != Tags["label_background"]))
            bbox = non_background.getbbox()
            for i in range(4):
                if bbox_lims[i] == 0 or (i < 2 and bbox[i] < bbox_lims[i]) or (i >= 2 and bbox[i] > bbox_lims[i]):
                    bbox_lims[i] = bbox[i]
            images[save_name] = new_image
            if debug:
                image_operations.save_img(row.label_img,
                                          os.path.join(output_dir, "plotted_landmarks", "orig_" + save_name))
                image_operations.save_img(new_image, os.path.join(output_dir, "plotted_landmarks", "new_" + save_name))

        # compare the distance of the original vtk landmarks to the distances in the new_image.
        # the ratio between images should be the same
        orig0 = next(iter(original_distances.values()))
        new0 = next(iter(new_distances.values()))
        for o, n in zip(original_distances.values(), new_distances.values()):
            print(f"original: {np.array(o) / np.array(orig0)}")
            print(f"new: {np.array(n) / np.array(new0)}")

        # make it so image is not right at borders
        bbox_lims[0] = max(bbox_lims[0] - 10, 0)
        bbox_lims[1] = max(bbox_lims[1] - 10, 0)
        bbox_lims[2] = min(bbox_lims[2] + 10, 400)
        bbox_lims[3] = min(bbox_lims[3] + 10, 400)
        print(f"bbox_lims = {bbox_lims} ({bbox_lims[2] - bbox_lims[0]}, {bbox_lims[3] - bbox_lims[1]})=> (256, 256)")
        for save_name, im in images.items():
            im = im.crop(bbox_lims)
            # im = im.resize((256, 256), resample=Image.NEAREST)
            im = utilities.shift_to_center(im, Tags["label_background"])
            im, _ = image_operations.process_output_image(im, include_blood=True)
            image_operations.save_img(im, os.path.join(output_dir, "labels", save_name))

    def save_images(self, output_dir):
        utilities.mkdirs(output_dir)
        utilities.mkdirs(os.path.join(output_dir, 'pseudos'))
        utilities.mkdirs(os.path.join(output_dir, 'labels'))
        utilities.mkdirs(os.path.join(output_dir, 'normals'))
        utilities.mkdirs(os.path.join(output_dir, 'cones'))
        utilities.mkdirs(os.path.join(output_dir, "metas"))
        for _, row in self._df.iterrows():
            save_name = '_'.join(['u{:03d}'.format(int(row.unique_id)),
                                  'model{:03d}'.format(int(row.model_id)),
                                  'r{:03d}'.format(int(row.rotation_id)),
                                  'v{:03d}'.format(int(row.vtk_id)),
                                  'i{:03d}'.format(int(row.img_df_num)),
                                  Path(row.dir).parts[-1] + '.png'])
            tissue_save_name = os.path.join(output_dir, 'pseudos', save_name)
            label_save_name = os.path.join(output_dir, 'labels', save_name)
            norm_save_name = os.path.join(output_dir, 'normals', save_name)
            cone_save_name = os.path.join(output_dir, 'cones', save_name)
            meta_save_name = os.path.join(output_dir, "metas", save_name)
            label_img, label_vals = image_operations.process_output_image(row.label_img, include_blood=True)
            image_operations.save_img(row.tissue_img, tissue_save_name)
            image_operations.save_img(row.normal_img, norm_save_name)
            image_operations.save_img(label_img, label_save_name)
            image_operations.save_img(255. * np.array(row.cone_img), cone_save_name)
            json.dump(row.transform_params, open(meta_save_name, 'w'))
            # meta = row.to_dict()
            # meta = {k: v for k, v in meta.items() if all([i not in k for i in ["_img", "landmarks", "origin", "rotation", "cone_params"]])}
            # meta = Box(utilities.clean_dict_for_json(meta))
            # meta.to_json(meta_save_name)
        label_path = os.path.join(output_dir, "labels.json")
        Box(label_vals).to_json(label_path)

    def _show_img(self, img):
        """ helper function for debugging """
        import matplotlib.pyplot as plt
        plt.imshow(img)
        plt.show()


list(range(1,100))
print('''list(range(1,100)):this is the first reuse''' )
def testfunc(MyName):
    print('hello %s'% MyName)
testfunc('Mary')
def testfunc(FName,LName):
    print('hello %s %s'%(FName ,LName))
testfunc('Mary','Smith')
def saving(pocket_money,paper_route,spending):
    return pocket_money + paper_route -spending
print(saving(10,10,5))
import sys
def silly_age_joke1():
    print('How old are you ?')
    age = int(sys.stdin.readline())
    if age>= 10 and age<=13:
        print('what is 13 + 49 + 84? A headache')
    else:
        print('Huh?')
silly_age_joke1()


# Connect Four Game

import numpy as np

# Define global (static) variables
ROWCNT = 6
COLCNT = 7

def create_board():
    board = np.zeros((6,7))  # a matrix of all zeros
    return board

def drop_token(board, row, selection, token):
    board[row][selection] = token

def valid_location(board, selection):  # checking that top row of selected column is empty
    return board[5][selection] == 0

def next_open_row(board, selection):
    for r in range(ROWCNT):
        if board[r][selection] == 0:
            return r      # returns first case where the slot is empty

def print_board(board):       # Flip board over so tokens drop to the bottom
    print(np.flip(board, 0))  # Flipped on 0 of x-axes

def win(board, token):
    # Checks for horizontal win
    for c in range(COLCNT-3):
        for r in range(ROWCNT):
            if board[r][c] == token and board[r][c+1] == token and board[r][c+2] == token and board[r][c+3] == token:
                return True

    # Checks for vertical win
    for c in range(COLCNT):
        for r in range(ROWCNT-3):
            if board[r][c] == token and board[r+1][c] == token and board[r+2][c] == token and board[r+3][c] == token:
                return True

    # Checks for diagonal up win
    for c in range(COLCNT-3):
        for r in range(ROWCNT-3):
            if board[r][c] == token and board[r+1][c+1] == token and board[r+2][c+2] == \
                    token and board[r+3][c+3] == token:
                return True

    # Checks for diagonal down win
    for c in range(COLCNT-3):
        for r in range(3, ROWCNT):
            if board[r][c] == token and board[r-1][c+1] == token and board[r-2][c+2] == \
                    token and board[r-3][c+3] == token:
                return True

board = create_board()
print_board(board)
game_over = False
# Set turn count to 0
turn = 0

while not game_over:
    # Player 1 Input
    if turn == 0:
        selection = int(input("Player 1: Choose the column to drop your token (0-6):"))   # User enters column number

        if valid_location(board, selection):
            row = next_open_row(board, selection)
            drop_token(board, row, selection, 1)     # Puts a 1 on the board in the spot player 1 drops their token

            if win(board, 1):
                print("Player 1 WINS!!! Game Over")
                game_over = True

    # Player 2 Input
    else:
        selection = int(input("Player 2: Choose the column to drop your token (0-6):"))

        if valid_location(board, selection):
            row = next_open_row(board, selection)
            drop_token(board, row, selection, 2)

            if win(board, 2):
                print("Player 2 WINS!!!  Game Over")
                game_over = True

    print_board(board)

    # Increase turn count by 1
    turn += 1
    # Turns by odd and even count
    turn = turn % 2


# author:hetao
# contact: ht2005112@hotmail.com
# datetime:2020/5/29 下午4:53
# software: PyCharm

"""
文件说明：
"""

import os
import sys

parent_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
sys.path.append(parent_dir)
from pprint import pprint

from file_stream.source import KafkaReader
from file_stream.writer import RedisWriter
from confg import kafka_config, redis_config
import logging

TOPIC = os.environ.get('TOPIC', 'demo.crawled_interaction_crawler')

def kafka_test():
    reader = KafkaReader(kafka_config, TOPIC)
    for item in reader:
        pprint(item)
        _ = input()

def redis_test():
    r = RedisWriter(redis_config)
    cninfo_platrorm_name = 'cninfo'
    print(cninfo_platrorm_name)
    out = r.writer.get('interaction_{}_reply_ids'.format(cninfo_platrorm_name))
    pprint(out)

if __name__ == '__main__':
    logging.basicConfig(format="%(asctime)s %(name)s:%(levelname)s:%(message)s",
                        datefmt='%Y-%m-%d %H:%M:%S', level=logging.DEBUG)
    # kafka_test()
    redis_test()


import sys

str_input = input().split('|')
init_energy = 100
init_coins = 100
flag = False
for a in str_input:
    b = a.split('-')
    b1 = int(b[1])
    b2 = b[0]
    if flag:
        print(f'Closed! Cannot afford {b2}.')
        sys.exit()
    elif b2 == 'rest':
        if init_energy < 100:
            init_energy += b1
            print(f'You gained {b1} energy.')
            print(f'Current energy: {init_energy}.')
            continue
        else:
            init_energy = 100
            print(f'You gained {0} energy.\nCurrent energy: {init_energy}.')
            continue
    elif b2 == 'order':
        if init_energy >= 30:
            init_energy -= 30
            init_coins += b1
            print(f'You earned {b1} coins.')
            continue
        else:
            init_energy += 50
            print('You had to rest!')
            continue
    if init_coins >= 0 and init_coins >= b1:
        print(f'You bought {b2}.')
        init_coins -= b1
    else:
        flag = True
print(f'Day completed!\nCoins: {init_coins}\nEnergy: {init_energy}')

SoftUni/Python fundamentals/Lists Basics - Exercise/Bread Factory.py


import bathtub as bt
import shapely
import pickle
import GLIB
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
from sklearn.linear_model import LinearRegression
from sklearn.ensemble import RandomForestRegressor
from sklearn.tree import DecisionTreeRegressor
from sklearn import preprocessing
from sklearn import tree
import numpy as np
from sklearn.metrics import r2_score
import woa
import cdw
from tqdm import tqdm
import xarray as xr
import winds as wind

# Create GLIB

writeBedMach = False
writeGLIB = False
writePolygons = False
writeGL =False
createWOA = False
createClosestWOA = False

###############################################
if writeBedMach:
    print("Grabbing BedMachine Data")
    bedmach = bt.convert_bedmachine("data/BedMachine.nc",coarsenfact=1)
    with open("data/bedmach.pickle","wb") as f:
        pickle.dump(bedmach,f)

with open("data/bedmach.pickle","rb") as f:
    bedmach = pickle.load(f)

## Load these into memory just once for speed
bedvalues = bedmach.bed.values
icemask = bedmach.icemask_grounded_and_shelves.values
##################################################

##################################################
if writeGLIB:
    print("Calculating Glibs")
    GLIB = GLIB.generateGLIBs(bedvalues,icemask)

    with open("data/bedmachGLIB.pickle","wb") as f:
        pickle.dump(GLIB,f)

with open("data/bedmachGLIB.pickle","rb") as f:
    GLIB = pickle.load(f)

##################################################

##################################################

if writePolygons:
    bt.save_polygons()

with open("data/shelfpolygons.pickle","rb") as f:
    polygons = pickle.load(f)

################################################

################################################

if writeGL:
    physical, grid, depths, shelves, shelf_keys = bt.get_line_points(bedmach,polygons)
    with open("data/groundinglinepoints.pickle","wb") as f:
        pickle.dump([physical,grid,depths,shelves,shelf_keys],f)

with open("data/groundinglinepoints.pickle","rb") as f:
    physical,grid,depths,shelves,shelf_keys = pickle.load(f)

################################################

baths = []
for l in range(len(grid)):
    baths.append(GLIB[grid[l][0]][grid[l][1]])

################################################

################################################

if createWOA:
    sal,temp = woa.create_WOA(bedmach)
    with open("data/woawithbed.pickle","wb") as f:
        pickle.dump([sal,temp],f)

with open("data/woawithbed.pickle","rb") as f:
    sal,temp = pickle.load(f)

if createClosestWOA:
    closest_points = cdw.closest_WOA_points(grid,baths,bedmach,method="simple")
    with open("data/closest_points.pickle","wb") as f:
        pickle.dump(closest_points,f)
with open("data/closest_points.pickle","rb") as f:
    closest_points = pickle.load(f)

depths = []
glibs = []
for l in range(len(baths)):
    depths.append(bedvalues[grid[l][0],grid[l][1]])
    glibs.append(baths[l])
    #if np.isnan(baths[l]):
        #baths[l]=bedvalues[grid[l][0],grid[l][1]]

with open("data/MSDGLIB.pickle","rb") as f:
    glibs = pickle.load(f)

areas = bt.shelf_areas()

baths=glibs
#
glibheats =  cdw.tempFromClosestPoint(bedmach,grid,physical,baths,closest_points,sal,temp)
glibheats_by_shelf = bt.shelf_sort(shelf_keys,glibheats)

glheats =  cdw.tempFromClosestPoint(bedmach,grid,physical,depths,grid,sal,temp)
glheats_by_shelf = bt.shelf_sort(shelf_keys,glheats)
with open("data/shelf_massloss.pickle","wb") as f:
    pickle.dump([glibheats_by_shelf,glheats_by_shelf],f)
with open("data/shelf_massloss.pickle","rb") as f:
    hubheats_by_shelf,glheats_by_shelf = pickle.load(f)

physical = np.asarray(physical)
gldepths_by_shelf = bt.shelf_sort(shelf_keys,depths)
glibs_by_shelf = bt.shelf_sort(shelf_keys,glibs)
rignot_shelf_massloss,rignot_shelf_areas,sigma = cdw.extract_rignot_massloss("data/rignot2019.xlsx")

hubheats=[]
glheats = []
ys = []
zs = []
znews= []
melts = []

with open("data/MSDS.pickle","rb") as f:
    MSDS = pickle.load(f)

winds_by_shelf = winds.AMPS_wind(polygons,"data/AMPS_winds.mat",icemask)

fig, (ax1,ax2) = plt.subplots(1,2)
for k in hubheats_by_shelf.keys():
    if k in rignot_shelf_massloss:
        hubheat = np.nanmean(hubheats_by_shelf[k])
        glheat = np.nanmean(glheats_by_shelf[k])
        melt = rignot_shelf_massloss[k]/np.sqrt(areas[k])
        hubheats.append(hubheat)
        glheats.append(glheat)
        melts.append(melt)
        ax1.annotate(k,(hubheat,melt))
        ax2.annotate(k,(glheat,melt))

ax1.scatter(hubheats,melts)
ax2.scatter(glheats,melts)
plt.show()


import doctest
from hypothesis import given
import hypothesis.strategies as st

def parent(i): return (i - 1)//2
def lchild(p): return 2 * p + 1
def rchild(p): return 2 * (p + 1)

def siftdown(items, start, last):
    p = start
    while p <= last:
        l = lchild(p)
        r = rchild(p)
        max = p
        if l <= last and items[max] <= items[l]: max = l
        if r <= last and items[max] <= items[r]: max = r
        if max == p: break
        items[p], items[max] = items[max], items[p]
        p = max
    return items

def bheapsort(items):
    """
    >>> bheapsort([])
    []
    >>> bheapsort([1])
    [1]
    >>> bheapsort([1,2,3,4,5])
    [1, 2, 3, 4, 5]
    >>> bheapsort([4,5,3,1,2])
    [1, 2, 3, 4, 5]
    """
    last = len(items) - 1
    start = parent(last)
    for start in reversed(range(0,start+1)):
        items = siftdown(items, start, len(items)-1)
    for i in reversed(range(len(items)-1)):
        items[i+1], items[0] = items[0], items[i+1]
        items = siftdown(items, 0, i)
    return items

@given(st.lists(elements=st.integers()))
def test_bheapsort(x):
    assert bheapsort(x) == sorted(x)

if __name__ == '__main__':
    test_bheapsort()
    doctest.testmod()


import os

test_folder = '/home/gaohan/Project/darknet/scripts/VOCdevkit/VOC2012/gesture_test_from_home'
save_folder = '/home/gaohan/Project/darknet/scripts/VOCdevkit/VOC2012/after_change_name'

fileNames = os.listdir(test_folder)
for i in fileNames:
    num = i.split('.')[0]
    change_num = num.zfill(6)
    change_name = change_num + '.jpg'
    print(change_name)
    os.rename(test_folder + '/' + i, save_folder + '/' + change_name)


from bureaucrat.SmarterBureaucrat import SmarterBureaucrat # https://github.com/SengerM/bureaucrat
from pathlib import Path
import pandas
import plotly.graph_objects as go
import plotly.express as px
import numpy as np
from statsmodels.distributions.empirical_distribution import ECDF
from scipy.interpolate import InterpolatedUnivariateSpline
from scipy.misc import derivative
from clean_beta_scan import binned_fit_langauss
from landaupy import langauss, landau # https://github.com/SengerM/landaupy
from grafica.plotly_utils.utils import scatter_histogram # https://github.com/SengerM/grafica
import warnings
from huge_dataframe.SQLiteDataFrame import load_whole_dataframe # https://github.com/SengerM/huge_dataframe

def hex_to_rgba(h, alpha):
    return tuple([int(h.lstrip('#')[i:i+2], 16) for i in (0, 2, 4)] + [alpha])

def script_core(directory: Path):
	John = SmarterBureaucrat(
		directory,
		_locals = locals(),
	)

	John.check_required_scripts_were_run_before('beta_scan.py')

	try:
		measured_data_df = pandas.read_feather(John.path_to_output_directory_of_script_named('beta_scan.py')/Path('measured_data.fd'))
	except FileNotFoundError:
		pass
	try:
		measured_data_df = pandas.read_csv(John.path_to_output_directory_of_script_named('beta_scan.py')/Path('measured_data.csv'))
	except FileNotFoundError:
		pass
	try:
		measured_data_df = load_whole_dataframe(John.path_to_output_directory_of_script_named('beta_scan.py')/Path('measured_data.sqlite'))
		measured_data_df.reset_index(inplace=True)
	except FileNotFoundError:
		raise FileNotFoundError(f'Cannot find "measured data file" for measurement {John.path_to_measurement_base_directory}.')

	for col in {'n_trigger'}:
		measured_data_df[col] = measured_data_df[col].astype(int)

	with John.do_your_magic():
		interesting_columns = set()
		for column in measured_data_df:
			if column in {'n_trigger','When','device_name'}:
				continue
			interesting_columns.add(column)
			fig = px.histogram(
				measured_data_df,
				x = column,
				color = 'device_name',
				barmode = 'overlay',
				opacity = .75,
				title = f'{column}
Measurement: {John.measurement_name}',
				marginal = 'rug',
			)
			fig.write_html(
				str(John.path_to_default_output_directory/Path(f'{column} histogram.html')),
				include_plotlyjs = 'cdn',
			)
			fig = px.ecdf(
				measured_data_df,
				x = column,
				color = 'device_name',
				title = f'{column}
Measurement: {John.measurement_name}',
				marginal = 'rug',
			)
			fig.write_html(
				str(John.path_to_default_output_directory/Path(f'{column} ECDF.html')),
				include_plotlyjs = 'cdn',
			)

		columns_for_scatter_matrix_plot = interesting_columns
		columns_for_scatter_matrix_plot -= {f't_{i*10} (s)' for i in [1,2,3,4,6,7,8,9]}
		columns_for_scatter_matrix_plot -= {f'Time over {i*10}% (s)' for i in [1,3,4,5,6,7,8,9]}
		columns_for_scatter_matrix_plot = columns_for_scatter_matrix_plot - {'Humidity (%RH)','Temperature (°C)','Bias voltage (V)','Bias current (A)'}
		df = measured_data_df
		fig = px.scatter_matrix(
			df,
			dimensions = sorted(columns_for_scatter_matrix_plot),
			title = f'Scatter matrix plot
Measurement: {John.measurement_name}',
			color = 'device_name',
			hover_data = ['n_trigger'],
		)
		fig.update_traces(diagonal_visible=False, showupperhalf=False)
		for k in range(len(fig.data)):
			fig.data[k].update(
				selected = dict(
					marker = dict(
						opacity = 1,
						color = 'black',
					)
				),
				# ~ unselected = dict(
					# ~ marker = dict(
						# ~ opacity = 0.01
					# ~ )
				# ~ ),
			)
		fig.write_html(
			str(John.path_to_default_output_directory/Path('scatter matrix')) + '.html',
			include_plotlyjs = 'cdn',
		)

		# Fit a Landau to the collected charge ---
		try:
			for column in measured_data_df.columns:
				if 'collected charge' not in column.lower():
					continue
				fig = go.Figure()
				fig.update_layout(
					title = f'Langauss fit
Measurement: {John.measurement_name}',
					xaxis_title = column,
					yaxis_title = 'Probability density',
				)
				colors = iter(px.colors.qualitative.Plotly)
				for device_name in sorted(set(measured_data_df['device_name'])):
					samples_for_langauss_fit = measured_data_df.query(f'device_name=={repr(device_name)}')[column].dropna()
					popt, _, hist, bin_centers = binned_fit_langauss(samples_for_langauss_fit)
					this_channel_color = next(colors)
					fig.add_trace(
						scatter_histogram(
							samples = samples_for_langauss_fit,
							bins = list(bin_centers - np.diff(bin_centers)[0]) + [bin_centers[-1]+np.diff(bin_centers)[-1]],
							name = f'Data {device_name}',
							error_y = dict(type='auto'),
							legendgroup = device_name,
							density = False,
							line = dict(color = this_channel_color),
						)
					)
					x_axis = np.linspace(min(bin_centers),max(bin_centers),999)
					fig.add_trace(
						go.Scatter(
							x = x_axis,
							y = langauss.pdf(x_axis, *popt)*len(samples_for_langauss_fit)*np.diff(bin_centers)[0],
							name = f'Langauss fit {device_name}
xMPV={popt[0]:.2e}
ξ={popt[1]:.2e}
σ={popt[2]:.2e}',
							line = dict(color = this_channel_color, dash='dash'),
							legendgroup = device_name,
						)
					)
					fig.add_trace(
						go.Scatter(
							x = x_axis,
							y = landau.pdf(x_axis, popt[0], popt[1])*len(samples_for_langauss_fit)*np.diff(bin_centers)[0],
							name = f'Landau component {device_name}',
							line = dict(color = f'rgba{hex_to_rgba(this_channel_color, .4)}', dash='dashdot'),
							legendgroup = device_name,
						)
					)
				fig.write_html(
					str(John.path_to_default_output_directory/Path(f'{column} langauss fit.html')),
					include_plotlyjs = 'cdn',
				)
		except Exception as e:
			warnings.warn(f'Cannot plot Langauss fit to data, reason: {repr(e)}')
		# ~ # Plot waveforms ---
		# ~ try:
			# ~ waveforms_df = pandas.read_feather(John.processed_by_script_dir_path('beta_scan.py')/Path('waveforms.fd'))
		# ~ except FileNotFoundError:
			# ~ waveforms_df = pandas.read_csv(John.processed_by_script_dir_path('beta_scan.py')/Path('waveforms.csv'))

		# ~ fig = px.line(
			# ~ waveforms_df,
			# ~ x = 'Time (s)',
			# ~ y = 'Amplitude (V)',
			# ~ facet_row = 'device_name',
			# ~ line_group = 'n_trigger',
			# ~ render_mode = 'webgl', # https://plotly.com/python/webgl-vs-svg/
		# ~ )
		# ~ fig.write_html(
			# ~ str(John.processed_data_dir_path/Path('waveforms.html')),
			# ~ include_plotlyjs = 'cdn',
		# ~ )

########################################################################

if __name__ == '__main__':
	import argparse

	parser = argparse.ArgumentParser(description='Makes plots with the distributions of the quantities parsed by the script "parse_raw_data_of_single_beta_scan.py".')
	parser.add_argument('--dir',
		metavar = 'path',
		help = 'Path to the base measurement directory.',
		required = True,
		dest = 'directory',
		type = str,
	)

	args = parser.parse_args()
	script_core(Path(args.directory))


#!/usr/bin/env python
from requests.api import get
import asyncio
import websocket
import json
import time
from functools import partial
import logging
import os, sys
import threading

class coin:
    def __init__(self, name):
        self.ticker = name.upper()
        self.message_sequence_id = 0
        self.message_buffer = []
        self.received_orders = []
        self.orderBook = {'bids':{}, 'asks':{}}
        self.trades = {'bought':{}, 'sold':{}} # in respect to the aggresor / taker behaviour - e.g market order to buy would be a 'bought event'
        self.got_orderbook_snapshot = False
        self.last_trade = []
        self.retreive_orderbook_pause_event = threading.Event()

    def get_orderbook_snapshot(self):
        self.retreive_orderbook_pause_event.wait(0.5) # nonblocking pause for 0.5 secs
        url = "https://api.exchange.coinbase.com/products/"+ self.ticker +"/book?level=3"
        headers = {"Accept": "application/json"}
        response = get(url=url, headers=headers)
        if response.ok:
            logger.info(f'Succesfully retrieved orderbook for {self.ticker}')
            orderbook_response = response.json()
            self.message_sequence_id = orderbook_response['sequence']
            for orders in orderbook_response['bids']:
                self.orderBook['bids'].update({orders[2]:[float(orders[0]), float(orders[1])]})
            for orders in orderbook_response['asks']:
                self.orderBook['asks'].update({orders[2]:[float(orders[0]), float(orders[1])]})
            # new messages may come in while we are processing old ones so we cant just loop over the buffer and process them
            # we must take the first element of the buffer, process it, remove it and move on until no more are left
            logger.debug(f"Processing message buffer.  Length of message buffer: {len(self.message_buffer)}")
            while len(self.message_buffer) > 0:
                logger.debug(f'message buffer length: {len(self.message_buffer)} ')
                if self.message_buffer[0]['sequence'] <= self.message_sequence_id:
                    logger.debug(f'Popped message with sequence id:{self.message_buffer[0]["sequence"]} - snapshot sequence id: {self.message_sequence_id} ')
                    self.message_buffer.pop(0)
                else:
                    logger.debug(f'Processing order in buffer - msg sequence: {self.message_buffer[0]["sequence"]} - ordbook sequence: {self.message_sequence_id} ')
                    self.process_message(self.message_buffer[0], True)
                    logger.debug(f'Removing processed message.')
                    self.message_buffer.pop(0)
            logger.debug(f"Done processing message buffer. Verifying length of message buffer is 0 - {'True' if len(self.message_buffer) == 0 else 'False' }")
            self.got_orderbook_snapshot = True
        else:
            logger.error(f'Error retieving order book. Status code : {str(response.status_code)} -- Reason : {response.reason}')
        self.retreive_orderbook_pause_event.clear()

    def process_message(self, message, *buffer_message):
        if self.got_orderbook_snapshot is False and not buffer_message: # need this buffer message check or else well just keep adding messages we intend to proces to the end of buffer
            logger.debug(f'appending with sequence_id {message["sequence"]} to buffer')
            self.message_buffer.append(message)
        else:
            # check messages are coming in correctly
            if message['sequence'] != self.message_sequence_id +   1:
                logger.error(f'message out of order - last sequence # applied: {self.message_sequence_id} - current message sequence #: {message["sequence"]}')
                return
            # ✔ confirmed received order - goes into matching engine
            if message['type'] == 'received':
                # !!! good location to track market orders (aggressive taker behavuior) vs limit orders
                self.received_orders.append(message)
                logger.debug(f'new order {message["order_id"]} being placed into "received but not yet open (RNYO)" state')
            # ✔  order moves from incoming order to orderbook  - e.g. whatever wasnt filled comes out of amtching engine onto orderbook
            elif message['type'] == 'open':
                logger.debug(f'Moving order# {message["order_id"]} from received state onto orderbook')
                # remove from "received but not yet open (RNYO)" state
                try:
                    order_location = [msg['order_id'] for msg in self.received_orders].index(message['order_id'])
                    self.received_orders.pop(order_location)
                except:
                    logger.warn(f'Couldnt find/remove order# {message["order_id"]} from "received but not yet open (RNYO)" state')
                # add to orderbook
                self.orderBook['asks' if message['side'] == 'sell' else 'bids'][message['order_id']] = [float(message['price']) , float(message['remaining_size'])]
            # ✔ order is no longer on the order book or in the "received but not yet open (RNYO)" state
            elif message['type'] == 'done':
                logger.debug(f'Order {message["order_id"]} finished reason: {message["reason"]} - removing from "received but not yet open (RNYO)" state and/or orderbook')
                try:
                    order_location = [msg['order_id'] for msg in self.received_orders].index(message['order_id'])
                    self.received_orders.pop(order_location)
                    logger.debug(f'{message["order_id"]} removed from "received but not yet open (RNYO)" state ')
                except:
                    if message['order_id'] in  self.orderBook['asks' if message['side'] == 'sell' else 'bids']:
                        del  self.orderBook['asks' if message['side'] == 'sell' else 'bids'][message['order_id']]
                        logger.debug(f'{message["order_id"]} removed from orderbook ')
                    else:
                        logger.warn(f'order was not found in orderbook or "received but not yet open (RNYO)" state')
                # !!!! need to log cancelled orders somehow for the machne learning model
                if message['reason'] == 'canceled':
                    pass
            # ✔ A trade occurred between two orders
            # this will only be used to log trades as the done messages should take care of maintaining orderbook
            # it can be used to create up to date candle sticks - sell this indicates the maker was a sell order and the match is considered an up-tick. A buy side match is a down-tick.
            elif message['type'] == 'match':
                # log the trade
                logger.debug(f'trade occured between {message["maker_order_id"]} and {message["taker_order_id"]} ')
                aggresor_side = 'bought' if message['side'] == 'sell' else 'sold'
                self.trades[aggresor_side][float(message['price'])] = message['size']
                self.last_trade = [message['price'], message['size']]
            # ✔ change an order - from what i understand only happens due to self trade prevention but can also happen to order on orderbook although not sure how this is accomplished
            elif message['type'] == 'change':
                try:
                    order_location = [msg['order_id'] for msg in self.received_orders].index(message['order_id'])
                    if self.received_orders[order_location]['size'] != message['old_size']:
                        logger.warn(f'order sizes for change message for {message["order_id"]} do not match up; original size:{self.received_orders[order_location]["size"]} - message "old size":{ message["old_size"]}')
                    elif self.received_orders[order_location]['price'] != message['price']:
                        logger.warn(f'order prices for change message for {message["order_id"]} do not match up; original price:{self.received_orders[order_location]["price"]} - message price:{ message["price"]}')
                    else:
                        self.received_orders[order_location]['size'] = message['new_size']
                        logger.debug(f'Changed order {message["order_id"]} size from {message["old_size"]} to {message["new_size"]}')
                except:
                    logger.debug(f"change order was not found in \"received but not yet open (RNYO)\" state, checking orderbook ")
                    if message['order_id'] in self.orderBook['asks' if message['side'] == 'sell' else 'bids']:
                        if self.orderBook['asks' if message['side'] == 'sell' else 'bids'][message['order_id']] == message['old_size']:
                            logger.debug(f'Updating orderbook order {message["order_id"]} size from {message["old_size"]} to {message["new_size"]}')
                            self.orderBook['asks' if message['side'] == 'sell' else 'bids'][message['order_id']] = {float(message['price']) : float(message['new_size'])}
                        else:
                            logger.error(f'Orderbook change message old_size {message["old_size"]} does not match with what is in our orderbook {self.orderBook["asks" if message["side"] == "sell" else "bids"][message["order_id"]]}')
                    else:
                        logger.warn(f"Order refrenced for change ({message['order_id']}) does not exist in either orderbook or \"received but not yet open (RNYO)\" state")
            # a stop order is triggered and another marekt/limit order takes its place
            elif message['type'] == 'activate':
                # !!!!!!!!!!! neeed to implement this  for the machne learning model
                logger.debug(f'received an activate order message - someones stop loss was triggered')
            else:
                logger.error(f'incoming message not handled correctly.  Message: {message}')
            # but not matter the message we have to keep track to make sure we didnt miss one
            self.message_sequence_id = message['sequence']
        # received
        # open
        # done
        # match
        # change
        # activate
        # if len(self.message_buffer)== 0: self.livelog()

    def livelog(self):
        UP = "\x1B[3A"
        CLR = "\x1B[0K"
        best_bid = max([ord[0] for ord in (self.orderBook['bids'][ord_id] for ord_id in self.orderBook['bids'])])
        best_ask = min([ord[0] for ord in (self.orderBook['asks'][ord_id] for ord_id in self.orderBook['asks'])])
        print('\n\n')
        print(f'{UP*2}Recieved orders: {len(self.received_orders)}{CLR}\n{[msg["order_id"] for msg in self.received_orders]}{CLR}\nBest bid:{best_bid} Best ask:{best_ask} last trade:{self.last_trade if len(self.last_trade) == 2 else "no trades yet"}{CLR}')

def on_message(ws, message, securities):
    msg = json.loads(message)
    try:
        if 'product_id' in msg:
            securities[msg['product_id']].process_message(msg)
        elif msg['type'] == 'subscriptions':
            logger.info(f"Successfully subscribed to {msg['channels'][0]['product_ids']}")
        else:
            print(f"message seems to have unexpected schema.  Message: {message}\nparsed message: {msg}")
            logger.warn(f"message seems to have unexpected schema.  Message: {message}\nparsed message: {msg}")
    except Exception as e:
        print(f'Error: {e}\nMessage not being handled correctly: {message}\nparsed message: {msg}\ntype:{msg} product id? {"True" if "product_id" in msg else "False"}')
        logger.error(f'Error: {e}\nMessage not being handled correctly: {message}\nparsed message: {msg}\ntype:{msg} product id? {"True" if "product_id" in msg else "False"}')

def on_error(ws, error):
    logger.error(f"Error occured: {error}")

def on_close(ws, close_status_code, close_msg):
    logger.info(f"Websocket connection closed - close_status_code : {close_status_code} close_msg: {close_msg} ")

def on_open(ws, securities):
    logger.info("Opened websocket connection")
    products = []
    for security in securities.values():
        products.append(security.ticker)
        retreive_orderbook_snapshot = threading.Thread(target=security.get_orderbook_snapshot, name=('get_'+security.ticker+'_orderbook_thread'), daemon=True)
        retreive_orderbook_snapshot.start()
    subscription_request = {'type':'subscribe', 'product_ids':products, 'channels':['full']}
    ws.send(json.dumps(subscription_request))

def main():
    #-----------------------------------------------------------------------------------------------------------------------
    # websocket.enableTrace(True)
    # wss://ws-feed.exchange.coinbase.com
    # coinbase is different than binance - they require a connection and then a subscribe message
    # they also do it differently (were going to use l2 channel - https://docs.cloud.coinbase.com/exchange/docs/websocket-channels#the-level2-channel )
    # we subscribe
    # they send a message with the current orderbook
    # subsequent messages are updates to the orderbook
    # ALSO: going to need to utilize full/heartbeat channel to ensure were getting messages in order and none are missed
    # ws = websocket.WebSocketApp("wss://api.gemini.com/v1/marketdata/BTCUSD", on_open=on_open, on_message=on_message, on_error=on_error, on_close=on_close)
    # ws.run_forever()
    #-----------------------------------------------------------------------------------------------------------------------
    securities = {'BTC-USD': coin("BTC-USD")}
    # for security in securities:
    #     securities[security].get_orderbook_snapshot()

    ws = websocket.WebSocketApp("wss://ws-feed.exchange.coinbase.com", on_open=partial(on_open, securities=securities), on_message=partial(on_message, securities=securities), on_error=on_error, on_close=on_close)
    # print(f'2 - thread count: {threading.active_count()}\nActive threads: {threading.enumerate()}')
    listen_for_msg_thread = threading.Thread(target = ws.run_forever, name='WebSocketAppThread', daemon=True)
    listen_for_msg_thread.start()
    # print(f'3 - thread count: {threading.active_count()}\nActive threads: {threading.enumerate()}')
    # pause until key is pressed
    try:
        # print(f'4 - thread count: {threading.active_count()}\nActive threads: {threading.enumerate()}')
        input("\n\n\nPress enter to continue\n")
    except SyntaxError:
        pass

if __name__ == "__main__":
    if os.path.exists("vars.env") and os.getenv('USE_ENV_FILE','False') == 'True':
        from dotenv import load_dotenv
        load_dotenv()
    #websocket.enableTrace(True)
    log_level = os.getenv('LOGLEVEL','Debug').upper()    # highest level of information that we should log
    logger = logging.getLogger()
    # logger.setLevel(logging.DEBUG)
    logger.setLevel(logging.getLevelName(log_level))
    stdout_handler = logging.StreamHandler(sys.stdout)
    # stdout_handler.setLevel(logging.DEBUG)
    stdout_handler.setLevel(logging.getLevelName(log_level))
    stdout_handler.setFormatter(logging.Formatter('%(asctime)s : %(levelname)s : %(message)s'))
    # logger.addHandler(stdout_handler)
    file_handler = logging.FileHandler('coinbase_log')
    file_handler.setLevel(logging.getLevelName(logging.DEBUG))
    file_handler.setFormatter(logging.Formatter('%(asctime)s : %(levelname)s : %(message)s'))
    logger.addHandler(file_handler)
    main()

# // Request
# // Subscribe to ETH-USD and ETH-EUR with the level2, heartbeat and ticker channels,
# // plus receive the ticker entries for ETH-BTC and ETH-USD
# {
#     "type": "subscribe",
#     "product_ids": [ "ETH-USD", "ETH-EUR"],
#     "channels": [ "level2", "heartbeat", { "name": "ticker", "product_ids": [ "ETH-BTC", "ETH-USD" ] }]
# }


# Usb based datalogging script, detach bluetooth pins to use
import serial,time,curses,pprint
sensors = serial.Serial(port='/dev/ttyACM0',baudrate=115200,timeout=1)

time.sleep(2)
gyro = [0,0,0]
acc = [0,0,0]
baro = [0,0]

def tc_eval():
    val = int(ord(sensors.read())<<8)+ord(sensors.read())
    if val >= (1<<15):
        val = val-(1<<16)
    return val

acc_scale = 9.81*2*2/(1<<12)
gyro_scale = 250.0*2/(1<<16)

f = open('datalog.csv', 'w')
intial_time = time.time();

while (True):
    sample_time = time.time() - intial_time
    f.write(str(sample_time))
    sensors.write('A')
    for i in range(3):
        acc[i] = tc_eval()*acc_scale
    f.write("," + str(acc[0]) + ", " + str(acc[1]) + "," + str(acc[2]))
    sensors.write('G')
    for i in range(3):
        gyro[i] = tc_eval()*gyro_scale
    f.write("," + str(gyro[0]) + ", " + str(gyro[1]) + "," + str(gyro[2]))
    sensors.write('B')
    baro[0] = float((ord(sensors.read())<<8)+ord(sensors.read()))+(float(ord(sensors.read()))/16)
    baro[1] = float(ord(sensors.read()))+(float(ord(sensors.read()))/16)
    f.write("," + str(baro[0]) + ", " + str(baro[1]) + "\n")


import os
import time

import portfolio_balance

import gym
import datetime

import matplotlib.pyplot as plt

import pickle

import random

# from baselines.ppo1 import pposgd_simple, mlp_policy
from baselines.ddpg import ddpg
import pandas as pd

from collections import deque

import numpy as np

from baselines.ddpg.ddpg_learner import DDPG
from baselines.ddpg.models import Actor, Critic
from baselines.ddpg.memory import Memory
from baselines.ddpg.noise import (
    AdaptiveParamNoiseSpec,
    NormalActionNoise,
    OrnsteinUhlenbeckActionNoise,
)
from baselines.common import set_global_seeds
import baselines.common.tf_util as U

STOCKS = open("stocklist.txt").read().strip().split()

NUM_STOCKS = 5
PRICE_HISTORY = 20

TRAIN_EPISODES = 50
TEST_EPISODES = 5

TRAIN_START_DATE = pd.to_datetime("2008-12-31").tz_localize("US/Eastern")
TRAIN_END_DATE = pd.to_datetime("2016-12-31").tz_localize("US/Eastern")

VALIDATE_START_DATE = pd.to_datetime("2016-12-31").tz_localize("US/Eastern")
VALIDATE_END_DATE = pd.to_datetime("2017-12-31").tz_localize("US/Eastern")

TEST_START_DATE = pd.to_datetime("2017-12-31").tz_localize("US/Eastern")
TEST_END_DATE = pd.to_datetime("2018-12-31").tz_localize("US/Eastern")

BUDGET = 1000

from portfolio_balance import observation_space, action_space

def learn(
    network,
    test=False,
    nb_epochs=None,  # with default settings, perform 1M steps total
    nb_epoch_cycles=20,
    nb_rollout_steps=100,
    reward_scale=1.0,
    noise_type="adaptive-param_0.2",
    normalize_returns=False,
    normalize_observations=True,
    critic_l2_reg=1e-2,
    actor_lr=1e-4,
    critic_lr=1e-3,
    popart=False,
    gamma=0.99,
    clip_norm=None,
    nb_train_steps=50,  # per epoch cycle and MPI worker,
    nb_eval_steps=100,
    batch_size=64,  # per MPI worker
    tau=0.01,
    param_noise_adaption_interval=50,
    **network_kwargs,
):
    nb_actions = action_space(NUM_STOCKS).shape[-1]
    assert (
        np.abs(action_space(NUM_STOCKS).low) == action_space(NUM_STOCKS).high
    ).all()  # we assume symmetric actions.

    memory = Memory(
        limit=int(1e6),
        action_shape=action_space(NUM_STOCKS).shape,
        observation_shape=observation_space(NUM_STOCKS, PRICE_HISTORY).shape,
    )
    critic = Critic(network=network, **network_kwargs)
    actor = Actor(nb_actions, network=network, **network_kwargs)

    param_noise = AdaptiveParamNoiseSpec(
        initial_stddev=float(0.2), desired_action_stddev=float(0.2)
    )

    max_action = action_space(NUM_STOCKS).high
    print("scaling actions by {} before executing in env".format(max_action))

    agent = DDPG(
        actor,
        critic,
        memory,
        observation_space(NUM_STOCKS, PRICE_HISTORY).shape,
        action_space(NUM_STOCKS).shape,
        gamma=gamma,
        tau=tau,
        normalize_returns=normalize_returns,
        normalize_observations=normalize_observations,
        batch_size=batch_size,
        action_noise=None,
        param_noise=param_noise,
        critic_l2_reg=critic_l2_reg,
        actor_lr=actor_lr,
        critic_lr=critic_lr,
        enable_popart=popart,
        clip_norm=clip_norm,
        reward_scale=reward_scale,
    )
    print("Using agent with the following configuration:")
    print(str(agent.__dict__.items()))

    eval_episode_rewards_history = deque(maxlen=100)
    episode_rewards_history = deque(maxlen=100)
    sess = U.get_session()

    # Prepare everything.
    agent.initialize(sess)
    sess.graph.finalize()

    agent.reset()

    for episode in range(TRAIN_EPISODES):
        print(f"======= Train Episode {episode} =======")
        assets = random.sample(STOCKS, NUM_STOCKS)
        env = gym.make(
            "PortfolioBalance-v0",
            assets=assets,
            start_date=TRAIN_START_DATE,
            end_date=TRAIN_END_DATE,
            budget=BUDGET,
            price_history=PRICE_HISTORY,
        )

        obs = env.reset()
        done = False

        while not done:
            # Perform rollouts.
            for t_rollout in range(nb_rollout_steps):
                # Predict next action.
                action, q, _, _ = agent.step(
                    np.array([obs]), apply_noise=True, compute_Q=True
                )
                new_obs, r, done, info = env.step(max_action * action)

                # Book-keeping.
                agent.store_transition(
                    np.array([obs]),
                    np.array([action]),
                    np.array([r]),
                    np.array([new_obs]),
                    np.array([done]),
                )  # the batched data will be unrolled in memory.py's append.

                obs = new_obs

                if done:
                    break

            agent.reset()

            # Train.
            for t_train in range(nb_train_steps):
                # Adapt param noise, if necessary.
                if (
                    memory.nb_entries >= batch_size
                    and t_train % param_noise_adaption_interval == 0
                ):
                    agent.adapt_param_noise()

                cl, al = agent.train()
                agent.update_target_net()

    # Test/Validate Environment
    portfolio, sortino, sharpe = [], [], []
    for episode in range(TEST_EPISODES):
        print(f"======= {'Test' if test else 'Validate'} Episode =======")
        assets = random.sample(STOCKS, NUM_STOCKS)
        env = gym.make(
            "PortfolioBalance-v0",
            assets=assets,
            start_date=TEST_START_DATE if test else VALIDATE_START_DATE,
            end_date=TEST_END_DATE if test else VALIDATE_END_DATE,
            budget=BUDGET,
            price_history=PRICE_HISTORY,
        )

        obs = env.reset()
        done = False

        while not done:
            action, q, _, _ = agent.step(np.array([obs]), apply_noise=True, compute_Q=True)
            obs, r, done, info = env.step(max_action * action)

        env.perf.portfolio_value.plot()

        portfolio.append(env.perf.portfolio_value[-1])
        sortino.append(env.perf.sortino[-1])
        sharpe.append(env.perf.sharpe[-1])

    print(f"Portfolio Value: {np.mean(portfolio)} +/- {np.std(portfolio)}")
    print(f"Sortino Ratio: {np.mean(sortino)} +/- {np.std(sortino)}")
    print(f"Sharpe Ratio: {np.mean(sharpe)} +/- {np.std(sharpe)}")

    plt.xlabel("Trading Day")
    plt.ylabel("Portfolio Value")

    plt.savefig("portfolio_return.png")

    return agent

if __name__ == "__main__":
    set_global_seeds(None)
    act = learn(network="mlp", test=False)


from django.db import models
from django_extensions.db.fields import AutoSlugField
from modelcluster.fields import ParentalKey
from wagtail.core.models import Page, Orderable
from wagtail.core.fields import RichTextField
from wagtail.admin.edit_handlers import FieldPanel, InlinePanel
from wagtail.images.edit_handlers import ImageChooserPanel
from longclaw.products.models import ProductVariantBase, ProductBase

class ProductIndex(Page):
    """Index page for all products
    """
    subpage_types = ('catalog.Product', 'catalog.ProductIndex')

class Product(ProductBase):
    parent_page_types = ['catalog.ProductIndex']
    description = RichTextField()
    content_panels = ProductBase.content_panels + [
        FieldPanel('description'),
        InlinePanel('images', label='Images'),
        InlinePanel('variants', label='Product variants'),

    ]

    @property
    def first_image(self):
        return self.images.first()

class ProductVariant(ProductVariantBase):
    """Represents a 'variant' of a product
    """
    # You *could* do away with the 'Product' concept entirely - e.g. if you only
    # want to support 1 'variant' per 'product'.
    product = ParentalKey(Product, related_name='variants')

    slug = AutoSlugField(
        separator='',
        populate_from=('product', 'ref'),
        )

    # Enter your custom product variant fields here
    # e.g. colour, size, stock and so on.
    # Remember, ProductVariantBase provides 'price', 'ref' and 'stock' fields
    description = RichTextField()

class ProductImage(Orderable):
    """Example of adding images related to a product model
    """
    product = ParentalKey(Product, related_name='images')
    image = models.ForeignKey('wagtailimages.Image', on_delete=models.CASCADE, related_name='+')
    caption = models.CharField(blank=True, max_length=255)

    panels = [
        ImageChooserPanel('image'),
        FieldPanel('caption')
    ]


import autograd.numpy as np
import autograd.scipy.misc as scpm

def kl_mvn_diag(m0, S0, m1, S1):
    """
    Kullback-Liebler divergence from Gaussian pm,pv to Gaussian qm,qv.
    Also computes KL divergence from a single Gaussian pm,pv to a set
    of Gaussians qm,qv.
    Diagonal covariances are assumed.  Divergence is expressed in nats.

    - accepts stacks of means, but only one S0 and S1

    From wikipedia
    KL( (m0, S0) || (m1, S1))
         = .5 * ( tr(S1^{-1} S0) + log |S1|/|S0| +
                  (m1 - m0)^T S1^{-1} (m1 - m0) - N )
    """
    # store inv diag covariance of S1 and diff between means
    N = m0.shape[1]
    iS1 = 1./S1
    diff = m1 - m0

    # kl is made of three terms
    tr_term   = np.sum(iS1 * S0)
    det_term  = np.sum(np.log(S1)) - np.sum(np.log(S0))
    quad_term = np.sum( (diff*diff) * iS1, axis=1)
    return .5 * (tr_term + det_term + quad_term - N)

def mog_logprob(x, means, icovs, lndets, pis):
    """ compute the log likelihood according to a mixture of gaussians
        with means  = [mu0, mu1, ... muk]
             icovs  = [C0^-1, ..., CK^-1]
             lndets = ln [|C0|, ..., |CK|]
             pis    = [pi1, ..., piK] (sum to 1)
        at locations given by x = [x1, ..., xN]
    """
    xx = np.atleast_2d(x)
    D  = xx.shape[1]
    centered = xx[:,:,np.newaxis] - means.T[np.newaxis,:,:]
    solved   = np.einsum('ijk,lji->lki', icovs, centered)
    logprobs = - 0.5*np.sum(solved * centered, axis=1) - (D/2.)*np.log(2*np.pi) \
               - 0.5*lndets + np.log(pis)
    logprob  = scpm.logsumexp(logprobs, axis=1)
    if np.isscalar(x) or len(x.shape) == 1:
        return logprob[0]
    else:
        return logprob

class WeightsParser(object):
    """A helper class to index into a parameter vector."""
    def __init__(self):
        self.idxs_and_shapes = {}
        self.num_weights = 0

    def add_shape(self, name, shape):
        start = self.num_weights
        self.num_weights += np.prod(shape)
        self.idxs_and_shapes[name] = (slice(start, self.num_weights), shape)

    def get(self, vect, name):
        idxs, shape = self.idxs_and_shapes[name]
        return np.reshape(vect[idxs], shape)

    def set(self, vect, name, val):
        idxs, shape = self.idxs_and_shapes[name]
        vect[idxs] = val.ravel()
        return vect


# -*- coding: utf-8 -*-
# @Time: 2019/5/5
# @File: ShopIntervalDays

'''
  平均复购天数，店铺所有订单间隔的平均天数，
  如果只下单一次，则认为平均天数为0
'''

from conn_mongo import ConnectDB

class ShopIntervalDays(ConnectDB):

    def __init__(self, **kwargs):
        self.args = kwargs['args']
        super().__init__(args=self.args)
        self.record_db = self.conn[self.args['record_db']]

    def query(self):
        cursor = self.conn['core']['StoreTransaction'].aggregate(
            self.args['filter'],
            allowDiskUse=True
        )
        data = [x for x in cursor]
        cursor.close()
        return data

    def handle(self):
        data = self.query()
        new_data = []
        for item in data:
            new_item = {}
            new_item['_id'] = item['_id']
            item['date'].sort()     # 日期排序
            if len(item['date']) >= 2:
                total_days = (item['date'][-1] - item['date'][0]).days  # 最后下单日期减去第一次下单的日期
                new_item['IntervalDays'] = float('{:.2f}'.format(total_days / (item['num'] - 1)))  # 保留两位小数
            else:
                new_item['IntervalDays'] = float('0.0')
            new_data.append(new_item)

        collection_name = 'IntervalDays'
        self.record_db[collection_name].drop()
        self.record_db[collection_name].insert_many(new_data)

    def run(self):
        self.handle()

if __name__ == "__main__":
    filter_interval = [
        {
            '$group': {
                '_id': '$storeId',
                'num': {
                    '$sum': 1
                },
                'date': {
                    '$push': '$date'
                }
            }
        }
    ]
    args = {
        'host': '10.15.101.63',
        'port': 27027,
        'filter': filter_interval,
        'record_db': 'analysis_pre'
    }
    opt = ShopIntervalDays(args=args)
    opt.run()


from django.core.management.base import BaseCommand, no_translations
from django.conf import settings
from apps.accounts.models import User

from .createfakeusers import FAKE_MAIL

class Command(BaseCommand):
    help = "Deletes all fake users ending in '%s'" % FAKE_MAIL

    def handle(self, *args, **options):

        self.stdout.write(str(options))

        if settings.DEBUG:
            self.delete_all_fakes()
            return
        return ("Access forbidden: Not in debug mode.")

    def delete_all_fakes(self):
        qs = User.objects.filter(email__contains=FAKE_MAIL)

        n = qs.count()
        if n == 0:
            self.stdout.write(self.style.SUCCESS("No fake users detected."))
            return

        is_sure = input(
                'You are about to delete %s users with emails including "%s". '
                "Are you sure you want to delete them? [y/n]" % (n, FAKE_MAIL)
            )

        if is_sure != "y":
            self.stdout.write(self.style.WARNING("Users NOT deleted."))
            return

        qs.delete()
        self.stdout.write(self.style.SUCCESS("Successfully deleted %s fake users." % n))


from scipy.io import loadmat
import requests
import os
import gzip
import numpy as np
from numba import njit
from config import ROOT_DIR

def reconstruction_error(X, X_tild):
    """Return the reconstruction error score for an array of inputs

        Parameters
        ----------
        X : ndarray
            (number of samples, input_size)
        X_tild : ndarray
            (number of samples, input_size)

        Returns
        -------
        float
            reconstruction error
        """
    return np.linalg.norm(X - X_tild)

def cross_entropy(predictions, targets):
    """Compute the Cross Entropy

    Parameters
    ----------
    predictions : ndarray
        (n_samples, n_classes)
    targets : ndarray
        (n_samples, n_classes)

    Returns
    -------
    Float
        number
    """
    likelihood = targets * np.log(predictions)
    return -np.sum(likelihood) / predictions.shape[0]

@njit(fastmath=True, parallel=True)
def sigmoid(X, W, b):
    """Sigmoid function

    Parameters
    ----------
    X : [type]
        [description]
    W : [type]
        [description]
    b : [type]
        [description]

    Returns
    -------
    [type]
        [description]
    """
    return 1 / (1 + np.exp(-(np.dot(X.astype(np.float64), W) + b)))

def accuracy_score(predictions, targets):
    """Compute accuracy score for one hot encoded vectors

    Parameters
    ----------
    predictions : [type]
        [description]
    targets : [type]
        [description]

    Returns
    -------
    [type]
        [description]
    """
    if len(predictions.shape) > 1:
        predictions = np.argmax(predictions, axis=1)
    if len(targets.shape) > 1:
        targets = np.argmax(targets, axis=1)

    return np.sum(predictions == targets) * 1 / predictions.shape[0]

def load_alpha_digits():
    os.chdir(ROOT_DIR)

    if (os.path.isdir('data') == 0):
        os.mkdir('data')

    if (os.path.isdir('data/alpha_digits') == 0):
        os.mkdir('data/alpha_digits')

    if (os.path.isfile('data/alpha_digits/binaryalphadigs.mat') == 0):
        r = requests.get("https://cs.nyu.edu/~roweis/data/binaryalphadigs.mat")
        open('data/alpha_digits/binaryalphadigs.mat', 'wb').write(r.content)

    return loadmat('data/alpha_digits/binaryalphadigs.mat')

def load_mnist(train_data=True, test_data=False):
    """
    Get mnist data from the official website and
    load them in binary format.

    Parameters
    ----------
    train_data : bool
        Loads
        'train-images-idx3-ubyte.gz'
        'train-labels-idx1-ubyte.gz'
    test_data : bool
        Loads
        't10k-images-idx3-ubyte.gz'
        't10k-labels-idx1-ubyte.gz'

    Return
    ------
    tuple
    tuple[0] are images (train & test)
    tuple[1] are labels (train & test)

    """
    os.chdir(ROOT_DIR)
    RESOURCES = [
        'train-images-idx3-ubyte.gz', 'train-labels-idx1-ubyte.gz',
        't10k-images-idx3-ubyte.gz', 't10k-labels-idx1-ubyte.gz'
    ]

    if (os.path.isdir('data') == 0):
        os.mkdir('data')
    if (os.path.isdir('data/mnist') == 0):
        os.mkdir('data/mnist')
    for name in RESOURCES:
        if (os.path.isfile('data/mnist/' + name) == 0):
            url = 'https://github.com/HIPS/hypergrad/raw/master/data/mnist/' + name
            r = requests.get(url, allow_redirects=True)
            open('data/mnist/' + name, 'wb').write(r.content)

    return get_images(train_data, test_data), get_labels(train_data, test_data)

def get_images(train_data=True, test_data=False):

    to_return = []

    if train_data:
        with gzip.open('data/mnist/train-images-idx3-ubyte.gz', 'r') as f:
            # first 4 bytes is a magic number
            magic_number = int.from_bytes(f.read(4), 'big')
            # second 4 bytes is the number of images
            image_count = int.from_bytes(f.read(4), 'big')
            # third 4 bytes is the row count
            row_count = int.from_bytes(f.read(4), 'big')
            # fourth 4 bytes is the column count
            column_count = int.from_bytes(f.read(4), 'big')
            # rest is the image pixel data, each pixel is stored as an unsigned byte
            # pixel values are 0 to 255
            image_data = f.read()
            train_images = np.frombuffer(image_data, dtype=np.uint8)\
                .reshape((image_count, row_count, column_count))
            to_return.append(np.where(train_images > 127, 1, 0))

    if test_data:
        with gzip.open('data/mnist/t10k-images-idx3-ubyte.gz', 'r') as f:
            # first 4 bytes is a magic number
            magic_number = int.from_bytes(f.read(4), 'big')
            # second 4 bytes is the number of images
            image_count = int.from_bytes(f.read(4), 'big')
            # third 4 bytes is the row count
            row_count = int.from_bytes(f.read(4), 'big')
            # fourth 4 bytes is the column count
            column_count = int.from_bytes(f.read(4), 'big')
            # rest is the image pixel data, each pixel is stored as an unsigned byte
            # pixel values are 0 to 255
            image_data = f.read()
            test_images = np.frombuffer(image_data, dtype=np.uint8)\
                .reshape((image_count, row_count, column_count))
            to_return.append(np.where(test_images > 127, 1, 0))

    return to_return

def get_labels(train_data=True, test_data=False):

    to_return = []

    if train_data:
        with gzip.open('data/mnist/train-labels-idx1-ubyte.gz', 'r') as f:
            # first 4 bytes is a magic number
            magic_number = int.from_bytes(f.read(4), 'big')
            # second 4 bytes is the number of labels
            label_count = int.from_bytes(f.read(4), 'big')
            # rest is the label data, each label is stored as unsigned byte
            # label values are 0 to 9
            label_data = f.read()
            train_labels = np.frombuffer(label_data, dtype=np.uint8)
            to_return.append(train_labels)
    if test_data:
        with gzip.open('data/mnist/t10k-labels-idx1-ubyte.gz', 'r') as f:
            # first 4 bytes is a magic number
            magic_number = int.from_bytes(f.read(4), 'big')
            # second 4 bytes is the number of labels
            label_count = int.from_bytes(f.read(4), 'big')
            # rest is the label data, each label is stored as unsigned byte
            # label values are 0 to 9
            label_data = f.read()
            test_labels = np.frombuffer(label_data, dtype=np.uint8)
            to_return.append(test_labels)

    return to_return


import logging
import os
import os.path as osp
import pickle
from typing import Callable, List, Optional

import torch

from ..data import Data, Batch, Dataset
from ..data.sys import download_url, extract_zip
from ..utils import remove_self_loops_from_tensor

class CSL(Dataset):
    r"""CSL dataset.

    Args:
        root (string): Root directory where the dataset should be saved.
        transform (callable, optional): A function/transform that takes in an
            :obj:`torch_geometric.data.Data` object and returns a transformed
            version. The data object will be transformed before every access.
            (default: :obj:`None`)
        pre_transform (callable, optional): A function/transform that takes in
            an :obj:`torch_geometric.data.Data` object and returns a
            transformed version. The data object will be transformed before
            being saved to disk. (default: :obj:`None`)
        pre_filter (callable, optional): A function that takes in an
            :obj:`torch_geometric.data.Data` object and returns a boolean
            value, indicating whether the data object should be included in the
            final dataset. (default: :obj:`None`)

    Stats:
            * - CSL
              - 150
              - ~41.0
              - ~164.0
              - 0
              - 10
    """
    url = 'https://www.dropbox.com/s/rnbkp5ubgk82ocu/CSL.zip?dl=1'

    def __init__(self, root: str,
                 transform: Optional[Callable] = None,
                 pre_transform: Optional[Callable] = None,
                 pre_filter: Optional[Callable] = None):
        self.name = 'CSL'
        """
        It is recommended to perform 5-fold cross validation with stratified
        sampling for CSL dataset.
        """
        super().__init__(root, transform, pre_transform, pre_filter)
        self.data_batch = torch.load(self.processed_paths[0])
        self.indices = torch.arange(len(self.data_batch))

    @property
    def raw_dir(self) -> str:
        return osp.join(self.root, 'raw')

    @property
    def processed_dir(self) -> str:
        return osp.join(self.root, 'processed')

    @property
    def raw_file_names(self) -> List[str]:
        return [
            'graphs_Kary_Deterministic_Graphs.pkl',
            'y_Kary_Deterministic_Graphs.pt'
        ]

    @property
    def processed_file_names(self) -> List[str]:
        return ['data.pt']

    def download(self):
        path = download_url(self.url, self.raw_dir)
        extract_zip(path, self.raw_dir)
        os.unlink(path)

    def process(self):
        with open(self.raw_paths[0], 'rb') as f:
            adjs = pickle.load(f)

        ys = torch.load(self.raw_paths[1]).tolist()

        data_list = []
        for adj, y in zip(adjs, ys):
            row, col = torch.from_numpy(adj.row), torch.from_numpy(adj.col)
            edge_index = torch.stack([row, col], dim=0).to(torch.long)
            edge_index, _ = remove_self_loops_from_tensor(edge_index)
            data = Data(edge_index=edge_index, y=torch.tensor([y], dtype=torch.long))
            data.__set_tensor_attr__('n_nodes',
                                     torch.tensor([adj.shape[0]], dtype=torch.long),
                                     'graph_feature')
            if self.pre_filter is not None and not self.pre_filter(data):
                continue
            if self.pre_transform is not None:
                data = self.pre_transform(data)
            data_list.append(data)
        torch.save(Batch.from_data_list(data_list), self.processed_paths[0])

    def __repr__(self) -> str:
        return f'{self.name}({len(self)})'


import time
from datetime import datetime
import requests
from bs4 import BeautifulSoup
import pandas as pd
import json
from pprint import pprint
import sys

train_num = input('Enter your train number: ')

url = ('https://asm.transitdocs.com/api/trainDetail.php?year=2021&month=2&day=21&train='+ str(train_num))

#url = 'https://asm.transitdocs.com/api/trainDetail.php?year=2021&month=2&day=16&train=66'
r = requests.get(url)

try:
    traindata = r.json()
    stationdata = traindata['stations']
except:
    print ('Train number '+str(train_num) + ' is not in service at this time')
    sys.exit()

# Using next() + dictionary comprehension
# Find dictionary matching value in list
pvd_data = next((sub for sub in stationdata if sub['code'] == 'PVD'), None)

try:
    pvd_est_dep = pvd_data['est_dep']
except:
    print('Train number ' + str(train_num) + ' departed PVD at ' + str(pvd_data['act_dep']))
    sys.exit()
if pvd_est_dep[1] == '/':
    pvd_est_dep = '0'+ pvd_est_dep

pvd_sch_dep = pvd_data['sch_dep']

if pvd_sch_dep[1] == '/':
    pvd_sch_dep = '0'+ pvd_sch_dep

pvd_est_time=datetime.strptime(pvd_est_dep, '%m/%d/%Y %I:%M%p')
pvd_sch_time= datetime.strptime(pvd_sch_dep, '%m/%d/%Y %I:%M%p')

minute_delta = (pvd_est_time - pvd_sch_time).total_seconds() / 60.0

if pvd_est_time == pvd_sch_time:
    print('Train '+ str(train_num)+ ' is currently on schedule')
elif pvd_est_time > pvd_sch_time:
    print('Train '+ str(train_num)+ ' is currently ' +str(minute_delta) + ' minutes behind schedule')
elif pvd_est_time < pvd_sch_time:
    print('Train '+ str(train_num)+ ' is currently ' +str(minute_delta) + ' minutes ahead of schedule')

print('************')

print('Train '+ str(train_num)+ ' is scheduled to depart PVD for BBY at ' + str(pvd_sch_time))
print('Train '+ str(train_num)+ ' is estimated to depart PVD for BBY at ' + str(pvd_est_time))