from uuid import uuid4

from demo.data_loading.data_fetching import get_countries_data
from demo.data_loading.fixes import fix_alpha2_value, fix_alpha3_value, fix_string_value
from demo.server.config import get_pyorient_client

def load_countries_and_regions(countries_df):
    graph = get_pyorient_client()

    country_cls = graph.registry['Country']
    region_cls = graph.registry['Region']
    subarea_cls = graph.registry['GeographicArea_SubArea']

    area_name_and_type_to_vertex = dict()

    # Create all countries.
    for _, country_item in countries_df.iterrows():
        name = fix_string_value(country_item['CLDR display name'])
        uuid = str(uuid4())
        alpha2 = fix_alpha2_value(country_item['ISO3166-1-Alpha-2'])
        alpha3 = fix_alpha3_value(country_item['ISO3166-1-Alpha-3'])

        props = {
            'name': name,
            'uuid': uuid,
            'alpha2': alpha2,
            'alpha3': alpha3,
        }
        vertex = graph.create_vertex(country_cls, **props)
        area_name_and_type_to_vertex[(name, 'Country')] = vertex

    # Create all non-country regions.
    for _, country_item in countries_df.iterrows():
        for region_column in ('Intermediate Region Name', 'Sub-region Name', 'Region Name'):
            name = fix_string_value(country_item[region_column])
            if name is None or (name, 'Region') in area_name_and_type_to_vertex:
                # Don't create regions with no name, or regions that were already added.
                continue

            uuid = str(uuid4())

            props = {
                'name': name,
                'uuid': uuid,
            }
            vertex = graph.create_vertex(region_cls, **props)
            area_name_and_type_to_vertex[(name, 'Region')] = vertex

    # Create all relationships between countries/regions.
    created_edges = set()
    for _, country_item in countries_df.iterrows():
        hierarchy_order = (
            ('CLDR display name', 'Country'),
            ('Intermediate Region Name', 'Region'),
            ('Sub-region Name', 'Region'),
            ('Region Name', 'Region'),
        )
        regions_in_order = [
            (region_name, kind)
            for region_name, kind in (
                (fix_string_value(country_item[column_name]), kind)
                for column_name, kind in hierarchy_order
            )
            if region_name is not None
        ]

        for index, (parent_region_name, parent_region_kind) in enumerate(regions_in_order):
            if index == 0:
                continue

            child_region_name, child_region_kind = regions_in_order[index - 1]

            uniqueness_key = (
                parent_region_name,
                parent_region_kind,
                child_region_name,
                child_region_kind,
            )
            if uniqueness_key not in created_edges:
                graph.create_edge(
                    subarea_cls,
                    area_name_and_type_to_vertex[(parent_region_name, parent_region_kind)],
                    area_name_and_type_to_vertex[(child_region_name, child_region_kind)])
                created_edges.add(uniqueness_key)

    # Link all currently parent-less regions to the World region.
    all_region_names = set(area_name_and_type_to_vertex.keys())
    all_regions_with_parents = {
        (child_region_name, child_region_kind)
        for _, _, child_region_name, child_region_kind in created_edges
    }
    all_regions_without_parents = all_region_names - all_regions_with_parents

    world_vertex = graph.create_vertex(region_cls, name='World', uuid=str(uuid4()))
    for region_name, region_kind in all_regions_without_parents:
        graph.create_edge(
            subarea_cls,
            world_vertex,
            area_name_and_type_to_vertex[(region_name, region_kind)])

def orientdb_load_all():
    countries_df = get_countries_data()
    load_countries_and_regions(countries_df)

if __name__ == '__main__':
    orientdb_load_all()


"""
GenT2_Rulebase.py
Created 9/1/2022
"""
from juzzyPython.generalType2zSlices.system.GenT2Engine_Intersection import GenT2Engine_Intersection
from juzzyPython.generalType2zSlices.system.GenT2Engine_Union import GenT2Engine_Union
from juzzyPython.generalType2zSlices.system.GenT2_Rule import GenT2_Rule
from juzzyPython.intervalType2.system.IT2_Rulebase import IT2_Rulebase
from juzzyPython.generalType2zSlices.system.GenT2_Antecedent import GenT2_Antecedent

from typing import List, OrderedDict

from juzzyPython.testing.timeRecorder import timeDecorator
class GenT2_Rulebase():
    """
    Class GenT2_Rulebase
    Keeps track of rules and generates results

    Parameters:
        None

    Functions:
        addRule
        addRules
        getRules
        getFuzzyLogicType
        get_GenT2zEngine_Intersection
        get_GenT2zEngineUnion
        getOverallOutput
        evaluateGetCentroid
        evaluate
        getIT2Rulebases
        getRule
        changeRule
        removeRule
        getNumberOfRules
        containsRule
        getRulesWithAntecedents
        getImplicationMethod
        setImplicationMethod
        toString

    """

    def __init__(self) -> None:
        self.rules = []
        self.outputs = []
        self.DEBUG = False
        self.CENTEROFSETS = 0
        self.CENTROID = 1
        self.implicationMethod = 1
        self.PRODUCT = 0
        self.MINIMUM = 1
        self.gzEU = GenT2Engine_Union()
        self.gzEI = GenT2Engine_Intersection()

    def addRule(self,r: GenT2_Rule) -> None:
        """Add a new rule to the rule set"""
        self.rules.append(r)
        it = r.getConsequents()
        for i in it:
            o = i.getOutput()
            if not o in self.outputs:
                self.outputs.append(o)

    def addRules(self,r: List[GenT2_Rule]) -> None:
        """Add multiple new rules to the rule set"""
        for i in range(len(r)):
            self.addRule(i)

    def getRules(self) -> List[GenT2_Rule]:
        """Return all the rules in the set"""
        return self.rules

    def getRule(self,ruleNum: int) -> GenT2_Rule:
        """Return a specific rule"""
        return self.rules[ruleNum]

    def getNumberOfRules(self) -> int:
        """Get the number of rules in the set"""
        return len(self.rules)

    def getFuzzyLogicType(self) -> int:
        """Returns the type of fuzzy logic that is employed.
        return 0: type-1, 1: interval type-2, 2: zSlices based general type-2"""
        return 2

    def containsRule(self,rule: GenT2_Rule) -> bool:
        """Check if a rule in the ruleset"""
        return rule in self.rules

    def getGenT2zEngineIntersection(self) -> GenT2Engine_Intersection:
        """Return the intersection engine"""
        return self.gzEI

    def getGenT2zEngineUnion(self) -> GenT2Engine_Union:
        """Return the union engine"""
        return self.gzEU

    def removeRule(self,ruleNumber: int) -> None:
        """Remove a rule based on its index"""
        del self.rules[ruleNumber]

    def getImplicationMethod(self) -> str:
        """Return if the implication is product or minimum"""
        if self.implicationMethod == self.PRODUCT:
            return "product"
        else:
            return "minimum"

    def setImplicationMethod(self,implicationMethod: int) -> None:
        """Sets the implication method, where by implication, we mean the implementation
        of the AND logical connective between parts of the antecedent.
        The desired implication method is applied for all rules."""
        if implicationMethod == self.PRODUCT:
            self.implicationMethod = self.PRODUCT
        elif implicationMethod == self.MINIMUM:
            self.implicationMethod = self.MINIMUM
        else:
            raise Exception("Only product (0) and minimum (1) implication is currently supported.")

    def toString(self) -> str:
        """Convert the class to string"""
        s = "General Type-2 Fuzzy Logic System with "+str(self.getNumberOfRules())+" rules:\n"
        for i in range(self.getNumberOfRules()):
            s += str(self.rules[i].toString())+"\n"
        return s

    def getOverallOutput(self) -> dict:
        """Return the overall output of the rules"""
        returnValue = OrderedDict()
        for r in range(len(self.rules)):
            temp = self.rules[r].getRawOutput()
            for o in self.outputs:
                if r == 0:
                    returnValue[o] = temp[o]
                else:
                    returnValue[o] = self.gzEU.getUnion(returnValue.get(o),temp.get(o))
        return returnValue

    def evaluateGetCentroid(self,typeReductionType: int) -> dict:
        """Returns the output of the FLS after type-reduction, i.e. the centroid.
        param: typeReductionType
        return: A TreeMap where Output is used as key and the value is an Object[]
        where Object[0] is a Tuple[] (the centroids, one per zLevel) and Object[1] is a Double holding the associated yValues for the centroids. If not rule fired for the given input(s),
        then null is returned as an Object[]."""
        returnValue = OrderedDict()
        rbsIT2 = self.getIT2Rulebases()
        zValues = self.rules[0].getAntecedents()[0].getSet().getZValues()

        for i in range(len(rbsIT2)):
            temp = rbsIT2[i].evaluateGetCentroid(typeReductionType)
            for o in temp.keys():
                if i == 0:
                    returnValue[o] = [[],[]]
                returnValue[o][0].append(temp[o][0])
                returnValue[o][1].append(zValues[i])
        return returnValue

    def evaluate(self,typeReductionType: int) -> dict:
        """The current evaluate function is functional but inefficient. It creates an IT2
        version of all the rules in the rulebase and computes each IT2 rule separately...
        param typeReductionType: 0: Center Of Sets, 1: Centroid
        param discretizationLevel: The discretization level on the xAxis"""
        returnValue = OrderedDict()
        rbsIT2 = self.getIT2Rulebases()
        rawOutputValues = []

        for i in range(len(rbsIT2)):
            rawOutputValues.append(rbsIT2[i].evaluate(typeReductionType))
        zValues = self.rules[0].getAntecedents()[0].getSet().getZValues()

        for o in self.outputs:
            i=0
            numerator = 0.0
            denominator = 0.0

            for outputValue in rawOutputValues:
                numerator += outputValue[o] * zValues[i]
                denominator += zValues[i]
                i+= 1
            returnValue[o] = numerator/denominator
        return returnValue

    def getIT2Rulebases(self) -> List[IT2_Rulebase]:
        """Returns the whole zSlices based rulebase as a series of interval type-2
        rule bases (one per zLevel) which can then be computed in parallel.
        param typeReductionMethod: The type-reduction method to be used at the IT2 level
        0: Center Of Sets,  1: Centroid.
        param discretizationLevelXAxis: The number of discretizations to be used at the IT2 level."""
        rbs = [0] * self.rules[0].getAntecedents()[0].getSet().getNumberOfSlices()
        for i in range(len(rbs)):
            rbs[i] = IT2_Rulebase()
            for currentRule in range(self.getNumberOfRules()):
                rbs[i].addRule(self.rules[currentRule].getRuleasIT2Rules()[i])
            rbs[i].setImplicationMethod(self.implicationMethod)
        return rbs

    def getRulesWithAntecedents(self,antecedents: List[GenT2_Antecedent]) -> List[GenT2_Rule]:
        """ Returns all rules with a matching (i.e. equal) set of antecedents."""
        matches = []
        for i in range(len(self.rules)):
            if self.rules[i].getAntecedents()==antecedents:
                matches.append(self.rules[i])
        return matches


from ansible.module_utils.basic import AnsibleModule

from ansible_collections.ansibleguy.opnsense.plugins.module_utils.base.api import \
    Session
from ansible_collections.ansibleguy.opnsense.plugins.module_utils.base.cls import GeneralModule

class General(GeneralModule):
    CMDS = {
        'set': 'set',
        'search': 'get',
    }
    API_KEY_PATH = 'proxy.forward'
    API_KEY_PATH_REQ = API_KEY_PATH
    API_MOD = 'proxy'
    API_CONT = 'settings'
    API_CONT_REL = 'service'
    API_CMD_REL = 'reconfigure'
    FIELDS_CHANGE = [
        'interfaces', 'port', 'port_ssl', 'transparent', 'ssl_inspection',
        'ssl_inspection_sni_only', 'ssl_ca', 'ssl_exclude', 'ssl_cache_mb',
        'ssl_workers', 'allow_interface_subnets', 'snmp', 'port_snmp',
        'snmp_password', 'interfaces_ftp', 'port_ftp', 'transparent_ftp',
    ]
    FIELDS_ALL = FIELDS_CHANGE
    FIELDS_TRANSLATE = {
        'port_ssl': 'sslbumpport',
        'transparent': 'transparentMode',
        'ssl_inspection': 'sslbump',
        'ssl_inspection_sni_only': 'sslurlonly',
        'ssl_ca': 'sslcertificate',
        'ssl_exclude': 'sslnobumpsites',
        'ssl_cache_mb': 'ssl_crtd_storage_max_size',
        'ssl_workers': 'sslcrtd_children',
        'allow_interface_subnets': 'addACLforInterfaceSubnets',
        'snmp': 'snmp_enable',
        'port_snmp': 'snmp_port',
        'interfaces_ftp': 'ftpInterfaces',
        'port_ftp': 'ftpPort',
        'transparent_ftp': 'ftpTransparentMode',
    }
    FIELDS_TYPING = {
        'bool': [
            'transparent_ftp', 'snmp', 'allow_interface_subnets', 'ssl_inspection_sni_only',
            'ssl_inspection', 'transparent',
        ],
        'list': ['interfaces', 'ssl_exclude', 'interfaces_ftp'],
        'int': ['port', 'port_ssl', 'ssl_cache_mb', 'ssl_workers', 'port_snmp'],
        'select': ['ssl_ca'],
    }
    FIELDS_IGNORE = ['acl', 'icap', 'authentication']
    INT_VALIDATIONS = {
        'ssl_workers': {'min': 1, 'max': 32},
        'ssl_cache_mb': {'min': 1, 'max': 65535},
        'port': {'min': 1, 'max': 65535},
        'port_ssl': {'min': 1, 'max': 65535},
        'port_snmp': {'min': 1, 'max': 65535},
    }
    FIELDS_DIFF_EXCLUDE = ['snmp_password']

    def __init__(self, module: AnsibleModule, result: dict, session: Session = None):
        GeneralModule.__init__(self=self, m=module, r=result, s=session)


import qrcode as qr
from PIL import Image
q=qr.QRCode(version=1,
            error_correction=qr.constants.ERROR_CORRECT_H,
            box_size=10,
            border=4,)

q.add_data("https://youtu.be/NaQ_4ZvCbOE")
q.make(fit=True)
img= q.make_image(fill_color='darkblue', back_color='steelblue')
img.save("x.png")


import torch
import torch.optim as optim
from torch.utils.data import DataLoader
from torchvision import transforms
from torchvision.datasets import MNIST

from dae.dae import DAE
from beta_vae.beta_vae import BetaVAE
from history import History

# hyperparameters
num_epochs = 100
batch_size = 128
lr = 1e-4
beta = 4
save_iter = 20

shape = (28, 28)
n_obs = shape[0] * shape[1]

# create DAE and ß-VAE and their training history
dae = DAE(n_obs, num_epochs, batch_size, 1e-3, save_iter, shape)
beta_vae = BetaVAE(n_obs, num_epochs, batch_size, 1e-4, beta, save_iter, shape)
history = History()

# fill autoencoder training history with examples
print('Filling history...', end='', flush=True)

transformation = transforms.Compose([
    transforms.ColorJitter(),
    transforms.ToTensor()
])

dataset = MNIST('data', transform=transformation)
dataloader = DataLoader(dataset, batch_size=1, shuffle=True)

for data in dataloader:
    img, _ = data
    img = img.view(img.size(0), -1).numpy().tolist()
    history.store(img)
print('DONE')

# train DAE
dae.train(history)

# train ß-VAE
beta_vae.train(history, dae)


import pandas as pd
import numpy as np
import geopandas as gpd
from helper_functions import add_subset_address_cols, interpolate_polygon
from data_constants import default_crs, make_data_dict
from name_parsing import combine_names
from address_parsing import clean_parse_address
from helper_functions import make_panel
from pathos.multiprocessing import ProcessingPool as Pool
import re
import fiona

import warnings
warnings.filterwarnings("ignore", 'This pattern has match groups')

# function for reading in corrupted gdb files. really only relevant for LA CAMS data
def readShp_nrow(path, numRows):
    fiona_obj = fiona.open(str(path))
    toReturn = gpd.GeoDataFrame.from_features(fiona_obj[0:numRows])
    toReturn.crs = fiona_obj.crs
    return (toReturn)

# classify & clean name columns+ clean & parse primary and mailing addresses
# function that runs code in parallel
def parallelize_dataframe(df:pd.DataFrame, func, n_cores=4) -> pd.DataFrame:
    df_split = np.array_split(df, n_cores)
    pool = Pool(n_cores)
    df = pd.concat(pool.map(func, df_split))
    pool.close()
    pool.join()
    # have to include this to prevent leakage and allow multiple parallel function calls
    pool.terminate()
    pool.restart()
    return df

# wrapper function to run each city in parallel
def clean_parse_parallel(df:pd.DataFrame) -> pd.DataFrame:
    df = clean_parse_address(
        dataframe=df, address_col='address_fa',st_name="address_sn", st_sfx="address_ss",
        st_d="address_sd", unit='address_u', st_num='address_n1',
        country='address_country', state='address_state', st_num2 ='address_n2',city='address_city',
        zipcode='address_zip', prefix2='parsed_', prefix1='cleaned_'
    )
    return df

# ADDRESS CLEANING FUNCTIONS #
# takes an address df (geopandas or pandas), stanardizes and cleans it and returns a standardized pandas dataframe
# these functions get address dataframes to be in standardized formats (renamed columns, added variables, etc)
# such that the dataframe can be passed to clean_parse_parallel and exported
# see address cols in data constants for full list of necessary columns needed for clean_parse_parallel
# ill note if there is anything special with the function, but otherwise assume that it follows a standard flow of
# 1. rename columns -> add columns -> subset to only needed columns -> clean_parse_parrallel -> return

# chicago cleaning functions:
# chicago address files come in two seperate files that together represent a full set of addresses in cook county

# clean chi_add_points cleans a points file that represents centroid points for cook county parcel polygons
def clean_chi_add_points(df):
    chicago_rename_dict = {
        'ADDRNOCOM': 'address_n1',
        'STNAMEPRD': 'address_sd',
        'STNAME': 'address_sn',
        'STNAMEPOT': 'address_ss',
        'PLACENAME': 'address_city',
        'ZIP5': 'address_zip',
        'CMPADDABRV': 'address_fa',
        'PIN': 'parcelID',
        'XPOSITION': 'long',
        'YPOSITION': 'lat'
    }
    df.rename(columns=chicago_rename_dict, inplace=True)
    df = add_subset_address_cols(df)
    df = parallelize_dataframe(df=df, func=clean_parse_parallel, n_cores=2)
    return df

# basically the same as address points, but these are for parcel polygons (lat long are centroid points, so it is
# basically equivalent, these just have some addresses not in the other df and vice versa
def clean_chi_add_parcels(df):
    chicago_rename_dict = {
        'property_address':'address_fa',
        'property_city': 'address_city',
        'property_zip': 'address_zip',
        'pin': 'parcelID',
        'latitude': 'lat',
        'longitude': 'long'
    }
    df.rename(columns=chicago_rename_dict, inplace=True)
    df = add_subset_address_cols(df)
    df = parallelize_dataframe(df=df, func=clean_parse_parallel, n_cores=4)
    return df

def concat_chi_add(df1, df2):
    df1 = df1.append(df2).drop_duplicates(subset = [
        'parcelID',
        "parsed_addr_n1",
        "parsed_addr_sn",
        "parsed_addr_ss",
        "parsed_city"

    ])
    return df1

# saint louis is a little strange because they provide parcel polygons for entire streets
# eg main st 100-900. This is fine for small streets as its not problematic to take centroid polygons, but
# it becomes an issue for larger streets. For larger streets I take a best guess on which way the street runs and
# linearly interpolate lat long between the bottom and top range of the address span
# so if main st 100-900 runs nw that means it has its smallest numbers in the south east and increases going north west

def clean_stl_add(df):
    df = df.rename(
        columns = {
             "STREETNAME": "address_sn", "STREETTYPE": "address_ss", "PREDIR": "address_sd", "ZIP_CODE": "address_zip"
        }
    )
    df['index'] = np.arange(df.shape[0])
    df = df.to_crs(default_crs)
    df.crs = default_crs
    bounds = df.bounds
    df['address_city'] = 'saint louis'
    df['latitude_min'] = bounds["miny"]
    df['latitude_max'] = bounds["maxy"]
    df['longitude_min'] = bounds["minx"]
    df['longitude_max'] = bounds["maxx"]
    df['direction'] = np.where(
        ((df['FROMLEFT'] < df['TOLEFT']) & (df['FROMRIGHT'] < df['TORIGHT'])),
        "NE",
        np.where(
            ((df['FROMLEFT'] < df['TOLEFT']) & (df['FROMRIGHT'] > df['TORIGHT'])),
            "NW",
            np.where(
                ((df['FROMLEFT'] > df['TOLEFT']) & (df['FROMRIGHT'] < df['TORIGHT'])),
                "SE",
                np.where(
                    ((df['FROMLEFT'] > df['TOLEFT']) & (df['FROMRIGHT'] > df['TORIGHT'])),
                    "SW",
                    "SW"

                )
            )
        )
    )
    df_r = df[[col for col in df.columns if not bool(re.search("LEFT", col))]]
    df_r['address_n1'] = np.where(
        df_r['FROMRIGHT'] > df_r['TORIGHT'],
        df_r['TORIGHT'],
        df_r['FROMRIGHT']
    )
    df_r['address_n2'] = np.where(
        df_r['TORIGHT'] > df_r['FROMRIGHT'],
        df_r['TORIGHT'],
        df_r['FROMRIGHT']
    )
    df_l = df[[col for col in df.columns if not bool(re.search("RIGHT", col))]]
    df_l['address_n1'] = np.where(
        df_l['FROMLEFT'] > df_l['TOLEFT'],
        df_l['TOLEFT'],
        df_l['FROMLEFT']
    )
    df_l['address_n2'] = np.where(
        df_l['TOLEFT'] > df_l['FROMLEFT'],
        df_l['TOLEFT'],
        df_l['FROMLEFT']
    )
    df = pd.concat([df_r, df_l])
    df = df[~((df['address_n1'] <= 0) & (df['address_n1'] <= 0))]
    df = make_panel(df,start_year="address_n1", end_year="address_n2", current_year=df['address_n2'],
                    evens_and_odds=True ).rename(columns = {'year': 'address_n1'})
    # interpolate lat long

    df = interpolate_polygon(df, "index", "direction")
    df['lat'] = df['lat_interpolated']
    df['long'] = df["long_interpolated"]

    df = add_subset_address_cols(df)
    df = parallelize_dataframe(df=df, func=clean_parse_parallel, n_cores=2)
    return df

def clean_la_add(df):
    la_rename_dict = {
        'AIN': 'parcelID',
        'UnitName': 'address_u',
        'Number': 'address_n1',
        'PostType': 'address_ss',
        'PreDirAbbr':   'address_sd',
        'ZipCode': 'address_zip',
        'LegalComm': 'address_city',
    }
    df.rename(columns=la_rename_dict, inplace=True)
    combine_names(df, name_cols=['PreType', 'StArticle', 'StreetName'], newCol="address_sn")
    df = df.to_crs(default_crs)
    df.crs = default_crs
    df['long'] = df.geometry.centroid.x
    df['lat'] = df.geometry.centroid.y
    df = add_subset_address_cols(df)
    df = parallelize_dataframe(df=df, func=clean_parse_parallel, n_cores=2)
    return df

def clean_sd_add(df):
    sd_rename_dict = {
        'addrunit': 'address_u',
        'addrnmbr': 'address_n1',
        'addrpdir':'address_sd',
        'addrname': 'address_sn',
        'addrsfx': 'address_ss',
        'addrzip': 'address_zip',
        'community': 'address_city',
        'PIN': 'parcelID',
    }
    df.rename(columns=sd_rename_dict, inplace=True)
    df = df.to_crs(default_crs)
    df.crs = default_crs
    df['long'] = df.geometry.centroid.x
    df['lat'] = df.geometry.centroid.y
    df = add_subset_address_cols(df)
    df = parallelize_dataframe(df=df, func=clean_parse_parallel, n_cores=2)
    return df

def clean_sf_add(df):
    sf_rename_dict = {
        "Parcel Number": 'parcelID',
        'Unit Number': 'address_u',
        'Address Number': 'address_n1',
        'Street Name': 'address_sn',
        'Street Type':   'address_ss',
        'ZIP Code': 'address_zip',
        'Address': 'address_fa',
        #'PIN': 'parcelID',
        'Longitude': 'long',
        'Latitude': 'lat'
    }
    df.rename(columns=sf_rename_dict, inplace=True)
    df['address_city'] = "San Francisco"
    df = add_subset_address_cols(df)
    df = parallelize_dataframe(df=df, func=clean_parse_parallel, n_cores=2)
    return df

def clean_seattle_add(df):
    seattle_rename_dict = {
        'PIN': 'parcelID',
        'ADDR_NUM': 'address_n1',
        'ADDR_SN': 'address_sn',
        'ADDR_ST': 'address_ss',
        'ADDR_SD':   'address_sd',
        'ZIP5': 'address_zip',
        'CTYNAME': 'address_city',
        'ADDR_FULL': 'address_fa',
        'LON': 'long',
        'LAT': 'lat'
    }
    df.rename(columns=seattle_rename_dict, inplace=True)
    df = add_subset_address_cols(df)
    df = parallelize_dataframe(df=df, func=clean_parse_parallel, n_cores=2)
    return df

def clean_orlando_add(df):
    orlando_rename_dict = {
        'OFFICIAL_P': 'parcelID',
       "COMPLETE_A": 'address_fa',
        "ADDRESS__1": 'address_n1',
        "ADDRESS__2": "address_n2",
        "BASENAME": "address_sn",
        "POST_TYPE":"address_ss",
        "POST_DIREC":   "address_sd",
        "MUNICIPAL_": 'address_city',
        "ZIPCODE": "address_zip",
        "LATITUDE": "lat",
        "LONGITUDE": "long",
    }
    df.rename(columns=orlando_rename_dict, inplace=True)
    df = add_subset_address_cols(df)
    df = parallelize_dataframe(df=df, func=clean_parse_parallel, n_cores=2)
    return df

def clean_baton_rouge_add(df):
    baton_rouge_rename_dict = {
        'ADDRNOCOM': 'address_n1',
        'ASTREET PREFIX DIRECTION': 'address_sd',
        'STREET NAME': 'address_sn',
        'STREET SUFFIX TYPE': 'address_ss',
        'CITY': 'address_city',
        'ZIP': 'address_zip',
        'FULL ADDRESS': 'address_fa'
    }
    df.rename(columns=baton_rouge_rename_dict, inplace=True)
    lat_long = df['GEOLOCATION'].str.extract('([0-9\.]+),([0-9\.]+)')
    df['lat'] = lat_long.iloc[:,0]
    df['long'] = lat_long.iloc[:,1]
    df = add_subset_address_cols(df)
    df = parallelize_dataframe(df=df, func=clean_parse_parallel, n_cores=4)
    return df

def merge_sac_parcel_id(sac_add = pd.DataFrame,  xwalk = pd.DataFrame):
    return pd.merge(
        sac_add,
        xwalk[xwalk['Parcel_Number'].notna()][["Address_ID", "Parcel_Number"]].drop_duplicates(),
        left_on = "Address_ID", right_on = "Address_ID", how = "left"
        )

def clean_sac_add(df):
    sac_rename_dict = {
        'APN': 'parcelID',
        "Address_Number": 'address_n1',
        "Street_Name": "address_sn",
        "Street_Suffix":"address_ss",
        "Pre_Directiona;":   "address_sd",
        "Postal_City": 'address_city',
        "Zip_Code": "address_zip",
        "Latitude_Y": "lat",
        "Longitude_X": "long",
    }
    df.rename(columns=sac_rename_dict, inplace=True)
    df = add_subset_address_cols(df)
    df = parallelize_dataframe(df=df, func=clean_parse_parallel, n_cores=2)
    return df

# used to reclean data in the event that you dont want to read in a shapefile
# mostly uses because its faster to read in a csv than a shp
def clean_int_addresses(df):
    df = add_subset_address_cols(df)
    df = parallelize_dataframe(df=df, func=clean_parse_parallel, n_cores=2)
    return df

if __name__ == "__main__":
    print("hello")
    data_dict = make_data_dict(use_seagate=False)
    # stl_add = gpd.read_file(data_dict['raw']['stl']['parcel'] + 'streets/tgr_str_cl.shp')
    # stl_add = clean_stl_add(stl_add)
    # stl_add.to_csv(data_dict['intermediate']['stl']['parcel'] + 'addresses.csv', index=False)
    # baton_rouge_add = pd.read_csv(
    #     data_dict['raw']['baton_rouge']['parcel'] + 'addresses_Property_Information_ebrp.csv')
    # baton_rouge_add = clean_baton_rouge_add(baton_rouge_add)
    # baton_rouge_add.to_csv(data_dict['intermediate']['baton_rouge']['parcel'] + 'addresses.csv', index=False)
    # chicago_add1 = pd.read_csv(data_dict['raw']['chicago']['parcel'] + 'Cook_County_Assessor_s_Property_Locations.csv')
    # chicago_add2 = pd.read_csv(data_dict['raw']['chicago']['parcel'] + 'Address_Points_cook_county.csv')

    # orlando_add = gpd.read_file(data_dict['raw']['orlando']['parcel'] + "Address Points/ADDRESS_POINT.shp")
    # clean_orlando_add(orlando_add).to_csv(data_dict['intermediate']['orlando']['parcel'] + 'addresses.csv', index=False)

    # la_add = gpd.read_file("/Users/JoeFish/Desktop/la_addresspoints.gdb", nrows = 100)
    # la_add = pd.read_csv(data_dict['intermediate']['la']['parcel'] + 'addresses.csv')
    # file is corrupted so we have to read it in this way...
    # print(la_add.head())
    #sd_add = gpd.read_file(data_dict['raw']['sd']['parcel'] + 'addrapn_datasd_san_diego/addrapn_datasd.shp')
    # sf_add = pd.read_csv(
    #     data_dict['raw']['sf']['parcel'] + 'Addresses_with_Units_-_Enterprise_Addressing_System_san_francisco.csv')
    # seattle_add = gpd.read_file(data_dict['raw']['seattle']['parcel'] +
    #                           'Addresses_in_King_County___address_point/Addresses_in_King_County___address_point.shp')
    #
    # # clean_baton_rouge_add(baton_rouge_add).to_csv(data_dict['intermediate']['baton_rouge']['parcel'] + 'addresses.csv', index=False)
    # clean_chi_add2(chicago_add1).to_csv(data_dict['intermediate']['chicago']['parcel'] + 'addresses_from_parcels.csv', index=False)
    # clean_chi_add1(chicago_add2).to_csv(data_dict['intermediate']['chicago']['parcel'] + 'addresses_from_points.csv', index=False)
    # clean_int_addresses(la_add).to_csv(data_dict['intermediate']['la']['parcel'] + 'addresses_temp.csv', index=False)
    # clean_sf_add(sf_add).to_csv(data_dict['intermediate']['sf']['parcel'] + 'addresses.csv', index=False)
    # #clean_sd_add(sd_add).to_csv(data_dict['intermediate']['sd']['parcel'] + 'addresses.csv', index=False)
    # clean_seattle_add(seattle_add).to_csv(data_dict['intermediate']['seattle']['parcel'] + 'addresses.csv', index=False)
    # chi1 = pd.read_csv(data_dict['intermediate']['chicago']['parcel'] + 'addresses_from_parcels.csv', dtype={"parsed_addr_n1": str})
    # chi2 = pd.read_csv(data_dict['intermediate']['chicago']['parcel'] + 'addresses_from_points.csv', dtype={"parsed_addr_n1": str})
    # concat_chi_add(chi1,chi2).to_csv(data_dict['intermediate']['chicago']['parcel'] + 'addresses_concat.csv', index=False)
    sac_add = pd.read_csv(data_dict['raw']['sac']['parcel'] + 'Address.csv')
    sac_xwalk = pd.read_csv(data_dict['raw']['sac']['parcel'] + 'Address_parcel_xwalk.csv')
    sac_add = merge_sac_parcel_id(sac_add=sac_add, xwalk=sac_xwalk)
    clean_sac_add(sac_add).to_csv(data_dict['intermediate']['sac']['parcel'] + 'addresses_concat.csv', index=False)

    pass


import time

from sqlalchemy import Column, Integer, String, Float, Boolean, ForeignKey
import sqlalchemy.types as types
from sqlalchemy.orm import relationship
from sqlalchemy.sql.expression import func
from sqlalchemy import or_, and_, desc
from marshmallow import Schema, fields

from database import Base

class KycRequestSchema(Schema):
    date = fields.Float()
    token = fields.String()
    greenid_verification_id = fields.String()
    status = fields.String()

class KycRequest(Base):
    __tablename__ = 'kyc_requests'
    id = Column(Integer, primary_key=True)
    date = Column(Float, nullable=False, unique=False)
    token = Column(String, nullable=False, unique=True)
    greenid_verification_id = Column(String, nullable=False, unique=True)
    status = Column(String )

    def __init__(self, token, greenid_verification_id):
        self.date = time.time()
        self.token = token
        self.greenid_verification_id = greenid_verification_id
        self.status = "created"

    @classmethod
    def count(cls, session):
        return session.query(cls).count()

    @classmethod
    def from_token(cls, session, token):
        return session.query(cls).filter(cls.token == token).first()

    def __repr__(self):
        return '<KycRequest %r>' % (self.token)

    def to_json(self):
        schema = KycRequestSchema()
        return schema.dump(self).data


import json
import os
from elasticsearch import Elasticsearch, helpers, exceptions

client = Elasticsearch(os.getenv("ELASTICSEARCH_URL"))

f = open("dump", "r")

def main():
    while True:
        line = f.readline()
        if len(line) == 0:
            break
        data = json.loads(line)
        yield {
            '_op_type': 'index',
            '_index': 'data',
            '_id': data["id"],
            'doc': data
        }

helpers.bulk(client, main(), stats_only=True, chunk_size=2000)


import os
import discord
import requests
import asyncio
from dotenv import load_dotenv
from discord.utils import get
from discord.ext import commands

compteur = 301
nbConnected = 0

load_dotenv()
TOKEN = os.getenv('DISCORD_TOKEN')
SERVER_IP = os.getenv('SERVER_IP')
SERVER_PORT = os.getenv('SERVER_PORT')
CHANNEL_ID = int(os.getenv('CHANNEL_ID'))
VOCAL_ID = int(os.getenv('VOCAL_ID'))

bot = commands.Bot(command_prefix="!")

# Background task
async def background_task():
    global compteur
    await bot.wait_until_ready()

    while not bot.is_closed():
        await call_api()
        await asyncio.sleep(30)
        compteur += 30

async def call_api():
    global nbConnected
    global compteur

    for guild in bot.guilds:
        if (guild.id == CHANNEL_ID):
            channel = discord.utils.get(guild.channels, id=VOCAL_ID)
            response = requests.get('https://minecraft-api.com/api/ping/online/' + SERVER_IP + '/' + str(SERVER_PORT))
            nbConnected2 = response.content.decode("utf-8")

            if nbConnected != nbConnected2 and compteur > 300:
                nbConnected = nbConnected2
                message = 'Il y a ' + str(nbConnected) + (' connectés' if int(nbConnected) > 1 else ' connecté')
                compteur = 0
                await channel.edit(name=message)

bot.loop.create_task(background_task())

# Start bot
bot.run(TOKEN)


from createProtocol import ARP, EthernetII
from parseProtocol import Parser
from optparse import OptionParser
from helper import subnet_creator, get_mac_address, get_ip_address
from rich.progress import track
from time import sleep

import socket
import netifaces
import threading

def get_user_parameters():
    parse_options = OptionParser()
    parse_options.add_option("-n", "--network", dest="sub_network", help="Enter Network Address \n[+] Example : 192.168.1.0/24")
    parse_options.add_option("-i", "--interface", dest="interface", help="Enter Your Interface")

    options, _ = parse_options.parse_args()

    interfaces = netifaces.interfaces()

    if not options.interface and not options.sub_network:
        print("\nPlease enter parameters. You can use '--help' for parameters.")

    if options.interface not in interfaces:
        print("\nThere is no such interface.")

    if not options.sub_network:
        print("\nEnter network address.")

    return options

def send_packet(interface, ip):
    ethernet = EthernetII(src_mac=get_mac_address(interface))
    arp = ARP(dst_mac="00:00:00:00:00:00", src_mac=get_mac_address(interface), src_ip=get_ip_address(interface))

    sock = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, socket.htons(0x0806))
    sock.bind((interface, 0x0806))
    arp._dst_ip = ip
    packet = ethernet() + arp()
    sock.send(packet)

def receive_packet(interface):
    sock = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, socket.htons(0x0806))
    sock.bind((interface, 0x0806))
    parser = Parser()

    while True:
        data, _ = sock.recvfrom(65535)
        _, _, _, otherData = parser.ethernetFrame(data)
        opcode, dst_mac, dst_ip, src_mac, src_ip = parser.arp_frame(otherData)
        if opcode == 2:
            parser.print_frame(dst_mac=dst_mac, dst_ip=dst_ip)

def main():
    user_params = get_user_parameters()
    user_network = user_params.sub_network
    user_interface = user_params.interface
    ip_list = subnet_creator(user_network)

    receive_thread = threading.Thread(target=receive_packet, args=(user_interface,), daemon=True)
    receive_thread.start()

    sleep(1.5)

    for ip in track(ip_list, "Sending Packet => "):
        send_packet(user_interface,ip)

if __name__ == "__main__":
    main()


"""empty message

Revision ID: 810e0afb57ea
Revises: 22771e69d10c
Create Date: 2022-01-19 19:59:08.027108

"""
import sqlalchemy as sa
from alembic import op

# revision identifiers, used by Alembic.
revision = "810e0afb57ea"
down_revision = "22771e69d10c"
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table(
        "techstack",
        sa.Column("id", sa.Integer(), nullable=False),
        sa.Column("technology", sa.String(length=50), nullable=False),
        sa.Column("label", sa.String(length=50), nullable=True),
        sa.PrimaryKeyConstraint("id"),
        sa.UniqueConstraint("technology"),
    )
    op.create_table(
        "team_techstack",
        sa.Column("team_id", sa.Integer(), nullable=True),
        sa.Column("techstack_id", sa.Integer(), nullable=True),
        sa.ForeignKeyConstraint(
            ["team_id"],
            ["team.id"],
        ),
        sa.ForeignKeyConstraint(
            ["techstack_id"],
            ["techstack.id"],
        ),
    )
    op.create_table(
        "user_skill",
        sa.Column("user_id", sa.Integer(), nullable=False),
        sa.Column("techstack_id", sa.Integer(), nullable=False),
        sa.Column("skill_level", sa.Integer(), nullable=True),
        sa.Column("is_learning_goal", sa.Boolean(), nullable=True),
        sa.ForeignKeyConstraint(
            ["techstack_id"],
            ["techstack.id"],
        ),
        sa.ForeignKeyConstraint(
            ["user_id"],
            ["user.id"],
        ),
        sa.PrimaryKeyConstraint("user_id", "techstack_id"),
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table("user_skill")
    op.drop_table("team_techstack")
    op.drop_table("techstack")
    # ### end Alembic commands ###


from setuptools import setup
from hostinfo.version import __version__ as VERSION
from build_utils import BuildCommand
from build_utils import PublishCommand
from build_utils import BinaryDistribution

PACKAGE_NAME = 'pimjpeg'
BuildCommand.pkg = PACKAGE_NAME
# BuildCommand.py2 = False
# BuildCommand.py3 = False
PublishCommand.pkg = PACKAGE_NAME
PublishCommand.version = VERSION
README = open('README.rst').read()
GITHUB = "https://github.com/walchko/{}".format(PACKAGE_NAME)
INSTALL_REQ = open("requirements.txt").readlines()
TEST_REQ = ['nose']
CMDS = {'publish': PublishCommand, 'make': BuildCommand}

setup(
	name=PACKAGE_NAME,
	version=VERSION,
	author="Kevin J. Walchko",
	keywords=['package', 'keywords'],
	author_email="kevin.walchko@outlook.com",
	description="raspbery pi camera mjpeg streamer",
	license="MIT",
	package_data={
		'package': ['templates', 'static'],
	},
	include_package_data=True,
	zip_safe=False,
	classifiers=[
		'Development Status :: 4 - Beta',
		'License :: OSI Approved :: MIT License',
		'Programming Language :: Python :: 2.7',
		'Programming Language :: Python :: 3.6',
		'Operating System :: Unix',
		'Operating System :: MacOS :: MacOS X',
		'Operating System :: POSIX',
		'Topic :: Utilities',
		'Topic :: Internet :: WWW/HTTP :: Dynamic Content',
		'Topic :: System :: Shells',
		'Environment :: Console'
	],
	install_requires=INSTALL_REQ,
	tests_require=TEST_REQ,
	url=GITHUB,
	long_description=README,
	cmdclass=CMDS,
	packages=[PACKAGE_NAME],
	# scripts=[
	# 	'bin/hello.py'
	# ]
)


# date: 2021/09/06
# link: https://programmers.co.kr/learn/courses/30/lessons/17680

from collections import deque

def solution(cacheSize, cities):
    answer = 0
    status = deque()

    if cacheSize == 0:
        answer = len(cities) * 5
    else:
        for city in cities:
            city = city.upper()
            if city in status:
                answer += 1
                status.remove(city)
            else:
                answer += 5
                if len(status) >= cacheSize:
                    status.popleft()
            status.append(city)

    return answer


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from skimage.color import rgb2lab
from skimage.color import lab2rgb
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import GaussianNB
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import FunctionTransformer
import sys

# representative RGB colours for each label, for nice display
COLOUR_RGB = {
    'red': (255, 0, 0),
    'orange': (255, 114, 0),
    'yellow': (255, 255, 0),
    'green': (0, 230, 0),
    'blue': (0, 0, 255),
    'purple': (187, 0, 187),
    'brown': (117, 60, 0),
    'pink': (255, 187, 187),
    'black': (0, 0, 0),
    'grey': (150, 150, 150),
    'white': (255, 255, 255),
}
name_to_rgb = np.vectorize(COLOUR_RGB.get, otypes=[np.uint8, np.uint8, np.uint8])

def plot_predictions(model, lum=71, resolution=256):
    """
    Create a slice of LAB colour space with given luminance; predict with the model; plot the results.
    """
    wid = resolution
    hei = resolution
    n_ticks = 5

    # create a hei*wid grid of LAB colour values, with L=lum
    ag = np.linspace(-100, 100, wid)
    bg = np.linspace(-100, 100, hei)
    aa, bb = np.meshgrid(ag, bg)
    ll = lum * np.ones((hei, wid))
    lab_grid = np.stack([ll, aa, bb], axis=2)

    # convert to RGB for consistency with original input
    X_grid = lab2rgb(lab_grid)

    # predict and convert predictions to colours so we can see what's happening
    y_grid = model.predict(X_grid.reshape((wid*hei, 3)))
    pixels = np.stack(name_to_rgb(y_grid), axis=1) / 255
    pixels = pixels.reshape((hei, wid, 3))

    # plot input and predictions
    plt.figure(figsize=(10, 5))
    plt.suptitle('Predictions at L=%g' % (lum,))
    plt.subplot(1, 2, 1)
    plt.title('Inputs')
    plt.xticks(np.linspace(0, wid, n_ticks), np.linspace(-100, 100, n_ticks))
    plt.yticks(np.linspace(0, hei, n_ticks), np.linspace(-100, 100, n_ticks))
    plt.xlabel('A')
    plt.ylabel('B')
    plt.imshow(X_grid.reshape((hei, wid, 3)))

    plt.subplot(1, 2, 2)
    plt.title('Predicted Labels')
    plt.xticks(np.linspace(0, wid, n_ticks), np.linspace(-100, 100, n_ticks))
    plt.yticks(np.linspace(0, hei, n_ticks), np.linspace(-100, 100, n_ticks))
    plt.xlabel('A')
    plt.imshow(pixels)

#to convert rgb to lab
def rgb_to_lab(X):
    X = pd.DataFrame(X)
    #print(X)
    X = X.values.reshape(1, -1, 3)
    X = rgb2lab(X)
    X = X.reshape(-1,3)
    return X

def main(infile):
#def main():
    data = pd.read_csv(infile)
    data = pd.read_csv("colour-data.csv")
    #print(data)
    X = data[['R', 'G', 'B']] # array with shape (n, 3). Divide by 255 so components are all 0-1.
    #print(X)
    X = X/255
    X = X.values.tolist()
    #print(X)
    #https://stackoverflow.com/questions/34165731/a-column-vector-y-was-passed-when-a-1d-array-was-expected
    y = data[['Label']].values.ravel() # array with shape (n,) of colour words.
    #print(y)

    # TODO: build model_rgb to predict y from X.
    # TODO: print model_rgb's accuracy score

    X_train, X_valid, y_train, y_valid = train_test_split(X, y)
    model_rgb = GaussianNB()
    model_rgb.fit(X_train, y_train)
    y_predicted = model_rgb.predict(X_valid)
    print(model_rgb.score(X_valid, y_valid))

    # TODO: build model_lab to predict y from X by converting to LAB colour first.
    # TODO: print model_lab's accuracy score

    #We can create a pipeline model where the first step is a transformer that converts from RGB to LAB, and the second is a Gaussian classifier, exactly as before.
    model_lab = make_pipeline(
        FunctionTransformer(rgb_to_lab, validate = False),
        GaussianNB()
    )

    model_lab.fit(X_train, y_train)
    lab_y_predicted = model_lab.predict(X_valid)
    print(model_lab.score(X_valid, y_valid))

    plot_predictions(model_rgb)
    plt.savefig('predictions_rgb.png')
    plot_predictions(model_lab)
    plt.savefig('predictions_lab.png')

if __name__ == '__main__':
    main(sys.argv[1])
    #main()


import openpyxl
import tkinter as tk

def add_data_to_excel(roll_number, name):
    # Open the Excel file or create a new one if it doesn't exist
    try:
        workbook = openpyxl.load_workbook('data.xlsx')
    except FileNotFoundError:
        workbook = openpyxl.Workbook()

    # Select the active sheet (default: first sheet)
    sheet = workbook.active

    # Append the data to the Excel sheet
    row = [roll_number, name]
    sheet.append(row)

    # Save the changes to the Excel file
    workbook.save('data.xlsx')

def on_submit():
    roll_number = roll_entry.get()
    name = name_entry.get()

    try:
        add_data_to_excel(roll_number, name)
        result_label.config(text="Data successfully stored in Excel!", fg="green")
    except Exception as e:
        result_label.config(text=f"Error occurred: {e}", fg="red")

# Create the tkinter window
root = tk.Tk()
root.title("Data Entry")

# Labels and Entry widgets for roll number and name
roll_label = tk.Label(root, text="Roll Number:")
roll_label.pack()
roll_entry = tk.Entry(root)
roll_entry.pack()

name_label = tk.Label(root, text="Name:")
name_label.pack()
name_entry = tk.Entry(root)
name_entry.pack()

submit_button = tk.Button(root, text="Submit", command=on_submit)
submit_button.pack()

result_label = tk.Label(root, text="", fg="green")
result_label.pack()

# Run the tkinter main loop
root.mainloop()


import random, os, shutil, yaml, gzip
import pandas as pd
import numpy as np
import prepare.configs as configs
from google.cloud import storage
import pickle
import time

storage_client = storage.Client()
bucket = storage_client.bucket(configs.bucketName)

def encodeConfigs(_confs):
    return [
        _confs['sim time settings']['time step'],
        _confs['sim time settings']['total time'],
        _confs['sim time settings']['sampling rate'],
        int(_confs['change Na mem']['event happens']),
        _confs['change Na mem']['change start'],
        _confs['change Na mem']['change finish'],
        _confs['change Na mem']['change rate'],
        _confs['change Na mem']['multiplier'],
        int(_confs['change K mem']['event happens']),
        _confs['change K mem']['change start'],
        _confs['change K mem']['change finish'],
        _confs['change K mem']['change rate'],
        _confs['change K mem']['multiplier']
    ]

def generateDataset():
    srcFolder = "storage/processed"
    destFolder = "storage/ready"
    inputMaxCellsNumber = 250
    retryCounter = 0

    # Fetch available simulation folders
    runsIdxs = []
    for blob in bucket.list_blobs(prefix=srcFolder):
        folderName = blob.name.split("/")
        if (folderName[2] not in runsIdxs and folderName[2] != ""):
            runsIdxs.append(folderName[2])

    # Ftech simulation folders already processed
    with open("./prepare/processed.txt","r") as f:
        processedRunsIdxs = f.readlines()
        processedRunsIdxs = [folder.strip() for folder in processedRunsIdxs]

    availableFolders = []
    for runIdx in runsIdxs:
        if (runIdx not in processedRunsIdxs):
            availableFolders.append(runIdx)

    print("[GENERATE DATASET] Folders {} | Processed {} | Left {}".format(len(runsIdxs), len(processedRunsIdxs), len(availableFolders)))
    for i, runFolderIdx in enumerate(availableFolders):
        # Keep track of the progress
        if (i in [int(len(availableFolders)*0.25), int(len(availableFolders)*0.5), int(len(availableFolders)*0.75)]):
            print(">> {} %".format(int(i / len(availableFolders) * 100)))

        try:
            data = pd.read_csv('gs://{}/{}/{}/simulation.csv'.format(configs.bucketName, srcFolder, runFolderIdx))
            print(">> {} | {}".format(runFolderIdx, data['folderName'][0]))

            # 1. Download Sim Config File and encode It
            fileDest = '/tmp/rawSimConfig.yml'
            bucket.blob('storage/raw/{}/configs.yml'.format(data['folderName'][0])).download_to_filename(fileDest)
            with open(fileDest, 'r') as stream:
                simConfigRaw = yaml.safe_load(stream)
            simConfigsEncoded = np.asarray(encodeConfigs(simConfigRaw))
            simConfigsEncoded = np.append(simConfigsEncoded, [0]) # Add timestamp information

            # 2. Download Sim.betse File and open it ( to extract Membrane permeabilities values)
            fileDest = '/tmp/sim_1.betse.gz'
            bucket.blob('storage/raw/{}/sim_1.betse.gz'.format(data['folderName'][0])).download_to_filename(fileDest)
            with gzip.open(fileDest, "rb") as f:
                sim, cells, params = pickle.load(f)

            # 3. Generate training examples files. One for each simulation timestep using sim config, sim.betse & vmems
            for timestampIdx in range(len(sim.time)):
                inputVmem = np.asarray(data[data['timestamp'] == timestampIdx]['vmem'])
                outputVmem = np.asarray(data[data['timestamp'] == timestampIdx + 1]['vmem'])

                # Update timestamp information
                simConfigsEncoded[simConfigsEncoded.shape[0] - 1] = timestampIdx

                # 1. Compute cells perms values from cells membranes perms values. From {3, 6} values to 1 (average)
                cellsPopulationSize = inputVmem.shape[0]
                cells_mems = [[] for cell in range(cellsPopulationSize)]
                for memUniqueIdx, cellIdx in enumerate(cells.mem_to_cells):
                    cells_mems[cellIdx].append(sim.dd_time[timestampIdx][:, memUniqueIdx])

                cells_permeabilities = []
                for cellMembranes in cells_mems:
                    cells_permeabilities.append(np.mean(cellMembranes, axis=0))
                cells_permeabilities = np.asarray(cells_permeabilities) # N, 4 # K, Na, M-, Proteins-

                # concat Vmem values with perms values
                inputVmem = np.concatenate((inputVmem.reshape((-1, 1)), cells_permeabilities), axis=1) # N, 5

                # concat cells centers to input vector
                inputVmem = np.concatenate((inputVmem, cells.cell_centres), axis=1) # N, 7

                # Concat env concentrations
                env_cc = np.transpose(sim.cc_env_time[timestampIdx])[ : inputVmem.shape[0]] # get only same shape as inputVmem since env cc all the same
                inputVmem = np.concatenate((inputVmem, env_cc), axis=1) # N, 11

                # Concat cytosilic concentrations
                cytosolic_cc = np.transpose(sim.cc_time[timestampIdx])
                inputVmem = np.concatenate((inputVmem, cytosolic_cc), axis=1) # N, 15

                #Pad Input
                '''
                TODO:
                - Not pad with 0 since it is a possible Vmem value.
                '''
                if (inputVmem.shape[0] < inputMaxCellsNumber):
                    inputVmemPad = np.zeros((inputMaxCellsNumber, inputVmem.shape[1]))
                    inputVmemPad[:inputVmem.shape[0]] = inputVmem
                    inputVmem = inputVmemPad

                    outputVmemPad = np.zeros((inputMaxCellsNumber))
                    outputVmemPad[:outputVmem.shape[0]] = outputVmem
                    outputVmem = outputVmemPad
                #Discard Input
                elif (inputVmem.shape[0] > inputMaxCellsNumber):
                    print("<<ATTENTION>> Found Input with Numbers of cells higher that current Max: {} > {}".format(inputVmem.shape[0], inputMaxCellsNumber))
                    continue

                # Discard example if data
                # - Vmem < - 100 || > 100
                # - K_env, Na_env, M_env, X_env, K_cc, Na_cc, M_cc, X_cc > 1000
                #

                if (np.any(inputVmem[:, 0] < -100) or np.any(inputVmem[:, 0] > 100)):
                    print("Discard example, Vmem {}".format(np.max(np.abs(inputVmem))))
                    continue

                if (np.any(inputVmem[: , 7:] > 1000)):
                    print("Discard example, Concentration {}".format(np.max(inputVmem[: , 7:])))
                    continue

                if (np.any(outputVmem[:, 0] < -100) or np.any(outputVmem[:, 0] > 100)):
                    print("Discard example, Vmem Output {}".format(np.max(np.abs(outputVmem))))
                    continue
                #print("inputVmem length: {}".format(inputVmem.shape[0]))
                #print("Configs length: {}".format(configs.shape[0]))
                #print("outputVmem length: {}".format(outputVmem.shape[0]))
                filePath = '/tmp/example.npy'
                np.save(filePath, np.asarray([
                    inputVmem,
                    simConfigsEncoded,
                    outputVmem
                ], dtype="object"))
                blob = bucket.blob('{}/{}/{}.npy'.format(destFolder, runFolderIdx, timestampIdx))
                blob.upload_from_filename(filePath)

            retryCounter = 0
            with open("./prepare/processed.txt","a+") as f:
                f.write(runFolderIdx + "\n")
        # If for some reason processing fails. Handle it. It will not save on the processed.txt allowing to be processed at the next restart
        except:
            print("Handle Excpetion | Sleeping for {}".format(2 ** retryCounter))
            time.sleep(2 ** retryCounter) # sleep since may be due to too many requests
            retryCounter += 1
            continue


from django.core.paginator import InvalidPage

class AlphabetGlossary(object):
    """Алфавитный глоссарий"""
    def __init__(self, object_list, on=None, num_groups=7):
        self.object_list = object_list  # список объектов
        self.count = len(object_list)   # количество объектов в списке
        self.max_froups = num_groups    # количество алфавитных групп
        self.groups = []                # список алфавитных групп

        # Словарь, в котором ключ - буква алфавита, а значение - список объектов на эту букву из object_list
        chunks = {}

        for obj in self.object_list:
            if on:
                obj_str = str(getattr(obj, on))
            else:
                obj_str = str(obj)

            letter = str.upper(obj_str[0])

            if letter not in chunks:
                chunks[letter] = []

            chunks[letter].append(obj)

        # Вычисляем предполагаемое количество объектов в алфавитной группе
        per_group = self.count / num_groups
        for letter in chunks:
            chunk_len = len(chunks[letter])
            if chunk_len > per_group:
                per_group = chunk_len

        # Распределяем объекты по алфавитным группам
        current_group = AlphabetGroup(self)

        for letter in sorted(chunks.keys()):
            sub_list = chunks[letter]  # элементы списка объектов на указанную букву

            # Определяем, уместится ли sub_list в текущую алфавитную группу, или его
            # нужно переносить в новую. Новая группа будет создана, если:
            # - добавление sub_list приведёт к переполнению текущей группы
            # - в текущей группе свободного места меньше, чем количество неумещающихся объектов
            # - текущая группа не пуста (в противном случае, это будет означать, что len(sub_list) > per_group
            new_group_count = len(sub_list) + current_group.count
            if new_group_count > per_group and \
                    abs(per_group - current_group.count) < abs(per_group - new_group_count) and \
                    current_group.count > 0:
                self.groups.append(current_group)
                current_group = AlphabetGroup(self)

            current_group.add(sub_list, letter)

        # Если по окончании цикла осталась непустая группа, добавляем её в глоссарий
        if current_group.count > 0:
            self.groups.append(current_group)

    def group(self, num):
        """Возвращает объект алфавитной группы"""
        if len(self.groups) == 0:
            return None
        elif num > 0 and num <= len(self.groups):
            return self.groups[num - 1]
        else:
            raise InvalidPage

    @property
    def num_groups(self):
        """Возвращает количество алфавитных групп"""
        return len(self.groups)

class AlphabetGroup(object):
    """Алфавитная группа глоссария"""
    def __init__(self, glossary):
        self.glossary = glossary
        self.object_list = []
        self.letters = []

    @property
    def count(self):
        """Возвращает количество объектов в группе"""
        return len(self.object_list)

    @property
    def start_letter(self):
        """Возвращает первую букву группы"""
        if len(self.letters) > 0:
            self.letters.sort(key=str.upper)
            return self.letters[0]
        else:
            return None

    @property
    def end_letter(self):
        """Возвращает последнюю букву группы"""
        if len(self.letters) > 0:
            self.letters.sort(key=str.upper)
            return self.letters[-1]
        else:
            return None

    @property
    def number(self):
        """Возвращает номер группы в глоссарии"""
        return self.glossary.groups.index(self) + 1

    def add(self, new_list, letter=None):
        """Добавляет список объектов в группу"""
        if len(new_list) > 0:
            self.object_list = self.object_list + new_list
        if letter:
            self.letters.append(letter)

    def __repr__(self):
        """Возвращает метку группы"""
        if self.start_letter == self.end_letter:
            return self.start_letter
        else:
            return '%c-%c' % (self.start_letter, self.end_letter)


import os
import torch
import csv

import pandas as pd

from config import FoldersConfig

def txt_to_csv(input_path, output_path):
    with open(input_path, 'r') as in_file:
        stripped = (line.strip() for line in in_file)
        lines = (line.split() for line in stripped if line)
        with open(output_path, 'w') as out_file:
            writer = csv.writer(out_file)
            writer.writerows(lines)

def get_categories_and_path(input_path, output_path):
    with open(input_path, 'r') as in_file:
        reader = csv.reader(in_file)
        next(reader)
        row0 = next(reader)
        with open(output_path, 'w') as out_file:
            writer = csv.writer(out_file)
            writer.writerow(["path", "deep_fashion_category_name", "dataset"])
            for r in reader:
                split_r = r[0].split('_')[-2]
                category = split_r.split('/')[-2]
                r.append(r[0])
                r.append(category)
                r.append('deep_fashion')
                writer.writerow( (r[2], r[3], r[4]) )

def add_column_with_article_type_equivalence(deep_fashion, map_to_product_fashion, output_path):
    deep_fashion_df = pd.read_csv(deep_fashion, error_bad_lines=False)
    map_to_product_fashion_df = pd.read_csv(map_to_product_fashion)

    deep_fashion_with_article_type_df = deep_fashion_df.merge(map_to_product_fashion_df, on='deep_fashion_category_name', how='left')
    deep_fashion_with_article_type_df['id'] = deep_fashion_with_article_type_df.index + 100000

    deep_fashion_with_article_type_df = deep_fashion_with_article_type_df[['id', 'path', 'deep_fashion_category_name', 'product_fashion_article_type', 'dataset']]
    deep_fashion_with_article_type_df.columns = ['id', 'path', 'categoryName', 'articleType', 'dataset']

    deep_fashion_with_article_type_df.to_csv(output_path, index=False)

def prepare_datasets():
    resources = FoldersConfig.RESOURCES_DIR
    list_categories_path = resources + 'deep_fashion/list_category_img.txt'
    list_categories_output_path = resources + 'deep_fashion/list_category_img.csv'
    path_category_dataset = resources + 'deep_fashion/path_category_dataset.csv'
    map_to_product_fashion_path = resources + 'map_deep_fashion_to_product_fashion.csv'
    deep_fashion_with_article_type_path = resources + 'deep_fashion/deep_fashion_with_article_type.csv'

    if not os.path.exists(list_categories_output_path):
        txt_to_csv(list_categories_path, list_categories_output_path)

    if not os.path.exists(path_category_dataset):
        get_categories_and_path(list_categories_output_path, path_category_dataset)

    if not os.path.exists(deep_fashion_with_article_type_path):
        add_column_with_article_type_equivalence(path_category_dataset, map_to_product_fashion_path, deep_fashion_with_article_type_path)

if __name__ == "__main__":
    prepare_datasets()


from flask import Blueprint, render_template

redspine = Blueprint('redspine',
                     __name__,
                     template_folder='./',
                     static_folder='./',
                     static_url_path='/')
redspine.display_name = "Redspine"
redspine.published = False
redspine.description = "A red-spine notebook. Art that folds in on itself across pages by bleeding through."

@redspine.route('/')
def _redspine():
    return render_template('redspine.html')


from django.shortcuts import render
from .models import Book, Shope

def home(request):
    # qs = Post.objects.all()
    # # The DB query has not been executed at this point
    # x = qs
    # # Just assigning variables doesn't do anything
    # for x in qs:
    #     print(x)
    # # The query is executed at this point, on iteration
    # for x in qs:
    #     print("%d" % x.id)
    # # The query is not executed this time, due to caching

    post_qs = Book.objects.order_by('id')
    for start, end, total, qs in batch_qs(post_qs):
        print("Now processing %s - %s of %s" % (start + 1, end, total))
        for post in qs:
            print(post.name)
    return render(request, 'query_optimization/home.html')

def batch_qs(qs, batch_size=10):
    """
    Returns a (start, end, total, queryset) tuple for each batch in the given
    queryset.

    Usage:
        # Make sure to order your querset
        article_qs = Article.objects.order_by('id')
        for start, end, total, qs in batch_qs(article_qs):
            print "Now processing %s - %s of %s" % (start + 1, end, total)
            for article in qs:
                print article.body
    """
    total = qs.count()
    for start in range(0, total, batch_size):
        end = min(start + batch_size, total)
        yield (start, end, total, qs[start:end])

# def home(request):
#     books = Book.objects.all().only("name", "create_date")
#     for each in books:
#         print(each.name)
#     print(f"Cache {books._result_cache}")
#     return render(request, 'query_optimization/home.html')

def home(request):
    queryset = Shope.objects.prefetch_related('book').all()
    stores = []
    for store in queryset:
        books = [book.name for book in store.book.all()]
        stores.append({'id': store.id, 'name': store.name, 'books': books})
    return render(request, 'query_optimization/home.html')

    queryset = Store.objects.prefetch_related(
            Prefetch('books', queryset=Book.objects.filter(price__range=(250, 300))))


#!/usr/bin/python3
'''Defines a Base class
'''
import json
from os import path

class Base:
    '''Represents a base class

    Attributes:
        __nb_objects: holds the number of Base instances created
    '''
    __nb_objects = 0

    def __init__(self, id=None):
        '''Instantiates a Base object

        Args:
            id: type int. Defaults to None
        '''
        if id is not None:
            self.id = id
        else:
            type(self).__nb_objects += 1
            self.id = type(self).__nb_objects

    @staticmethod
    def to_json_string(list_dictionaries):
        '''Returns the JSON string representation of list_dictionaries
        '''
        if list_dictionaries is None:
            return '[]'
        if type(list_dictionaries) is not list:
            raise TypeError('to_json_string argument must be a list of dicts')
        for obj in list_dictionaries:
            if type(obj) is not dict:
                raise TypeError('items in to_json_string arg must be dicts')
        return json.dumps(list_dictionaries)

    @classmethod
    def save_to_file(cls, list_objs):
        '''Writes the JSON string representation of list_objs to a file
        '''
        if type(list_objs) not in (None, list):
            raise TypeError('list_objs must be of type list')
        for obj in list_objs:
            if type(obj) is not cls:
                raise TypeError('items in list_objs must be of same type as cls')
        filename = cls.__name__ + '.json'
        with open(filename, 'w', encoding='utf-8') as f:
            if list_objs is None:
                f.write('[]')
            else:
                list_dicts = [obj.to_dictionary() for obj in list_objs]
                # json.dump(list_dicts, f) achieves same thing as next line
                f.write(Base.to_json_string(list_dicts))

    @staticmethod
    def from_json_string(json_string):
        '''Returns the list of JSON string representation
        '''
        if json_string is None or json_string == '':
            return []
        if type(json_string) is not str:
            raise TypeError('json_string must be str repr of a list of dicts')
        return json.loads(json_string)

    @classmethod
    def create(cls, **dictionary):
        '''Returns an instance with all attributes already set
        '''
        # Create a dummy Rectangle or Square instance
        if cls.__name__ == 'Rectangle':
            dummy = cls(1, 1)
        elif cls.__name__ == 'Square':
            dummy = cls(1)
        dummy.update(**dictionary)
        return dummy

    @classmethod
    def load_from_file(cls):
        '''Returns a list of instances
        '''
        filename = cls.__name__ + '.json'
        if path.exists(filename):
            with open(filename, 'r', encoding='utf-8') as f:
                json_string = f.read()
            list_dict = cls.from_json_string(json_string)
            return [cls.create(**d) for d in list_dict]
        return []


import gc
import numpy as np
import xarray as xr
import scipy.ndimage.filters as conv

from . import dc_utilities as utilities
from datetime import datetime

####################################################
# ｜ ＴＳＭ ｜
####################################################
# 0.0001 for the scale of ls7 data.
def _tsmi(dataset):
    return (dataset.red.astype('float64') + dataset.green.astype('float64'))*0.0001 / 2

def tsm(dataset_in, clean_mask=None, no_data=0):
    # Create a clean mask from cfmask if the user does not provide one
    if clean_mask is None:
        cfmask = dataset_in.cf_mask
        clean_mask = utilities.create_cfmask_clean_mask(cfmask)

    tsm = 3983 * _tsmi(dataset_in)**1.6246
    tsm.values[np.invert(clean_mask)] = no_data # Contains data for clear pixels

    # Create xarray of data
    time = dataset_in.time
    latitude = dataset_in.latitude
    longitude = dataset_in.longitude
    dataset_out = xr.Dataset({'tsm': tsm},
                             coords={'time': time,
                                     'latitude': latitude,
                                     'longitude': longitude})
    return dataset_out

def mask_tsm(dataset_in, wofs):
    wofs_criteria = wofs.copy(deep=True).normalized_data.where(wofs.normalized_data > 0.8)
    wofs_criteria.values[wofs_criteria.values > 0] = 0
    kernel = np.array([[1,1,1],[1,1,1],[1,1,1]])

    mask = conv.convolve(wofs_criteria.values, kernel, mode ='constant')
    mask = mask.astype(np.float32)

    dataset_out = dataset_in.copy(deep=True)
    dataset_out.normalized_data.values += mask
    dataset_out.total_clean.values += mask
    dataset_out.normalized_data.values[np.isnan(dataset_out.normalized_data.values)] = 0
    dataset_out.total_clean.values[np.isnan(dataset_out.total_clean.values)] = 0

    return dataset_out


from dataclasses import dataclass

@dataclass
class block:
    name: str
    seperatorStart: str
    seperatorEnd: str

def getBlock(blocks: list, input: list):
    strings = list()
    index = 0
    offsetindex = 0
    foundBlock = False
    dontAppend = False

    for string in input:
        dontAppend = False
        if (foundBlock and blocks[index].name == "*"):
            if (type(strings) == list): strings = dict()

            if (string.__contains__(blocks[index].seperatorStart)):
                if (offsetindex == 0):
                    foundBlockName = string.split(blocks[index].seperatorStart)[0].strip().lower()
                    dontAppend = True
                offsetindex += 1
            elif (string.__contains__(blocks[index].seperatorEnd)):
                if (offsetindex == 0): break
                else: offsetindex -= 1

            if (dontAppend == False and offsetindex > 0):
                if (strings.__contains__(foundBlockName)):
                    strings[foundBlockName].append(string)
                else:
                    strings[foundBlockName] = [string]

        elif (foundBlock == True):
            if (string.__contains__(blocks[index].seperatorStart)):
                offsetindex += 1
            elif (string.__contains__(blocks[index].seperatorEnd)):
                if (offsetindex == 0): break
                else: offsetindex -= 1
            strings.append(string)
        else:
            if (string.__contains__(blocks[index].seperatorStart)):
                stringSplit = string.split(blocks[index].seperatorStart, 1)
                if (stringSplit[0].strip().lower() == blocks[index].name.lower().strip() and offsetindex == 0):
                    if (len(stringSplit[1].strip()) > 0):
                        strings.append(stringSplit[1].strip())
                    if (index + 1 <= len(blocks) - 1 ): index += 1
                    if (index == len(blocks) - 1): foundBlock = True
                else:
                    offsetindex += 1
            elif (string.__contains__(blocks[index].seperatorEnd)):
                if (offsetindex > 0):
                    offsetindex -= 1

                else: index -= 1
    return strings

def getVariable(name: str, blocks: list, seperator: str, input: list):
    if (len(blocks) > 0): block = getBlock(blocks, input)
    else: block = input
    if (name == "*"):
        output = dict()
        for string in block:
            if (string.__contains__(seperator)):
                stringSplit = [ stringSplit.strip() for stringSplit in string.split(seperator)]
                output[stringSplit[0].lower()] = stringSplit[1]
    else:
        for string in block:
            if (string.__contains__(seperator)):
                stringSplit = [ stringSplit.strip() for stringSplit in string.split(seperator)]
                if (stringSplit[0].lower() == name.lower()):
                    output = stringSplit[1]
                    break

    return output


#!/usr/bin/python
# -*- coding: utf-8 -*-

from flask import Flask, render_template
import platform
import netifaces

myApp = Flask(__name__)

@myApp.route('/')
def home():
    data = {'user': 'ramy', 'machine':platform.node(), 'os':platform.system(), 'dist':platform.linux_distribution(), 'interfaces':netifaces.interfaces()}

    return render_template('index.system.html', title='Home', data=data)

if __name__ == '__main__':
    myApp.run(host='0.0.0.0', port=999)


# -*- coding: utf-8 -*-
"""
Created on Thu Jun 15 09:32:16 2017

@author: Francesco
"""

from sklearn.preprocessing import StandardScaler
import numpy as np
import threading as th
import time
import re
import matplotlib.pyplot as plt

movement_kind = ["wrist up",
                 "wrist down",
                 "wrist rotation out",
                 "wrist rotation inside",
                 "hand open",
                 "hand closed"]

class up(object):
    index_array = []
    value_array = []
    def __init__(self):
        self.index_array.append(0)
    def update(self,index,value):
        self.index_array.append(index)
        self.value_array.append(value)

def load_from_C_formatting(file):
    f = open(file,'r').read()

    temp = f.split('\n')

    Ltemp =  len(temp)

    state = int(temp[0][-1])
    start_index = 0

    X = []
    Y = []

    for j in range(1,Ltemp-1):
        new_state = int(temp[j][-1])
        if(int(new_state != state)):
            Xraw = temp[start_index:j-1]
            start_index = j

            L = len(Xraw)
            X_current = np.zeros((L,8))

            i = 0
            k = 0

            for line in Xraw:
                #print(line[:-2])
                for value in line[:-2].split(','):

                    X_current[i,k] = value
                    k+=1
                i+=1
                k=0

            Y.append(state)
            X.append(X_current)

        state = new_state

    #last start index is the index of the last recording

    Xraw = temp[start_index:-1]

    L = len(Xraw)
    X_current = np.zeros((L,8))

    i = 0
    k = 0

    for line in Xraw:
        #print(line[:-2])
        for value in line[:-2].split(','):

            X_current[i,k] = value
            k+=1
        i+=1
        k=0

    Y.append(state)
    X.append(X_current)

    figures = []

    for movement in np.unique(Y):

        figures.append(plt.subplots(1,1))
        for p in [2,3,4,5,6,7]:
            y = X[movement][:,p]
            moving_average(y,10)

            #figures è una tupla (fig,axes) e noi dobbiamo
            #plottare su axes
            movement = int(movement)
            figures[movement][1].plot(y,color=colorarray[p],label='ch'+str(p))

        legend = figures[movement][1].legend(loc='upper left', shadow=True)
        figures[movement][0].suptitle(movement_kind[movement])

    plt.show()

    return (X,Y)

def load_dataset(file):
    """ RIFARE INIZIO """
    n_channels = 8

    f = open(file,'r')
    #jump to the second block(the first is corrupted)
    while(1):
        if(f.read(1) == '-'):
            start = f.tell()+2
            break
    f.seek(start)
    #now we are ready to read the first block, which is the first feature actually
    #understand the block length, must be equal for each block
    dataset = f.read()
    n_linee = 0

    for line in dataset.split('\n'):
        n_linee+=1
        if(line == '-'):
            n_linee -= 1
            break
    len_blocco = n_linee+1

    #create the structure that will hold the features
    #each feature is a matrix n_linee*9 (n_channels + classe movimento)

    n_blocks = (len(dataset.split('\n'))-1)/len_blocco

    features = np.zeros((n_linee,n_channels+1,int(n_blocks)+1))

    i = 0
    j = 0
    block = 0
    for line in dataset.split('\n'):
        if(len(line)<5):
            block+=1
            i = 0
            #print(line)
        else:
            for value in line.split(','):
                features[i,j,block] = value
                j+=1
            #print(line)
            j=0
            i+=1

    return features

def gradient(data,channels):
    der = np.zeros((len(data),channels))
    for i in range(1,len(data)):
        der[i,:] = data[i,:]-data[i-1,:]
    return der

def moving_average(data,samp_for_average):
    n_windows = int(len(data)/samp_for_average)
    for i in range(n_windows):
        data[i*samp_for_average:(i+1)*samp_for_average] = np.average(data[i*samp_for_average:(i+1)*samp_for_average])

def open_outfile(file,rep):
    f = open(file,'r').read()
    lines = f.split('\n')
    info_decoded = lines[rep].split('\t')
    first_matrix = info_decoded[:-1]
    n_cluster = int(info_decoded[-1])
    #this code fails when there is a number without decimals, because 2. doesn't match the pattern
    #since it searches for another number after the dot, that's the reason why the second "try"
    #to catch this behaviour we say that two possible patterns may exist, 3.0 is recognized as well as 3.
    patterns=re.compile(r'-\d+\.\d+|\d+\.\d+|-\d+\.|\d+\.')
    #as a note: we search for both positive or negative(minus sign) but the order is important,
    #because if -\d+\. was before -\d+\.\d+, the number -2.3 would be recognized as -2.
    matrix = np.array(patterns.findall(first_matrix[0]),dtype='f')

    for row in first_matrix[1:]: #the first has alread been taken
        try:
            temp = np.array(patterns.findall(row),dtype='f')
            matrix = np.vstack((matrix,temp))
        except ValueError:
            print("Error:",row)

    return (matrix,n_cluster)

#load data
def load_data_into_matrix(file,startline=0,endline=-1,cols=8,mode="signal"):
    graph_data = open(file,'r').read()

    lines = graph_data.split('\n')
    n_channels = cols
    n_lines = (len(lines))
    vertical_lines = len(lines[startline:endline])
    data = np.zeros((vertical_lines,n_channels)) #read all channels (8), plot only the desired

    #the last acquisition may be corrupted, sudden termination of serial comm
    #the first lines may be corrupted by giggering of the sensors/serial reads garbage
    if mode == "signal":
        i=0
        j=0
        for line in lines[startline:endline]:
            if(len(line)>1):
                t = line.split(',')
                for value in t:
                    data[i,j] = t[j]
                    j+=1
                j=0
                i+=1

        return data

    if mode == "encoded":
        i=0
        j=0
        data = np.chararray((n_lines - (startline-endline),n_channels))
        for line in lines[startline:endline]:
            if(len(line)>1):
                t = line.split(',')
                for value in t:
                    data[i,j] = t[j]

                    j+=1
                j=0
                i+=1

        return data

def unsigned_derivative(x_t,x_tmen1):
    return np.abs((x_t - x_tmen1)/x_t)

colorarray = ['b','g','r','c','m','y','k','0.75']

mode = {'polso_piegato_alto':[0,1,4],   #estensori
        'polso_piegato_basso':[2,3,7],   #flessori
        'polso_ruotato_esterno':[0,3],  #ulnari
        'polso_ruotato_interno':[1,2],  #radiali
        'updown':[0,1],
        'intest':[2,3],
        'tutti':range(8)}

class track(object):
    def __init__(self,data):
        self.data = data
        self.channels = data.shape[1] #number of channels
        self.samples = data.shape[0] #number of samples

    def set_baseline(self,number_of_samples = 30):
        #define the baseline for each channel, with this code we
        #don't care about how many channels are there, 2 or 3 or n
        #the shape of baseline will be 1xn
        #basically this code is doing this: for each column sum the first
        #30 values and do the average, the subtract this value from
        #all the values
        self.baseline = np.sum(self.data[0:number_of_samples,:],axis=0)/number_of_samples
        self.data -= self.baseline

    def moving_avg(self,samp_for_average):
        n_windows = int(len(self.data)/samp_for_average)
        for s in range(self.channels):
            for i in range(n_windows):
                self.data[i*samp_for_average:(i+1)*samp_for_average,s] = np.average(self.data[i*samp_for_average:(i+1)*samp_for_average,s])

    def __getitem__(self,index):
        return self.data[index[0]][index[1]]

    def read_channel(self,channel):
        return self.data[:,channel]

    def shape(self):
        return self.data.shape

class computation(th.Thread):
    def __init__(self,name,signal):
        th.Thread.__init__(self)
        self.signal = signal
        self.name = name

    def run(self):
        #we use alto/basso together with esterno/interno since
        #they aren't mutual exclusive movements, the wirst can
        #in fact go up while it may be extern/intern, but cannot go
        #up while it is down

        #we somehow simulate the fact that we are reading a stream of data
        #so we don't use all the data together, but once more at each step

        #feature extraction: position: baseline and movement: derivative

        #t represents time that goes by
        """ !!!! MUST BE A MULTIPLE OF 10 !!!! """
        windows_length = 10
        n_chann = self.signal.shape()[1]
        encoder = (lambda Y: 'a' if Y > windows_length/100 else 'b' if Y > -windows_length/100 else 'c')

        encoded = ['x']*8

        t = 0

        outfile = open('thread_data'+self.name+'.txt','w')
        #outfilerrr = open('prova_pos'+self.name+'.txt','w')

        flag = 1

        print("%s: samples %d, channels %d"%(self.name,self.signal.shape()[0],self.signal.shape()[1]) )
        try:
            while(1):

                der_ = self.signal[t,:] - self.signal[t+windows_length,:]
                #print(der_[0], self.signal[t,0], self.signal[t+windows_length,0] )
                #se deltaY > deltaX .... calcoli sul quaderno,
                #qua aggiungo solo deltaX è sempre "window length" perchè è la distanza alla quale sono presi i punti

                i=0
                encoded[0] = encoder(der_[0])
                outfile.write("%c"%encoded[0])
                for i in range(1,8):
                    encoded[i] = encoder(der_[i])
                    outfile.write(',')
                    outfile.write("%c"%encoded[i])

                #slide window
                t += windows_length #deve essere almeno superiore alla media mobile

                #print(line)
                flag+=1
                outfile.write('\n')
                #print(time.time()-start_time)

        except IndexError:

            outfile.close()
            print(flag)

"""
*********************** MAIN **********************

"""

class offline_process(object):

    def __init__(self,filename):
        """ LOAD DATA """
        data = load_data_into_matrix(filename,0,-1,8)

        """ DIVIDE INTO MEANINGFUL CHANNELS """
        self.polso_updown = track(data[:,mode['tutti']])
        #self.polso_intest = track(data[:,mode['intest']])

        """ REMOVE BASELINE """
#        self.polso_alto_track.set_baseline()
#        self.polso_basso_track.set_baseline()
#        self.polso_esterno_track.set_baseline()
#        self.polso_interno_track.set_baseline()

        """ LOW PASS FILTER """
        self.polso_updown.moving_avg(10)
        #self.polso_intest.moving_avg(30)

        """ START TWO THREADS TO COMPUTE"""

        self.thread_updown = computation("-encoding",self.polso_updown)
        #self.thread_leftright = computation("intest",self.polso_updown)

    def __call__(self):
        #start a thread for each computation, which is left-right or
        #up down

        try:
            self.thread_updown.start()
            #self.thread_leftright.start()

            self.thread_updown.join()
            #self.thread_leftright.join()
        except KeyboardInterrupt:

            self.thread_updown.join()
            #self.thread_leftright.join()

class occurrence_table(object):
    def __init__(self):
        self.items = []
        self.number_of_occurrence = []
        self.l = 0
        self.total = 0
    def __repr__(self):
        return "Object filled with %d items"%self.l
    def __str__(self):
        for i in range(self.l):
            print("%s: %d"%(self.items[i],self.number_of_occurrence[i]))
        return "----- End ------ "
    def append(self,item):
        j=0
        for occurrence in self.items:
            if occurrence != item:
                j=j+1
            else:
                self.number_of_occurrence[j]+=1
                self.total += 1
                return
        #se hai fatto tutto il for senza entrare nell'else vuol
        #dire che è nuovo, quindi lo appendo
        self.items.append(item)
        #ovviamente metto nel conteggio che ne ho aggiunto uno
        self.number_of_occurrence.append(1)
        self.l += 1
        self.total += 1
        #conteggio e item sono due liste separate ma l'elemento
        #j esimo di number_of.. indica quante volte l'elemento
        #j esimo di items è presente

    def get(self):
        return (self.items,self.number_of_occurrence)

    def prob(self):
        temp = [1]*self.l
        for i in range(self.l):
            temp[i] = self.number_of_occurrence[i]/self.total

        return temp

if __name__ == "__main__":

    p = offline_process("model_updown.txt")
    p()

    encoded_signal = load_data_into_matrix("thread_data-encoding.txt",mode="encoded")

    entropy = lambda p: 0 if p==0 else -p*np.log2(p)

    symbols_taken = 3
    n_samples = encoded_signal.shape[0]
    window_len = 30
    start = 0
    start_for_plot = 0

    channel = encoded_signal.shape[1]

    n_steps= window_len - symbols_taken + 1

    print("n_steps:",n_steps)

    ch = np.zeros((n_samples - window_len,channel),dtype='f')

    outfile = open('entropy.txt','w')

    while(start < n_samples - window_len):
        table = []
        for i in range(channel):
            table.append(occurrence_table())

        window = encoded_signal[start:start+window_len,:]
        for i in range(n_steps):
            for j in range(channel):
                table[j].append(window[i:i+symbols_taken,j].tostring())

        entropy_per_channel = [0]*channel #il massimo dell'entropia quando ho tutto uguale, 3**3 perchè ho 3 simboli per 3 posizioni

        for j in range(channel):
            list_of_prob = table[j].prob()
            #print(list_of_prob)
            for i in range(len(list_of_prob)):
                entropy_per_channel[j] += entropy(list_of_prob[i])

            ch[start_for_plot,j] = entropy_per_channel[j]
            outfile.write(str(entropy_per_channel[j]))
            outfile.write('\t')

        start += 1
        start_for_plot += 1
        outfile.write('\n')
        #print(table[0])

    outfile.close()

    fig2, ax2 = plt.subplots(1,1)
    for p in range(channel):

        y = ch[:,p]
        ax2.plot(y,color=colorarray[p],label='ch'+str(p))

    legend = ax2.legend(loc='upper left', shadow=True)

    plt.show()

    #