__author__ = 'Chi'

from time import time
from bin.headlinegeneration.utils.loginfor import loginformation
from gensim.models import word2vec
import re
import multiprocessing
import gensim
import os

def cleandata(rawdata):
    try:
        cleanr = re.compile('<.*>?>')
        cleantext = re.sub(cleanr, ' ', rawdata)
    except:
        pass
        #loginformation("", "debug", "Error occurs while cleaning data: " + rawdata)
        #loginformation("", "info", "Finished clean the data: " + rawdata)
    return cleantext

#生成迭代器，以防止数据太大，内存不够
class GetSentence(object):
    def __init__(self, dirname):
        self.dirname = dirname

    def __iter__(self):        #__iter__方法的对象是可迭代的。
        for file in os.listdir(self.dirname):
            filePath = self.dirname + '/' + file
            try:
                for line in open(filePath):
                    sline = line.strip()
                    if sline == "":
                        continue
                    rline = cleandata(sline)
                    yield rline.split(" ")
            except IOError:
                loginformation("", "info", "open file: " + self.dirname + '/' + file + " error ")
            else:
                loginformation("", "info", "finish the file: " + self.dirname + '/' + file)

def wordVectorTrainModel(trainDataPath, modelPath, vocabularyPath, word2VecOrgPath):
    begintime = time()
    sentences = GetSentence(trainDataPath)
    #对训练数据的参数配置
    model = gensim.models.Word2Vec(sentences,
                                   size = 100,
                                   window = 5,
                                   min_count = 1,
                                   workers = 4)
    loginformation("", "info", "The process is saving the model.")
    try:
        model.save(modelPath)
        model.wv.save_word2vec_format(word2VecOrgPath,
                                  vocabularyPath,
                                  binary=False)
    except:
        loginformation("", "info", "Error occurs while saving the word vector model!!")
    else:
        loginformation("", "info", "Finished the process of saving word vector model.")
    endtime = time()
    loginformation("", "info", "Total word vector training time is: " + (endtime - begintime))

#get the word vector of sentence after segment, requirement: space between word and word
def getSentenceVector(sentence, wordVectorModel, wordVectorModelDim):
    sentenceList = sentence.rstrip('\n').split("  ")
    sentenceWordVectorList = []
    for word in sentenceList:
        wordVector = []
        if(word != ' ' or word != ''):
            try:
                for i in range(wordVectorModelDim):
                    wordVector.append(str(wordVectorModel[word][i].strip))
            except KeyError:
                loginformation("", "debug", word + "not in the vocabulary !!!")
                for i in range(wordVectorModelDim):
                    wordVector.append("0.0")
                pass
            else:
                loginformation("", "info", "Get the vector of " + word)
        else:
            loginformation("", "info", "null word occrs")
        sentenceWordVectorList.append(wordVector)
    loginformation("", "info", "transfered sentence " + sentence + " to word vector.")
    return sentenceWordVectorList

if __name__ == "__main__":
    rootPath = os.path.abspath(".") + "/"
    trainDataPath = rootPath + "../../../../../Documents/weibocluster_segmentByPynlpir/"
    modelPath = rootPath + "../../../data/preprocess/word2vectormodel/dim100Window5Min1.model"
    vocabularyPath = rootPath + "../../../data/preprocess/wordVectorVocabulary/dim100Window5Min1Vocabulary"
    word2VecOrgPath = rootPath + "../../../data/preprocess/word2VecOrgPath/dim100Window5Min1Word2VecOrgPath"
    wordVectorTrainModel(trainDataPath, modelPath, vocabularyPath, word2VecOrgPath)


from sklearn.datasets import load_breast_cancer
import numpy as np
import pandas as pd

cancer= load_breast_cancer()
data = cancer.data
column_names = cancer.feature_names
x = pd.DataFrame(data, columns=column_names)
y = cancer.target

#The following 2 methods can be used to categorize continuous variables by using decision trees.
#Univariate decision trees are generated separately with each variable and the buckets that trees generated are being used to
#create new variables that are the categorical versions of the original ones.

#This algorithm is just about finding the optimal depth for each tree. The output of this method will be input to the following one which generates the final trees.
# The method should be called in a for loop which scans all the variables in the 'x_train' data frame. 'var' parameter represents this loop value.
def OptDepth_DecTree(x_train,y_train,var):
    from sklearn.model_selection import cross_val_score
    from sklearn.tree import DecisionTreeClassifier
    score_ls = []     # here I will store the roc auc
    score_std_ls = [] # here I will store the standard deviation of the roc_auc
    for tree_depth in [1,2]:
        tree_model = DecisionTreeClassifier(max_depth=tree_depth)
        scores = cross_val_score(tree_model,x_train.iloc[:,int(var)].to_frame(),
        y_train, cv=3, scoring='roc_auc')
        score_ls.append(np.mean(scores))
        score_std_ls.append(np.std(scores))
    temp = pd.concat([pd.Series([1,2]), pd.Series(score_ls), pd.Series(score_std_ls)], axis=1)
    temp.columns = ['depth', 'roc_auc_mean', 'roc_auc_std']
    a=np.where(temp.iloc[:,1]==temp.iloc[:,1].max()) #gives the index of maximum auc
    s = int(a[0])
    return s+1 #gives the value that have maximum auc

#This is the main algorithm that categorizes continuous variables with the help of decision tree.
#If you want to categorize the null values separately, just leave them as null. This algorithm puts them to bucket '1'. If there is no null, bucket numbers start from '2'.
def Binnig_DecTree(x_train,y_train,opt_depth):
    from sklearn.tree import DecisionTreeClassifier
    c=len(x_train.columns)
    for j in range(c): #column
        x=x_train.iloc[:,j]
        #tree_model=DecisionTreeClassifier(min_samples_leaf=int(len(x)*0.1)) #if you don't want to restrict the max_depth.
        tree_model=DecisionTreeClassifier(max_depth=opt_depth[j], min_samples_leaf=int(len(x_train)*0.1)) #There should be at least 10% of the total data in a leaf
        null_index     =x[pd.isnull(x)].index
        not_null_index =x[pd.notnull(x)].index
        y_null=y_train[null_index]
        x_not_null=x[not_null_index]
        y_not_null=y_train[not_null_index]
        tree_model.fit(x_not_null.to_frame(), y_not_null)
        #Generates 'TREE_' columns which give the decision tree predictions.
        x_train.loc[not_null_index,'TREE_'+str(x.name)]=tree_model.predict_proba(x_not_null.to_frame())[:,1]
        x_train.loc[null_index,    'TREE_'+str(x.name)]=np.mean(y_null)
        #Shows the max value in each bucket (break point)
        bins=(pd.concat( [x_train.groupby (['TREE_'+str(x.name)]) [x.name].max()], axis=1))
        bins=np.sort(bins,0)
        bins=[float(i) for i in bins] #float to list
        if(pd.isnull(bins[len(bins)-1])):
            del bins[len(bins)-1]
        del bins[len(bins)-1]
        bins.insert(0,-100000000000)  #if you have greater or smaller values in your data, just change the limits manually.
        bins.append(1000000000000)
        labels=[i+2 for i in range(len(bins)-1)] #Labels start from '2'.

        category = pd.cut(x,bins=bins,labels=labels) #Groups the variable based on bin separations.
        category = category.to_frame()
        category[x.name] = category[x.name].cat.add_categories([1])
        category[x.name].fillna(1, inplace=True) #All the nulls are being labeled as '1'.
        category.columns = ['BIN_'+x.name]
        x_train = pd.concat([x_train,category],axis = 1)
        print(j, ' finished')
    return x_train

#Determines the optimum depth for each variable and keeps them in a list.
opt_depth=[]
for i in range(len(x.columns)):
    print("started",i)
    opt_depth.append(OptDepth_DecTree(x,y,i))
    print("finished",i)

#Generates a dataframe that consists of all categorized versions of variables.
#'1' is the category that keeps null's. If the variable has no null, categories start from '2'.
x_categorized=Binnig_DecTree(x,y,opt_depth)

#After categorizing, you can use the categories as continuous values where the values are the predictions of trees
#(in this case, mean of target in the corresponding bucket).This data frame is x_TREE.
# On the other hand, you can continue with categorical values which is x_BIN.
x_TREE=x_categorized.iloc[:,x_categorized.columns.str.contains('TREE_')]
x_BIN=x_categorized.iloc[:,x_categorized.columns.str.contains('BIN_')]


# -*- coding: utf-8 -*-
"""
In this module you find the base workflow for writing out the kkr_flexfiles and
some helper methods to do so with AiiDA
"""
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
from aiida.orm import Code, load_node
from aiida.plugins import DataFactory
from aiida.engine import WorkChain, ToContext, if_
from masci_tools.io.kkr_params import kkrparams
from aiida_kkr.tools.common_workfunctions import test_and_get_codenode, get_parent_paranode, update_params_wf, get_inputs_kkr
from aiida_kkr.calculations.kkr import KkrCalculation
from aiida.engine import CalcJob
from aiida.orm import CalcJobNode
from masci_tools.io.common_functions import get_Ry2eV
from aiida.orm import WorkChainNode
from aiida.common.exceptions import InputValidationError

__copyright__ = (u"Copyright (c), 2018, Forschungszentrum Jülich GmbH, "
                 "IAS-1/PGI-1, Germany. All rights reserved.")
__license__ = "MIT license, see LICENSE.txt file"
__version__ = "0.4.3"
__contributors__ = (u"Fabian Bertoldo", u"Philipp Ruessmann")

# ToDo: add more default values to wf_parameters

RemoteData = DataFactory('remote')
StructureData = DataFactory('structure')
Dict = DataFactory('dict')
FolderData = DataFactory('folder')

class kkr_flex_wc(WorkChain):
    """
    Workchain of a kkr_flex calculation to calculate the Green function with
    KKR starting from the RemoteData node of a previous calculation (either Voronoi or KKR).

    :param options: (Dict), Workchain specifications
    :param wf_parameters: (Dict), Workflow parameters that deviate from previous KKR RemoteData
    :param remote_data: (RemoteData), mandatory; from a converged KKR calculation
    :param kkr: (Code), mandatory; KKR code running the flexfile writeout
    :param impurity_info: Dict, mandatory: node specifying information
                          of the impurities in the system

    :return workflow_info: (Dict), Information of workflow results
                            like success, last result node, list with convergence behavior
    :return GF_host_remote: (RemoteData), host GF of the system
    """

    _workflowversion = __version__
    _wf_label = 'kkr_flex_wc'
    _wf_description = 'Workflow for a KKR flex calculation starting from RemoteData node of previous converged KKR calculation'

    _options_default = {'queue_name' : '',                        # Queue name to submit jobs too
                        'resources': {"num_machines": 1},         # resources to allowcate for the job
                        'max_wallclock_seconds' : 60*60,          # walltime after which the job gets killed (gets parsed to KKR)}
                        'custom_scheduler_commands' : '',         # some additional scheduler commands
                        'use_mpi' : True}                         # execute KKR with mpi or without

    _wf_default = {'ef_shift': 0. ,                                  # set costum absolute E_F (in eV)
                   'dos_params': {'nepts': 61,                       # DOS params: number of points in contour
                                  'tempr': 200, # K                  # DOS params: temperature
                                  'emin': -1, # Ry                   # DOS params: start of energy contour
                                  'emax': 1,  # Ry                   # DOS params: end of energy contour
                                  'kmesh': [10, 10, 10]},            # DOS params: kmesh for DOS calculation (typically higher than in scf contour)
                   'dos_run': False
                   }

    @classmethod
    def get_wf_defaults(self):
        """
        Print and return _wf_defaults dictionary. Can be used to easily create set of wf_parameters.
        returns _wf_defaults
        """

        print('Version of workflow: {}'.format(self._workflowversion))
        return self._wf_default

    @classmethod
    def define(cls, spec):
        """
        Defines the outline of the workflow
        """

        # Take input of the workflow or use defaults defined above
        super(kkr_flex_wc, cls).define(spec)

        spec.input("kkr", valid_type=Code, required=True)
        spec.input("options", valid_type=Dict, required=False,
                       default=Dict(dict=cls._options_default))
        spec.input("wf_parameters", valid_type=Dict, required=False)
        spec.input("remote_data", valid_type=RemoteData, required=True)
        spec.input("impurity_info", valid_type=Dict, required=True)
        spec.input("params_kkr_overwrite", valid_type=Dict, required=False,
                   help="Set some input parameters of the KKR calculation.")

        # Here the structure of the workflow is defined
        spec.outline(
            cls.start,
            if_(cls.validate_input)(
                cls.set_params_flex,
                # TODO: encapsulate this in restarting mechanism (should be a base class of workflows that start calculations)
                # i.e. use base_restart_calc workchain as parent
                cls.get_flex), # calculate host GF and kkr-flexfiles
            cls.return_results)

        # ToDo: improve error codes
        spec.exit_code(101, 'ERROR_INVALID_INPUT_IMP_INFO',
            message="ERROR: the 'impurity_info' input Dict node could not be used")
        spec.exit_code(102, 'ERROR_INVALID_INPUT_KKR',
            message="ERROR: the code you provided for kkr does not use the plugin kkr.kkr")
        spec.exit_code(103, 'ERROR_INVALID_INPUT_REMOTE_DATA',
            message="ERROR: No remote_data was provided as Input")
        spec.exit_code(104, 'ERROR_INVALID_CALC_PARAMETERS',
            message='ERROR: calc_parameters given are not consistent! Hint: did you give an unknown keyword?')
        spec.exit_code(105, 'ERROR_CALC_PARAMETERS_INCOMPLETE',
            message='ERROR: calc_parameters misses keys')
        spec.exit_code(106, 'ERROR_KKR_CALCULATION_FAILED',
            message='ERROR: KKR calculation to write out kkrflex files unsuccessful')

        # specify the outputs
        #spec.output('remote_folder', valid_type=RemoteData)
        spec.output('workflow_info', valid_type=Dict)
        spec.output('GF_host_remote', valid_type=RemoteData)

    def start(self):
        """
        init context and some parameters
        """

        self.report('INFO: started KKR flex workflow version {}'
                    ''.format(self._workflowversion))

        ####### init #######
        # internal para / control para
        self.ctx.abort = False

        # input both wf and options parameters
        options_dict = self.inputs.options.get_dict()
        if 'wf_parameters' in self.inputs:
            wf_dict = self.inputs.wf_parameters.get_dict()
            if wf_dict == {}:
                wf_dict = self._wf_default
                self.report('INFO: using default wf parameters')
        else:
            wf_dict = self._wf_default
            self.report('INFO: using default wf parameters')

        if options_dict == {}:
            options_dict = self._options_default
            self.report('INFO: using default options parameters')

        # set values, or defaults
        # ToDo: arrange option assignment differently (look at scf.py from aiida-fleur)
        self.ctx.use_mpi = options_dict.get('use_mpi', self._options_default['use_mpi'])
        self.ctx.resources = options_dict.get('resources', self._options_default['resources'])
        self.ctx.max_wallclock_seconds = options_dict.get('max_wallclock_seconds', self._options_default['max_wallclock_seconds'])
        self.ctx.queue = options_dict.get('queue_name', self._options_default['queue_name'])
        self.ctx.custom_scheduler_commands = options_dict.get('custom_scheduler_commands', self._options_default['custom_scheduler_commands'])

        self.ctx.ef_shift = wf_dict.get('ef_shift', self._wf_default['ef_shift'])
        self.ctx.dos_run = wf_dict.get('dos_run', self._wf_default['dos_run'])
        self.ctx.dos_params_dict = wf_dict.get('dos_params', self._wf_default['dos_params'])

        self.ctx.description_wf = self.inputs.get('description', self._wf_description)
        self.ctx.label_wf = self.inputs.get('label', self._wf_label)

        self.report('INFO: use the following parameter:\n'
                    'use_mpi: {}\n'
                    'Resources: {}\n'
                    'Walltime (s): {}\n'
                    'queue name: {}\n'
                    'scheduler command: {}\n'
                    'description: {}\n'
                    'label: {}\n'.format(self.ctx.use_mpi, self.ctx.resources, self.ctx.max_wallclock_seconds,
                                         self.ctx.queue, self.ctx.custom_scheduler_commands,
                                         self.ctx.description_wf, self.ctx.label_wf))

        # return para/vars
        self.ctx.successful = True
        self.ctx.errors = []
        self.ctx.formula = ''

    def validate_input(self):
        """
        Validate input
        """

        inputs = self.inputs
        input_ok = True

        if not 'impurity_info' in inputs:
            input_ok = False
            return self.exit_codes.ERROR_INVALID_INPUT_IMP_INFO

        if 'remote_data' in inputs:
            input_ok = True
        else:
            input_ok = False
            return self.exit_codes.ERROR_INVALID_REMOTE_DATA

        # extract correct remote folder of last calculation if input remote_folder node
        # is not from KKRCalculation but kkr_scf_wc workflow
        input_remote = self.inputs.remote_data
        # check if input_remote has single KKRCalculation parent
        parents = input_remote.get_incoming(node_class=CalcJobNode)
        nparents = len(parents.all_link_labels())
        if nparents!=1:
            # extract parent workflow and get uuid of last calc from output node
            parent_workflow = input_remote.inputs.last_RemoteData
            if not isinstance(parent_workflow, WorkChainNode):
                raise InputValidationError("Input remote_data node neither output of a KKR calculation nor of kkr_scf_wc workflow")
                parent_workflow_out = parent_workflow.outputs.output_kkr_scf_wc_ParameterResults
                uuid_last_calc = parent_workflow_out.get_dict().get('last_calc_nodeinfo').get('uuid')
                last_calc = load_node(uuid_last_calc)
                if not isinstance(last_calc, KkrCalculation):
                    raise InputValidationError("Extracted last_calc node not of type KkrCalculation: check remote_data input node")
                # overwrite remote_data node with extracted remote folder
                output_remote = last_calc.outputs.remote_folder
                self.inputs.remote_data = output_remote

        if 'kkr' in inputs:
            try:
                test_and_get_codenode(inputs.kkr, 'kkr.kkr', use_exceptions=True)
            except ValueError:
                error = ("The code you provided for kkr does not "
                         "use the plugin kkr.kkr")
                self.ctx.errors.append(error)
                input_ok = False
                return self.exit_codes.ERROR_INVALID_INPUT_KKR

        # set self.ctx.input_params_KKR
        self.ctx.input_params_KKR = get_parent_paranode(self.inputs.remote_data)

        if input_ok:
            self.report('INFO: checking inputs successful')

        return input_ok

    def set_params_flex(self):
        """
        Take input parameter node and change to input from wf_parameter and options
        """

        self.report('INFO: setting parameters ...')

        params = self.ctx.input_params_KKR
        input_dict = params.get_dict()
        para_check = kkrparams()

        # step 1: try to fill keywords
        try:
            for key, val in input_dict.items():
                para_check.set_value(key, val, silent=True)
        except:
            return self.exit_codes.ERROR_INVALID_CALC_PARAMETERS

        # step 2: check if all mandatory keys are there
        label = ''
        descr = ''
        missing_list = para_check.get_missing_keys(use_aiida=True)
        if missing_list != []:
            kkrdefaults = kkrparams.get_KKRcalc_parameter_defaults()[0]
            kkrdefaults_updated = []
            for key_default, val_default in list(kkrdefaults.items()):
                if key_default in missing_list:
                    para_check.set_value(key_default, kkrdefaults.get(key_default), silent=True)
                    kkrdefaults_updated.append(key_default)
                    missing_list.remove(key_default)
            if len(missing_list)>0:
                self.report('ERROR: calc_parameters misses keys: {}'.format(missing_list))
                return self.exit_codes.ERROR_CALC_PARAMETERS_INCOMPLETE

            else:
                self.report('updated KKR parameter node with default values: {}'.format(kkrdefaults_updated))
                label = 'add_defaults_'
                descr = 'added missing default keys, '

        # updatedict contains all things that will be updated with update_params_wf later on
        updatedict = {}

        runopt = para_check.get_dict().get('RUNOPT', None)
        if runopt == None:
            runopt = []

        # overwrite some parameters of the KKR calculation by hand before setting mandatory keys
        if "params_kkr_overwrite" in self.inputs:
            for key, val in self.inputs.params_kkr_overwrite.get_dict().items():
                if key != runopt:
                    updatedict[key] = val
                else:
                    runopt = val
                self.report('INFO: overwriting KKR parameter: {} with {} from params_kkr_overwrite input node'.format(key, val))

        runopt = [i.strip() for i in runopt]
        if 'KKRFLEX' not in runopt:
            runopt.append('KKRFLEX')

        updatedict['RUNOPT'] = runopt

        self.report('INFO: RUNOPT set to: {}'.format(runopt))

        if 'wf_parameters' in self.inputs:
            if self.ctx.dos_run:
                for key, val in {'EMIN': self.ctx.dos_params_dict['emin'], 'EMAX': self.ctx.dos_params_dict['emax'],
                                 'NPT2': self.ctx.dos_params_dict['nepts'], 'NPOL': 0, 'NPT1': 0, 'NPT3': 0,
                                 'BZDIVIDE': self.ctx.dos_params_dict['kmesh'], 'IEMXD': self.ctx.dos_params_dict['nepts'],
                                 'TEMPR': self.ctx.dos_params_dict['tempr']}.items():
                    updatedict[key] = val
            elif self.ctx.ef_shift != 0:
                # extract old Fermi energy in Ry
                remote_data_parent = self.inputs.remote_data
                parent_calc = remote_data_parent.get_incoming(link_label_filter='remote_folder').first().node
                ef_old = parent_calc.outputs.output_parameters.get_dict().get('fermi_energy')
                # get Fermi energy shift in eV
                ef_shift = self.ctx.ef_shift #set new E_F in eV
                # calculate new Fermi energy in Ry
                ef_new = (ef_old + ef_shift/get_Ry2eV())
                self.report('INFO: ef_old + ef_shift = ef_new: {} eV + {} eV = {} eV'.format(ef_old*get_Ry2eV(), ef_shift, ef_new*get_Ry2eV()))
                updatedict['ef_set'] = ef_new

        #construct the final param node containing all of the params
        updatenode = Dict(dict=updatedict)
        updatenode.label = label+'KKRparam_flex'
        updatenode.description = descr+'KKR parameter node extracted from parent parameters and wf_parameter and options input node.'
        paranode_flex = update_params_wf(self.ctx.input_params_KKR, updatenode)
        self.ctx.flex_kkrparams = paranode_flex
        self.ctx.flex_runopt = runopt

        self.report('INFO: Updated params= {}'.format(paranode_flex.get_dict()))

    def get_flex(self):
        """
        Submit a KKRFLEX calculation
        """

        label = 'KKRFLEX calc.'
        description = 'KKRFLEX calculation to write out host GF'
        code = self.inputs.kkr
        remote = self.inputs.remote_data
        params = self.ctx.flex_kkrparams
        imp_info = self.inputs.impurity_info
        options = {"max_wallclock_seconds": self.ctx.max_wallclock_seconds,
                   "resources": self.ctx.resources,
                   "queue_name": self.ctx.queue}
        if self.ctx.custom_scheduler_commands:
            options["custom_scheduler_commands"] = self.ctx.custom_scheduler_commands
        inputs = get_inputs_kkr(code, remote, options, label, description, parameters=params, serial=(not self.ctx.use_mpi), imp_info=imp_info)

        # run the KKRFLEX calculation
        self.report('INFO: doing calculation')
        flexrun = self.submit(KkrCalculation, **inputs)

        return ToContext(flexrun=flexrun)

    def return_results(self):
        """
        Return the results of the KKRFLEX calculation.
        This should run through and produce output nodes even if everything failed,
        therefore it only uses results from context.
        """

        # capture error of unsuccessful flexrun
        if not self.ctx.flexrun.is_finished_ok:
            self.ctx.successful = False
            return self.exit_codes.ERROR_KKR_CALCULATION_FAILED

        # create dict to store results of workflow output
        outputnode_dict = {}
        outputnode_dict['workflow_name'] = self.__class__.__name__
        outputnode_dict['workflow_version'] = self._workflowversion
        outputnode_dict['use_mpi'] = self.ctx.use_mpi
        outputnode_dict['resources'] = self.ctx.resources
        outputnode_dict['max_wallclock_seconds'] = self.ctx.max_wallclock_seconds
        outputnode_dict['queue_name'] = self.ctx.queue
        outputnode_dict['custom_scheduler_commands'] = self.ctx.custom_scheduler_commands
        outputnode_dict['successful'] = self.ctx.successful
        outputnode_dict['pk_flexcalc'] = self.ctx.flexrun.pk
        outputnode_dict['list_of_errors'] = self.ctx.errors

        outputnode = Dict(dict=outputnode_dict)
        outputnode.label = 'kkr_flex_wc_results'
        outputnode.description = ''
        outputnode.store()

        # return the input remote_data folder as output node
        #self.out('remote_data', self.inputs.remote_data)
        # return Dict node containing information about previous calculation
        self.out('workflow_info', outputnode)
        # return retrieved data from kkrflex calculation
        self.out('GF_host_remote', self.ctx.flexrun.outputs.remote_folder)

        self.report('INFO: created GF writeout result nodes')

#        self.report("INFO: create GF writeout results nodes: outputnode={}".format(outputnode))
#        try:
#            self.report("INFO: create GF writeout results nodes. KKRFLEX calc retrieved node={}".format(self.ctx.flexrun.outputs.retrieved))
#            has_flexrun = True
#        except AttributeError as e:
#            self.report("ERROR: no KKRFLEX calc retrieved node found")
#            self.report("Caught AttributeError {}".format(e))
#            has_flexrun = False

        #for link_name, node in outdict.iteritems():
            #self.report("INFO: storing node '{}' with link name '{}'".format(node, link_name))
            #self.report("INFO: node type: {}".format(type(node)))
            #self.out(link_name, node)

        self.report("INFO: done with KKRFLEX GF writeout workflow!\n")
#        self.report("Successful run: {}".format(has_flexrun))


import sys
import os

def getHeaders(read_file):

    with open(read_file, "r") as rf:
        with open("temp.txt", "w+") as wf:

            if rf.mode == "r":
                lines = rf.readlines()

                wf.write("\n")
                for x in lines:
                    if(x[0:3] == "## "):
                        wf.write("- " + x[3:])
                    if(x[0:4] == "### "):
                        wf.write("\t- " + x[4:])
                    if(x[0:5] == "#### "):
                        wf.write("\t\t- " + x[5:])
                    if(x[0:6] == "##### "):
                        wf.write("\t\t\t- " + x[6:])
                wf.write("\n")

    rf.close()
    wf.close()

if __name__ == "__main__":

    if len(sys.argv) != 2:
        raise Exception("You've entered the wrong number of command line arguments. This script takes 1 command line argument")

    read_file = sys.argv[1]
    getHeaders(read_file)


"""
* Author:  Michael Barber
* File:    wine_quality.py
* Purpose: This module reads in data file(s) of wine characteristics obtained
           from <http://archive.ics.uci.edu/ml/datasets/Wine+Quality>,
           standardizes the independent variables,

"""

import numpy as np
import matplotlib.pyplot as plt
import csv
import sys
import random
from linear_lib import *

def main():
    # Red wine data file
    red_file = 'data/winequality-red.csv'
    # White wine data file
    white_file = 'data/winequality-white.csv'

    # open the red wine data file
    original_data = open_file(red_file)

    # prepare the data for the linear model w./ 80% distribution
    (training_set_ind, training_set_dep, \
            testing_set_ind, testing_set_dep, \
            dependent_name, independent_names) = prepareDataForModel(original_data, 0.95)

    # make the linear model
    weights = makeLLS(training_set_ind, training_set_dep)

    # print the weights of the training data
    printWeights(weights, independent_names)

    # use the linear model of the training set to predict
    # the quality of the testing set
    prediction = useLLS(weights, testing_set_ind)

    # plot the prediction versus observed
    plotPredictions(prediction, testing_set_dep)

    # calculate the root-mean-square errors (RMSE)
    error = calculateRMSE(weights, testing_set_ind, testing_set_dep)

    print("error: "+str(error))

#prefer to explicitly call my main()
if __name__ == '__main__':
    main()


# coding=utf-8
# /usr/bin/env python

"""
Author: ranlaysu
date: 2019/7/5 17:09
desc: 代理proxy handler
"""
import urllib2

# 代理开关，表示是否开启代理
proxy_swith = False
# 构建一个Hander处理器对象，参数是一个dict，包括代理类型(http/https)和代理服务器IP+port
httpproxy_handler = urllib2.ProxyHandler({"http":"117.91.131.54:9999"})
# 构建一个没有代理的Handler处理器对象
nullproxy_handler = urllib2.ProxyHandler({})

if proxy_swith:
	opener = urllib2.build_opener(httpproxy_handler)
else:
	opener = urllib2.build_opener(nullproxy_handler)

# 构建一个全局的opener，之后所有的请求都可以用到urlopen()方法去发送，也附带Handler功能
urllib2.install_opener(opener)

request = urllib2.Request('http://www.baidu.com')
response = urllib2.urlopen(request)
print(response.read())


from __future__ import print_function
from myo.myo_raw import MyoRaw
import sys
import time
import uuid
import h5py
import numpy as np
import csv
import os
import math
import threading

from sklearn.externals import joblib
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.model_selection import cross_val_score
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import GradientBoostingClassifier

#----------------------------------------------------------------------------------------------
#下载特征值极值，用于在线特征值归一化；
#下载训练好的模型
featureFolder = os.getcwd() + "\\DataSets\\" + \
				os.listdir(os.getcwd() + "\\DataSets")[len(os.listdir(os.getcwd() + "\\DataSets"))-1]#[]中表示找到DataSets下的最后一个文件夹
print('featureFolder-->'+str(featureFolder))
featureMax = np.loadtxt(featureFolder+'//featureMax.txt')
featureMin = np.loadtxt(featureFolder+'//featureMin.txt')
model_RF  = joblib.load("model//model_RF.m")
model_SVM = joblib.load("model//model_SVM.m")
model_KNN = joblib.load("model//model_KNN.m")
#----------------------------------------------------------------------------------------------
#全局变量以及特征函数
windowDataLength = 20 #时间窗，与训练模型时保持一致
slideDataLength  = 4  #滑动窗
emg_data  = []		  #存储 windowDataLength+slideDataLength 行肌电数据
stop_flag = 0
# 3个特征函数
def featureRMS(data):
    return np.sqrt(np.mean(data**2, axis=0))
def featureMAV(data):
    return np.mean(np.abs(data), axis=0)
def featureWL(data):
    return np.sum(np.abs(np.diff(data, axis=0)),axis=0)/data.shape[0]

#----------------------------------------------------------------------------------------------
#肌电数据格式处理
def proc_emg(emg, moving):
	global emg_data
	timeStrick = time.time()	#time.time() 返回当前时间的时间戳
	emg = np.append(emg ,timeStrick)
	emg_data = np.append(emg_data, emg)

global SVM_prediction

SVM_prediction=np.array([4.])

#----------------------------------------------------------------------------------------------
#处理肌电信号
###  mMyo.hnd = EMGHandler(mMyo.m, mMyo.storedPath) #实例化对象mMyo.hnd
# self == mMyo.hnd
class EMGHandler(object):
	def __init__(self, m, storedPath):
		self.recording = -1
		self.dtype = np.int16
		self.m = m
		self.storedPath = storedPath
		self.rawLength = 0
	# def return_label(self):
		# return
		# self.__call__.RF_prediction,\
		# self.__call__.SVM_prediction,\
		# self.__call__.KNN_prediction

	def __call__(self, emg, moving):
		self.emg = emg
		global RF_prediction,SVM_prediction,KNN_prediction
		global emg_data, featureMax, featureMin, windowDataLength, slideDataLength
		if self.recording >= 0:
			if len(emg_data) != 0:
				emg_data = emg_data.reshape((int(len(emg_data)/9),9))#int(len(emg_data)/9)行，9列
				self.rawLength = emg_data.shape[0]
			if self.rawLength >= windowDataLength+slideDataLength:
				#达到24行数据，提取后20行，提取特征值分类
				emg_data = emg_data[slideDataLength:windowDataLength+slideDataLength,:]
				feature = []
				for i in range(8):
					data = emg_data[:,i]
					############测试###################
					# print("emg_data[:,%i] " % i)
					# print("data is :",data)
					############测试###################
					RMS  = featureRMS(data)
					MAV  = featureMAV(data)
					WL   = featureWL(data)
					feature = np.append(feature, (RMS, MAV, WL))
					############测试###################
					# print("feature is :\t\n",feature)
					############测试###################
				feature = feature.reshape((1,24))
				# 归一化后进行预测
				feature = ((feature-featureMin)/(featureMax-featureMin))[0]
				############测试###################
				# print("feature is :\t\n",feature)
				#########测试###################
				for i in range(24):
					if(feature[i]>1):
						feature[i] = 1
					elif(feature[i]<0):
						feature[i] = 0
				feature = feature.reshape((1,24))
				#########测试###################
				# print("feature is :\t\n",feature)
				#########测试###################
				RF_prediction = model_RF.predict(feature)
				SVM_prediction = model_SVM.predict(feature)
				KNN_prediction = model_KNN.predict(feature)
				print('RF_prediction-->'+str(RF_prediction)+' , '+'SVM_prediction-->'+str(SVM_prediction)+' , '+'KNN_prediction-->'+str(KNN_prediction))
				feature = []#单次判断结束，置空，便于下次处理
				# return RF_prediction,SVM_prediction,KNN_prediction
		else:
			feature  = []
			emg_data = []

##mMyo = myMyo(storedPath='online\\')	#实例化 myMyo 类
class myMyo(object):
	# def __init__(self):
	def __init__(self, storedPath):
		# self.m =  MyoRaw(sys.argv[1] if len(sys.argv) >= 2 else None)
		self.m =  MyoRaw()
		print("1")
		#from myo.myo_raw import MyoRaw #实例化对象self.m
		self.storedPath = storedPath
		self.hnd = EMGHandler(self.m, self.storedPath)#实例化对象self.hnd
###########################################################################
		# self.hnd.return_label()
###########################################################################
		print("2")
		self.m.add_emg_handler(self.hnd)#add_emg_handler 是MyoRaw类下的一个方法
		print("3")
		self.m.add_emg_handler(proc_emg)
	def start(self):
		self.hnd.recording = 1 #recording类属性，开始标志符为 1
		print('recording flag --> ' + str(self.hnd.recording))
		self.m.connect()#connect 是MyoRaw类下的一个方法
	def run(self):
		self.m.run()
	def stop(self):
		self.hnd.recording = -1
		print('recording flag --> ' + str(self.hnd.recording))
		self.m.disconnect()

def waitForStop():
	time.sleep(10)
	stop_flag = 1

if __name__ == '__main__':
    mMyo = myMyo(storedPath='online\\')	#实例化 myMyo 类
    threadWaitForStop = threading.Thread(target = waitForStop) #创建线程。
    threadWaitForStop.start()#方法threading.Thread.start():启动线程。

    mMyo.start()
    print("myo is runing.....")
    while (stop_flag == 0):
        mMyo.run()
    threadWaitForStop.join()#等待线程终止

    mMyo.stop()
    print('end')


from __future__ import division, absolute_import, print_function
import sys
import os
from os.path import abspath, isdir, join, dirname, basename
import shutil
import glob

from ..utils import ensure_dir

def _noop(*args, **kwargs):
    pass

def make_data_package(pandeia_refdata=None, destination=None, log=_noop, overwrite=False):
    if pandeia_refdata is None:
        if os.environ.get('pandeia_refdata'):
            pandeia_refdata = os.environ.get('pandeia_refdata')
        else:
            raise RuntimeError("Supply pandeia_refdata path or set $pandeia_refdata")
    if destination is None:
        destination = abspath('./pandeia_wfirst_data')
    if isdir(destination) or len(glob.glob(join(destination, '*'))) != 0:
        if not overwrite:
            raise RuntimeError("Remove existing folder ({}) "
                               "or pass overwrite=True".format(destination))
        shutil.rmtree(destination)
    destination = ensure_dir(destination)

    # Populate only WFIRST and mission-independent data files
    for folder_name in ('wfirst', 'source', 'normalization', 'sed', 'strategy',
                        'background', 'extinction'):
        shutil.copytree(join(pandeia_refdata, folder_name), join(destination, folder_name))

    # Patch the data package's config file for WFIRST WFI
    shutil.copy(join(dirname(__file__), 'data_patch', 'wfirstimager_config.json'),
                join(destination, 'wfirst', 'wfirstimager', 'config.json'))

    log("New data package in", destination)
    return destination

if __name__ == "__main__":
    # Make package
    data_dir = make_data_package(log=print, overwrite=True)

    # Test package
    os.environ['pandeia_refdata'] = data_dir
    from pandeia.engine.perform_calculation import perform_calculation
    from pandeia.engine.calc_utils import build_default_calc
    calc = build_default_calc("wfirst", "wfirstimager", "imaging")
    result = perform_calculation(calc, dict_report=True)
    print("Successfully performed the default calculation for wfirstimager")
    if sys.version_info > (3, 2):
        archive_name = shutil.make_archive('pandeia_wfirst_data', 'gztar',
                                           root_dir=dirname(data_dir),
                                           base_dir=basename(data_dir))
        print("Compressed WFIRST Pandeia data into {}".format(archive_name))


"""
    :module:: analyze
    :platform: Unix, Windows
    :synopsis: Analyze data from info dir and export distributions to list dir
    moduleauthor:: Ricardo Silveira <ricardosilveira@poli.ufrj.br>
"""
class Person(object):

  def __init__(self):
    self._name = ''
    self._group = ''
    self._link = ''
    self._screen_name = ''
    self._lang = ''
    self._twitterId = 0
    self._followers_count = 0
    self._listed_count = 0
    self._statuses_count = 0
    self._friends_count = 0
    self._favourites_count = 0
    self._url = ''
    self._created_at = ''
    self._time_zone = ''

  #Receives the json string containing user information
  def loadFromJSON(self, info, group, subgroup):
    self._name = info["name"]
    self._group = group
    self._subgroup = subgroup
    self._screen_name = info['screen_name']
    self._lang = info['lang']
    self._twitterId = info['id']
    self._followers_count = info['followers_count']
    self._listed_count = info['listed_count']
    self._statuses_count = info['statuses_count']
    self._friends_count = info['friends_count']
    self._favourites_count = info['favourites_count']
    self._url = info['url']
    self._created_at = info['created_at']
    self._time_zone = info['time_zone']

  def __str__(self):
    ret = str(self._group)+';'+str(self._subgroup)+';'+str(self._name)+';'+str(self._screen_name)+';'+str(self._lang)+';'
    ret = ret+str(self._twitterId)+';'+str(self._followers_count)+';'+str(self._listed_count)+';'
    ret = ret+str(self._statuses_count)+';'+str(self._friends_count)+';'+str(self._favourites_count)+';'
    ret = ret+str(self._url)+';'+str(self._created_at)+';'+str(self._time_zone)+'\n'
    return ret


'''四.创建一个tcp服务器
（1）能接收tcp客户端发来的请求
（2）能并发处理客户端连接
（3）能接收tcp客户端发来的数据、每次收到数据打印出来并dA回复客户端“已收到！谢谢！”、直到客户端主动断开（线程版）'''
import socket
tcp_socket=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# 2、需要使用bind来绑定端口
tcp_socket.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,True)
tcp_socket.bind(" ",5000)
tcp_socket.send('123'.encode('gbk'),('192.168.44.1',8080))
tcp_socket.listen()

0416/考试/新建文件夹/编码题/第四题.py


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras import metrics
from sklearn.preprocessing import MinMaxScaler

data = pd.read_csv("GOOG.csv")

L = len(data)

print(L)

high = np.array([data.iloc[:,2]])
low = np.array([data.iloc[:,3]])
close = np.array([data.iloc[:,4]])

H, = plt.plot(high[0,:])
L, = plt.plot(low[0,:])
C, = plt.plot(close[0,:])

plt.legend([H,L,C],["High","Low","Close"])
plt.show(block=False)

X = np.concatenate([high,low],axis=0)

X = np.transpose(X)
plt.plot(X)

Y = close
Y = np.transpose(Y)
plt.plot(Y)

scaler = MinMaxScaler()
scaler.fit(X)
X = scaler.transform(X)

scaler1 = MinMaxScaler()
scaler1.fit(Y)
Y = scaler1.transform(Y)

X = np.reshape(X,(X.shape[0],1,X.shape[1]))
print(X.shape)

model = Sequential()
model.add(LSTM(100,activation="tanh",input_shape=(1,2),recurrent_activation="hard_sigmoid"))
model.add(Dense(1))

model.compile(loss='mean_squared_error',optimizer="rmsprop",metrics=[metrics.mae])
newfit = model.fit(X,Y,epochs=15,batch_size=50,verbose=2)

Predict = model.predict(X,verbose=1)
print(Predict)

plt.figure(2)
plt.scatter(Y,Predict)
plt.show(block=False)
plt.figure(3)

Test, = plt.plot(Y)

Predict, = plt.plot(Predict)

plt.legend([Predict, Test],["Predicted Data","Real Data"])
plt.show()


'''
Created on Jun 29, 2014

@author: Federico Reghenzani <federico.dev@reghe.net>
'''

import web.login
import threading
import web.bottle

class HttpServer(threading.Thread):
    '''
    classdocs
    '''

    urls = (
            '/' , 'index'
            )

    @staticmethod
    def start_server(ip, port):
        hs = HttpServer(ip, port)
        hs.start()

    def __init__(self, ip : str, port : int):
        '''
        Constructor for http server object. This do nothing: save only
        port and ip address
        '''
        threading.Thread.__init__(self)
        self._ip = ip;
        self._port = port;

    def run(self) -> None:
        '''
        The main thread execution: it manages all requests from web
        '''

        web.bottle.run(host=self._ip, port=self._port, debug=True)


# encoding:utf-8

import re
from conf.field_conf import PHONE_PREFIX
from conf.code_msg import CODE_MSG_DICT
from tools.filtering_char import automatic_filtering_invisible_char
import datetime
from conf.time_conf import DATETIME_FORMAT_STR

class CheckField(object):
    def __init__(self, *args, **kwargs):
        self.phone_prefix = PHONE_PREFIX
        self.code_msg = CODE_MSG_DICT

    @classmethod
    def instance(cls, *args, **kwargs):
        """
        :param args:
        :param kwargs:
        :return:
        """
        if not hasattr(CheckField, "_instance"):
            CheckField._instance = CheckField(*args, **kwargs)
        return CheckField._instance

    # 看看是不是一个手机号码，11为，纯数字，前面是三大运营商的
    def is_telephon(self, telephone):
        """
        :param telephone：传递过来的是只包含可见字符的字符串
        :return: (code, msg)
        """
        # 检测号码是否为空。
        if telephone is None:
            raise ValueError('手机号码不能为空')

        # 判断类型
        if not isinstance(telephone, str):
            raise TypeError('手机号码只能是数字字符串')

        # 吸收掉空格等字符
        telephone = automatic_filtering_invisible_char(telephone)
        if len(telephone) != 11:
            raise ValueError('手机号码必须为11位')
        else:
            # 检查是不是全部为数字
            if telephone.isdigit():
                # 检查是不是包含在三大运营商提供的前缀里面
                if telephone[:3] in self.phone_prefix:
                    return telephone
                else:
                    raise ValueError('该手机号码不在三大运营商之内')
            else:
                # 手机号码应该为数字字符串
                raise TypeError('手机号码只能是数字字符串')

    # 检查是不是合法是用户名
    def is_username(self, username):
        """
        :param username: 只能是可见字符
        :return: (code, msg)
        """
        # 空检查
        if username is None:
            # 用户名不能为空
            raise TypeError('用户名不能为空')
            # return 121, self.code_msg['121']
        # 类型检查
        if not isinstance(username, str):
            # 用户名必须为字符串
            raise TypeError('用户名必须为字符串')
            # return 122, self.code_msg['122']
        # 空串检查
        username = automatic_filtering_invisible_char(username)
        if len(username) == 0:
            # 用户名不能为空
            raise ValueError('用户名不能为空')
            # return 121, self.code_msg['121']
        # 长度检查
        if len(username) > 16 or len(username) < 2:
            # 用户名长度必须大于2位并且小于16位!!!
            raise ValueError('用户名长度必须大于2位并且小于16位!!!')
            # return 123, self.code_msg['123']
        else:
            return username

    # 是不是 一个合法的 email
    def is_email(self, email):
        """
        邮箱检查
        :param email:
        :return:
        """
        # 空检查
        if email is None:
            # 邮箱不能为空
            raise TypeError('邮箱不能为空')
            # return 141, self.code_msg['141']
        # 类型检查
        if not isinstance(email, str):
            # 邮箱必须为字符串
            raise TypeError('邮箱必须为字符串')
            # return 142, self.code_msg['142']
        # 空串检查
        email = automatic_filtering_invisible_char(email)
        if len(email) == 0:
            # 邮箱不能为空
            raise ValueError('邮箱不能为空')
            # return 141, self.code_msg['141']
        # 邮箱合法性检查
        email_rule = r'^[0-9a-zA-Z\_\-]+(\.[0-9a-zA-Z\_\-]+)*@[0-9a-zA-Z]+(\.[0-9a-zA-Z]+){1,}$'
        if not re.match(email_rule, email):
            # 非法邮箱
            raise ValueError('非法邮箱')
            # return 143, self.code_msg['141']
        return email

    # 订单编号 必须大于0
    def is_order_id(self, order_id):
        """
        检查订单id
        :param order_id:
        :return: 失败只返回2个参数，成功返回3个
        """
        # 空检查
        if order_id is None:
            # 订单编号不能为空
            raise TypeError('订单编号不能为空')
            # return 161, self.code_msg['161']
        # 类型检查
        if not isinstance(order_id, int):
            # 订单编号必须为数字
            raise TypeError('订单编号必须为数字')
            # return 162, self.code_msg['162']

        # 不能小于 0
        if order_id < 0:
            # 订单编号必须 >= 0
            raise ValueError('订单编号必须 >= 0')
            # return 163, self.code_msg['163']
        return order_id

    # 检查是不是合法的时间周期，必须大于1
    def is_time_interval(self, time_interval):
        """
        检查时间周期
        :param time_interval:
        :return: (code, msg, [time_interval])
        180: (code, msg, time_interval)

        """
        # 空检查
        if time_interval is None:
            # 时间周期不能为空
            raise TypeError('时间周期不能为空')
            # return 181, self.code_msg['181']
        # 类型检查
        if not isinstance(time_interval, int):
            # 时间周期必须为数字
            raise TypeError('时间周期必须为数字')
            # return 182, self.code_msg['182']

        # 不能小于 1
        if int(time_interval) < 1:
            # 时间周期必须 >= 1
            raise ValueError('时间周期必须 >= 1')
            # return 183, self.code_msg['183']
        return time_interval

    # 检查是不是 标准的 时间字符串
    def is_time_str(self, time):
        """
        时间检查
        :param time:
        :return:  (code, msg [timestamp])
        200 正常, 后面携带timestamp,类型为 datetime
        1. 为空 201
        2. 不是字符串类型 202
        3. 不是标准时间字符串 203
        """
        # 空检查
        if time is None:
            # 时间不能为空
            raise TypeError('时间不能为空')
            # return 201, self.code_msg['201']
        # 类型检查
        if not isinstance(time, str):
            # 时间必须为字符串类型
            raise TypeError('时间必须为字符串类型')
            # return 202, self.code_msg['202']
        # 空串检查
        time = time.strip()
        if len(time) == 0:
            # 时间不能为空
            raise ValueError('时间不能为空')
            # return 201, self.code_msg['201']
        try:
            # 检查是否为标准时间字符串
            # print(time)
            timestamp = datetime.datetime.strptime(time, DATETIME_FORMAT_STR)
        except ValueError as err:
            # 时间必须为'%Y-%m-%d %H:%M:%S'格式例如'2011-11-11 11:11:11'
            raise ValueError("时间必须为'%Y-%m-%d %H:%M:%S'格式例如'2011-11-11 11:11:11'")
            # return 203, self.code_msg['203']
        else:
            return timestamp

    # 是一个 数字字符串
    def is_int_str(self, int_str):
        if not isinstance(int_str, (str, int)):
            raise TypeError('类型错误')
        else:
            try:
                temp = int(int_str)
            except ValueError as err:
                raise ValueError('应该为数字字符串或者是数字, 不能包含其他字符')
            else:
                return temp

    # 确定为一个url字符串
    def is_url(self, url):
        if not isinstance(url, str):
            raise TypeError('url类型错误')
        url = automatic_filtering_invisible_char(url)
        if len(url) < 5:
            raise ValueError('url值错误')
        else:
            return url

    # 确定是字符串，并且只能是 中性，负面，正面
    def is_attribute(self, attribute):
        if not isinstance(attribute, str):
            raise TypeError('类型错误')
        else:
            attribute = automatic_filtering_invisible_char(attribute)
            if attribute == '正面':
                return attribute
            elif attribute == '中性':
                return attribute
            elif attribute == '负面':
                return attribute
            else:
                raise ValueError('属性必须3种属性中之一')

    # 返回一个吸收掉空格和控制符的字符串
    def is_str(self, china_str):
        if china_str is None:
            raise ValueError('关键字段不能为空')
        if not isinstance(china_str, str):
            raise TypeError('关键字段应该为字符串')
        china_str = automatic_filtering_invisible_char(china_str)
        if len(china_str) < 1:
            raise ValueError('非法关键字')
        return china_str

    # 监测排序参数是不是  time, 或者是 score
    def is_sort_rule(self, sort_key):
        if not isinstance(sort_key, str):
            raise TypeError('排序参数必须为字符串')
        if sort_key == 'time' or sort_key == 'score':
            return sort_key
        else:
            raise ValueError('排序参数目前只支持 "time" or "score"')


import h5py
import numpy as np
import PIL
import subprocess
import os
from PIL import Image
import sys
from tifffile import imread, TiffFile
from bfio import BioReader, BioWriter, LOG4J, JARS
import javabridge, math
from pathlib import Path
from multiprocessing import cpu_count

def rescale(size,img,mode='uint8'):

    if mode == 'float32':
        #for floating point images:
        img = np.float32(img)
        img_PIL = PIL.Image.fromarray(img,mode='F')
    elif mode == 'uint8':
        #otherwise:
        img_PIL = PIL.Image.fromarray(img)
    else:
        raise(Exception('Invalid rescaling mode. Use uint8 or float32'))

    return np.array(img_PIL.resize(size,PIL.Image.BILINEAR))

def normalize(img):

    ###normalize image
    img_min = np.min(img)
    img_max = np.max(img)
    img_centered = img - img_min
    img_range = img_max - img_min
    return np.true_divide(img_centered, img_range)

def unet_segmentation(input_img,img_pixelsize_x,img_pixelsize_y,
                          modelfile_path,weightfile_path,iofile_path,
                          tiling_x=4,tiling_y=4,gpu_flag='0',
                          cleanup=True):

    #fix parameters
    n_inputchannels=1
    n_iterations=0

    ## prepare image rescaling
    np.set_printoptions(threshold=sys.maxsize)

    #get model resolution (element size) from modelfile
    modelfile_h5 = h5py.File(modelfile_path,'r')
    modelresolution_y = modelfile_h5['unet_param/element_size_um'][0]
    modelresolution_x = modelfile_h5['unet_param/element_size_um'][1]
    modelfile_h5.close()
    #get input image absolute size
    abs_size_x = input_img.shape[1] * img_pixelsize_x
    abs_size_y = input_img.shape[0] * img_pixelsize_y
    #get rescaled image size in pixel
    rescaled_size_px_x = int(np.round(abs_size_x / modelresolution_x))
    rescaled_size_px_y = int(np.round(abs_size_y / modelresolution_y))
    rescale_size = (rescaled_size_px_x,rescaled_size_px_y)
    ### preprocess image and store in IO file

    #normalize image, then rescale
    normalized_img = normalize(input_img)
    rescaled_img = np.float32(rescale(rescale_size,normalized_img,mode='float32'))
    #prepending singleton dimensions to get the desired blob structure
    h5ready_img = np.expand_dims(rescaled_img, axis=(0,1))
    iofile_h5 = h5py.File(iofile_path,mode='x')
    iofile_h5.create_dataset('data',data=h5ready_img)
    iofile_h5.close()

    # ### run caffe_unet commands

    # #assemble sanity check command
    command_sanitycheck = []
    command_sanitycheck.append("caffe_unet")
    command_sanitycheck.append("check_model_and_weights_h5")
    command_sanitycheck.append("-model")
    command_sanitycheck.append(modelfile_path)
    command_sanitycheck.append("-weights")
    command_sanitycheck.append(weightfile_path)
    command_sanitycheck.append("-n_channels")
    command_sanitycheck.append(str(n_inputchannels))
    if gpu_flag:
        command_sanitycheck.append("-gpu")
        command_sanitycheck.append(gpu_flag)
     #runs command and puts console output to stdout
    sanitycheck_proc = subprocess.run(command_sanitycheck,stdout=subprocess.PIPE)
    # #aborts if process failed
    sanitycheck_proc.check_returncode()

    #assemble prediction command
    command_predict = []
    command_predict.append("caffe_unet")
    command_predict.append("tiled_predict")
    command_predict.append("-infileH5")
    command_predict.append(iofile_path)
    command_predict.append("-outfileH5")
    command_predict.append(iofile_path)
    command_predict.append("-model")
    command_predict.append(modelfile_path)
    command_predict.append("-weights")
    command_predict.append(weightfile_path)
    command_predict.append("-iterations")
    command_predict.append(str(n_iterations))
    command_predict.append("-n_tiles")
    command_predict.append(str(tiling_x)+'x'+str(tiling_y))
    command_predict.append("-gpu")
    command_predict.append(gpu_flag)
    if gpu_flag:
        command_predict.append("-gpu")
        command_predict.append(gpu_flag)
    #run command
    output = subprocess.check_output(command_predict, stderr=subprocess.STDOUT).decode()
    print(output)

    # load results from io file and return
    output_h5 = h5py.File(iofile_path)
    score = output_h5['score'][:]
    output_h5.close()
    # #get segmentation mask by taking channel argmax
    segmentation_mask = np.squeeze(np.argmax(score, axis=1))
    return segmentation_mask

def read_file(input_directory, pixelsize, output_directory):

    img_pixelsize_x = pixelsize
    img_pixelsize_y = pixelsize
    modelfile_path = "2d_cell_net_v0-cytoplasm.modeldef.h5"
    weightfile_path = "snapshot_cytoplasm_iter_1000.caffemodel.h5"
    iofile_path = "output.h5"
    out_path = Path(output_directory)
    rootdir1 = Path(input_directory)
    """ Convert the tif to tiled tiff """
    javabridge.start_vm(args=["-Dlog4j.configuration=file:{}".format(LOG4J)],
                        class_path=JARS,
                        run_headless=True)
    i = 0
    try:
        for PATH in rootdir1.glob('**/*'):
                tile_grid_size = 1
                tile_size = tile_grid_size * 1024

                # Set up the BioReader
                with BioReader(PATH,backend='java',max_workers=cpu_count()) as br:

                    # Loop through timepoints
                    for t in range(br.T):

                        # Loop through channels
                        for c in range(br.C):

                            with BioWriter(out_path.joinpath(f"final{i}.ome.tif"),metadata = br.metadata, backend='java') as bw:

                                 # Loop through z-slices
                                for z in range(br.Z):

                                    # Loop across the length of the image
                                    for y in range(0,br.Y,tile_size):
                                        y_max = min([br.Y,y+tile_size])

                                        # Loop across the depth of the image
                                        for x in range(0,br.X,tile_size):
                                            x_max = min([br.X,x+tile_size])

                                            input_img = np.squeeze(br[y:y_max,x:x_max,z:z+1,c,t])
                                            img = unet_segmentation(input_img,img_pixelsize_x, img_pixelsize_y,modelfile_path,weightfile_path,iofile_path)
                                            bw[y:y_max, x:x_max,...] = img
                                            os.remove("output.h5")

                i+=1

    finally:
        # Close the javabridge. Since this is in the finally block, it is always run
        javabridge.kill_vm()


from typing import List, Sequence, Tuple

import multiaddr

from libp2p import new_node
from libp2p.host.basic_host import BasicHost
from libp2p.host.host_interface import IHost
from libp2p.kademlia.network import KademliaServer
from libp2p.network.stream.net_stream_interface import INetStream
from libp2p.network.swarm import Swarm
from libp2p.peer.peerinfo import info_from_p2p_addr
from libp2p.routing.interfaces import IPeerRouting
from libp2p.routing.kademlia.kademlia_peer_router import KadmeliaPeerRouter
from libp2p.typing import StreamHandlerFn, TProtocol

from .constants import MAX_READ_LEN

async def connect_swarm(swarm_0: Swarm, swarm_1: Swarm) -> None:
    peer_id = swarm_1.get_peer_id()
    addrs = tuple(
        addr
        for transport in swarm_1.listeners.values()
        for addr in transport.get_addrs()
    )
    swarm_0.peerstore.add_addrs(peer_id, addrs, 10000)
    await swarm_0.dial_peer(peer_id)
    assert swarm_0.get_peer_id() in swarm_1.connections
    assert swarm_1.get_peer_id() in swarm_0.connections

async def connect(node1: IHost, node2: IHost) -> None:
    """Connect node1 to node2."""
    addr = node2.get_addrs()[0]
    info = info_from_p2p_addr(addr)
    await node1.connect(info)

async def set_up_nodes_by_transport_opt(
    transport_opt_list: Sequence[Sequence[str]]
) -> Tuple[BasicHost, ...]:
    nodes_list = []
    for transport_opt in transport_opt_list:
        node = await new_node(transport_opt=transport_opt)
        await node.get_network().listen(multiaddr.Multiaddr(transport_opt[0]))
        nodes_list.append(node)
    return tuple(nodes_list)

async def set_up_nodes_by_transport_and_disc_opt(
    transport_disc_opt_list: Sequence[Tuple[Sequence[str], IPeerRouting]]
) -> Tuple[BasicHost, ...]:
    nodes_list = []
    for transport_opt, disc_opt in transport_disc_opt_list:
        node = await new_node(transport_opt=transport_opt, disc_opt=disc_opt)
        await node.get_network().listen(multiaddr.Multiaddr(transport_opt[0]))
        nodes_list.append(node)
    return tuple(nodes_list)

async def set_up_routers(
    router_confs: Tuple[int, int] = (0, 0)
) -> List[KadmeliaPeerRouter]:
    """The default ``router_confs`` selects two free ports local to this
    machine."""
    bootstrap_node = KademliaServer()  # type: ignore
    await bootstrap_node.listen(router_confs[0])

    routers = [KadmeliaPeerRouter(bootstrap_node)]
    for port in router_confs[1:]:
        node = KademliaServer()  # type: ignore
        await node.listen(port)

        await node.bootstrap_node(bootstrap_node.address)
        routers.append(KadmeliaPeerRouter(node))
    return routers

async def echo_stream_handler(stream: INetStream) -> None:
    while True:
        read_string = (await stream.read(MAX_READ_LEN)).decode()

        resp = "ack:" + read_string
        await stream.write(resp.encode())

async def perform_two_host_set_up(
    handler: StreamHandlerFn = echo_stream_handler
) -> Tuple[BasicHost, BasicHost]:
    transport_opt_list = [["/ip4/127.0.0.1/tcp/0"], ["/ip4/127.0.0.1/tcp/0"]]
    (node_a, node_b) = await set_up_nodes_by_transport_opt(transport_opt_list)

    node_b.set_stream_handler(TProtocol("/echo/1.0.0"), handler)

    # Associate the peer with local ip address (see default parameters of Libp2p())
    node_a.get_peerstore().add_addrs(node_b.get_id(), node_b.get_addrs(), 10)
    return node_a, node_b


def array_make_relative(array):
    returned_array = []
    for number in array:
        returned_array.append(round(number - array[0],5))
    return returned_array

def combinations_dict(array, pyaterka_length, desyatka_length, min_quantity = 3, probability_edge = 0.99, min_profit = 0, bad_combinations = 0, sectors = 0):
    combinations = {}
    if sectors == 1:
        min_quantity = 1
    comission = 0.0003

    from pickle import load, dump

    def nearest_key_in_dictionary(dictionary, number):
        min_difference = -100
        for key in dictionary:
            if min_difference != -100:
                if abs(key - number) < min_difference:
                    nearest_key = key
                    min_difference = abs(key - number)
            else:
                nearest_key = key
                min_difference = abs(key - number)
        return nearest_key

    for i in range(len(array) - pyaterka_length - desyatka_length + 1):
        if 0 not in array[i: i + 15]:
            name = str(array_make_relative(array[i:i + pyaterka_length]))
            if str(array_make_relative(array[i:i + pyaterka_length])) not in combinations:
                combinations[name] = {'self':array_make_relative(array[i:i + pyaterka_length]), 'after':array_make_relative(array[i + pyaterka_length:i + pyaterka_length + desyatka_length]), 'quantity' : 1}
            else:
                combinations[name]['quantity'] += 1
                combinations[name]['after'] = sum_array([combinations[name]['after'], array_make_relative(array[i + pyaterka_length:i + pyaterka_length + desyatka_length])])
    for element in combinations:
        for i in range(len(combinations[element]['after'])):
            combinations[element]['after'][i] = round(combinations[element]['after'][i] / combinations[element]['quantity'], 7)

    profit_probabilities = load(open('profit_probabilities', 'rb'))
    for key in combinations.copy():
        q = combinations[key]['quantity']
        if q > profit_probabilities['max_semerka_length']:
                q = profit_probabilities['max_semerka_length']
        combinations[key]['min_i'] = combinations[key]['after'].index(min(combinations[key]['after']))
        combinations[key]['max_i'] = combinations[key]['after'].index(max(combinations[key]['after']))
        min_i = combinations[key]['min_i']
        max_i = combinations[key]['max_i']
        diff = combinations[key]['after'][max_i] - combinations[key]['after'][min_i]
        profit = combinations[key]['after'][max_i] - combinations[key]['after'][min_i] - comission
        profit_acceptable = 0
        profit_probability_acceptable = 0
        if (bad_combinations and profit < min_profit) or ((not bad_combinations) and profit > min_profit):
            profit_acceptable = 1
        #print(profit_probabilities[q][nearest_key_in_dictionary(profit_probabilities[q], diff)], probability_edge)
        if sectors == 1 or profit_probabilities[q][nearest_key_in_dictionary(profit_probabilities[q], diff)] > probability_edge:
            profit_probability_acceptable = 1
        if combinations[key]['quantity'] < min_quantity or not profit_probability_acceptable or not profit_acceptable:
            del combinations[key]

    #dump(combinations, open('name1', 'wb'))
    return combinations

def sum_array(input_arrays):
    max_len = 0
    max_len_i = -1
    for i in range(len(input_arrays)):
        if len(input_arrays[i]) > max_len:
            max_len = len(input_arrays[i])
            max_len_i = i
    for i in range(len(input_arrays)):
        if i != max_len_i:
            for j in range(len(input_arrays[i])):
                input_arrays[max_len_i][j] += input_arrays[i][j]
    return input_arrays[max_len_i]

def real_like_random_array(length, leaps_array):
    from random import random
    real_like_random_array = []
    real_like_random_array.append(0)
    number_sum = leaps_array[0] - 0.00000001
    for i in range(1, length):
        number = random() * number_sum
        j = 1
        while not (leaps_array[j][0][0] <= number and number <= leaps_array[j][0][1]):
            j += 1
        real_like_random_array.append(round(real_like_random_array[i-1] + leaps_array[j][1], 4))
    return real_like_random_array

def paint_graph(array, y_zoom = 1, label_quantity = 4):
    import tkinter
    root = tkinter.Tk()
    width = len(array)
    height = max(abs(max(array)), abs(min(array))) * 2 + 20
    canv = tkinter.Canvas(root, width = width, height = height, bg = 'lightblue', cursor = "pencil")
    coords = []
    for i in range(len(array)):
        coords.append([i, -array[i] * y_zoom + height/2])
    canv.create_line(coords, smooth = 1, splinesteps = 0)
    canv.pack()
    root.mainloop()

def profit_probabilities(semerka_length_range, desyatka_length, repeat_q, length, leaps_array, profit_after_comma):
    def sub_random_function_data(random_function):
        '''Получаем средние значения десятки для k повторений десятки'''
        import math
        result = []
        for k in range(max(semerka_length_range)+1):
            result.append([])
        for k in semerka_length_range:
            for i in range(0, length - k * desyatka_length, k * desyatka_length): #Пробегаемся по всем семёркам
                buff_desyatka = []
                for j in range(desyatka_length):
                    buff_desyatka.append(0)
                for j in range(0, k*desyatka_length, desyatka_length): #Получаем среднее для десятки
                    for l in range(desyatka_length):
                        buff_desyatka[l] += random_function[i+j+l] - random_function[i+j]
                for j in range(desyatka_length):
                    buff_desyatka[j] /= k
                buff = round((max(buff_desyatka)-min(buff_desyatka)), profit_after_comma) #Получаем прибыль для усреднённой десяти и округляем её
                result[k].append(buff)
        return result
    result = []
    for i in range(max(semerka_length_range)+1):
        result.append([])
    for i in range(repeat_q):
        random_function = real_like_random_array(length, leaps_array)
        buff = sub_random_function_data(random_function)
        for j in range(len(buff)):
            result[j] += buff[j]
            print(result[j])
            #больший процент успешных комбинаций при не-отсеивании!!!!!!!!!!!!!!!!!!!!!!!
    result_dict = {}
    for i in range(min(semerka_length_range),len(result)):
        sub_dict = {}
        for j in range(len(result[i])):
            if result[i][j] not in sub_dict:
                sub_dict[result[i][j]] = 1
            else:
                sub_dict[result[i][j]] += 1
        key_list = []
        for key in sub_dict:
                key_list.append(key)
        key_list.sort()
        for k in range(1, len(key_list)):
            sub_dict[key_list[k]] += sub_dict[key_list[k-1]]
            sub_dict[key_list[k-1]] /= (length * repeat_q) / (desyatka_length * i) - 1
            sub_dict[key_list[k-1]] = round(sub_dict[key_list[k-1]], 2)
        sub_dict[key_list[len(key_list)-1]] /= (length  / (desyatka_length * i) - 1)* repeat_q
        sub_dict[key_list[len(key_list)-1]] = round(sub_dict[key_list[len(key_list)-1]], 3)
        result_dict[i] = sub_dict
    result_dict['max_semerka_length'] = max(semerka_length_range)
    return result_dict

def leaps_array(function):
    relative_function = []
    for i in range(1, len(function)):
        if function[i-1] != 0 and function[i] != 0:
            relative_function.append(function[i]-function[i-1])
    leap_dict = {}
    for leap in relative_function:
        if round(leap,4) not in leap_dict:
            leap_dict[round(leap,4)] = 1
        else:
            leap_dict[round(leap,4)] +=1
    for key in leap_dict:
        leap_dict[key] /= len(function) - 1
    prev_number = 0
    sub_array = []
    for key in leap_dict:
        sub_array.append([leap_dict[key], key])
    sub_array.sort(key = lambda array: array[0])
    sub_array.reverse()
    leap_array = []
    leap_sum = 0
    for array in sub_array:
        leap_sum += array[0]
    leap_array.append(leap_sum)
    leap_array.append([[0, sub_array[0][0]], sub_array[0][1]])
    for i in range(1, len(sub_array)):
        prev_number = leap_array[i][0][1]
        leap_array.append([[prev_number, prev_number + sub_array[i][0]], sub_array[i][1]])
    return leap_array

def play(function, combinations, pyaterka_length = 5, desyatka_length = 10, comission = 0.0003):
    current_money = 1
    for i in range(len(function) - pyaterka_length - desyatka_length):
        if str(array_make_relative(function[i: i + pyaterka_length])) in combinations and 0 not in function[i + pyaterka_length: i + pyaterka_length + desyatka_length]:
            desyatka = combinations[str(array_make_relative(function[i: i + pyaterka_length]))]['after']
            max_digit = desyatka[0]
            max_digit_j = 0
            for j in range(1,len(desyatka)):
                if desyatka[j] > max_digit:
                    max_digit = desyatka[j]
                    max_digit_j = j
            min_digit = desyatka[0]
            min_digit_j = 0
            for j in range(1,max_digit_j):
                if desyatka[j] < min_digit:
                    min_digit = desyatka[j]
                    min_digit_j = j
            current_money += (function[i+ pyaterka_length+max_digit_j] - function[i+ pyaterka_length+min_digit_j] - comission) * 100 / function[i+ pyaterka_length+min_digit_j]
    return current_money

def sort_combinations(all_combinations, self_length, after_length):
    from pickle import load, dump
    from copy import copy

    sectors = load(open('sectors', 'rb'))

    good_combinations = {}
    bad_combinations = {}

    for sector in sectors:
        bad_combinations.update(combinations_dict(sector, self_length, after_length, bad_combinations = 1, sectors = 1))
        good_combinations.update(combinations_dict(sector, self_length, after_length, sectors = 1))

    for combination in copy(all_combinations):
        if combination in bad_combinations or combination not in good_combinations:
            del all_combinations[combination]

    return all_combinations

def trade(combinations, self_length, after_length):
    import pickle
    function = pickle.load(open('function_2', 'rb'))
    comission = 0.0003

    def array_make_relative(array):
        returned_array = []
        for number in array:
            returned_array.append(round(number - array[0],4))
        return returned_array

    def do_transactions(coord, money, combination):
        if 0 not in function[coord: coord + after_length]:
            multiplier = ((function[coord + combination['max_i']] - comission) / function[coord + combination['min_i']] - 1) * 100 + 1
            money *= multiplier
            #print(money)
        return money

    money = 1
    operation_count = 0
    sucsessful_count = 0
    for key in combinations:
        combinations[key]['multiplier'] = 1

    for i in range(len(function) - self_length - after_length):
        name = str(array_make_relative(function[i : i+self_length]))
        if name in combinations:
            prev_money = money
            money = do_transactions(i+self_length, money, combinations[name])
            combinations[name]['multiplier'] *= money / prev_money
            operation_count += 1
            if money < 10e-323:
                break

    for key in combinations:
        if combinations[key]['multiplier'] > 1:
            sucsessful_count += 1

    if len(combinations):
        sucsessful_percentage = round(sucsessful_count / len(combinations) * 100, 3)
    else:
        sucsessful_percentage = 'no combinations'

    return money, operation_count, sucsessful_percentage
    #open('log.txt', 'a').write('operation count: ' + str(operation_count) + ', sucsessful percent: ' + str(round(sucsessful_count / len(combinations) * 100, 3)) + '\n')
    #print('Operation count: ' + str(operation_count) + '.')
    #print('Sucsessful combinations percentage: ' + str(round(sucsessful_count / len(combinations) * 100, 3)) + '.')

биржевая программа/Комбинации/lib.py


# -*- coding: utf-8 -*-
"""
Created on Mon Oct 14 14:41:53 2019

@author: amanoels
"""

from __future__ import absolute_import, division, print_function, unicode_literals
# TensorFlow e tf.keras
import tensorflow as tf
from tensorflow import keras

# Librarie sauxiliares
import numpy as np
import matplotlib.pyplot as plt

print(tf.__version__)

# Baixando pacote fashion_mnist
fashion_mnist = keras.datasets.fashion_mnist

# Carregando dados para treinamento e testes
(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()

"""
Label 	Classe
0 	    Camisetas/Top (T-shirt/top)
1 	    Calça (Trouser)
2 	    Suéter (Pullover)
3 	    Vestidos (Dress)
4 	    Casaco (Coat)
5 	    Sandálias (Sandal)
6 	    Camisas (Shirt)
7 	    Tênis (Sneaker)
8 	    Bolsa (Bag)
9 	    Botas (Ankle boot) """

# A base de dados não traz os labels, sendo assim estão sendo armazenados aqui
class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',
               'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']

# Shape para determinar o dimensionamento
train_images.shape

# Labels com um conjunto de treinamento
len(train_labels)

# Cada label é um inteiro entre 0 e 9
train_labels

# Existem 10000 imagens no cojunto de teste
test_images.shape

# labels no conjunto de teste
len(test_labels)

# Os dados precisam ser pré-processados antes de treinar a rede. Verificando a primeira
# imagem percebemos que os pixels estão entre 0 e 255.
plt.figure()
plt.imshow(train_images[0])
plt.colorbar()
plt.grid(False)
plt.show()

# Estes valores precisam ser escalados no intervalor entre 0 e 1 antes de alimentar
# o modelo da rede neural para tal os valores devem ser divididos por 255.
train_images = train_images / 255.0
test_images = test_images / 255.0

# Para garantir que os dados estão no formato correto e prontos para construir e treinar
# a rede, apresento as 25 primeiras imagens.
plt.figure(figsize=(10,10))
for i in range(25):
    plt.subplot(5,5,i+1)
    plt.xticks([])
    plt.yticks([])
    plt.grid(False)
    plt.imshow(train_images[i], cmap=plt.cm.binary)
    plt.xlabel(class_names[train_labels[i]])
plt.show()

# As figuras estão sendo apresentadas e montadas corretamente, desta forma começo a construção
# do modelo

# Duas etapas principas
# 1 Montar as camadas
# 2 Compilar as camadas

# Modelagem
model = keras.Sequential([
        keras.layers.Flatten(input_shape=(28,28)),
        keras.layers.Dense(128, activation='relu'),
        keras.layers.Dense(10, activation='softmax')
])

# Compilação ( Função loss mede o erro, o otimizador aponta para o caminho correto,
# baseado nos resultados de loss e a métrica de medição é a acuracia.
model.compile(optimizer='adam',
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])

# Treinando o modelo
model.fit(train_images, train_labels, epochs=50)

# Avaliação do modelo e sua acuracia.
test_loss, test_acc = model.evaluate(test_images,  test_labels, verbose=2)

print('\nTest accuracy:', test_acc)
# A acuracio não esta boa ainda, precisa ser ajustado, pois o modelo esta com overfiting

# Prevendo classificação
predictions = model.predict(test_images)

# Primeira posiçao da previsão, é um array de 10 posições
predictions[0]

# Buscando a maior confiana exitente neste array
np.argmax(predictions[0])
# Desta forma o modelo esta confiante de que esta imagem é uma "Ankle boot".

# Para validar que a classe é mesmo esta, vou checar a massa de testes
train_labels[0]

# Função de preparação das imagens
def plot_image(i, predictions_array, true_label, img):
  predictions_array, true_label, img = predictions_array[i], true_label[i], img[i]
  plt.grid(False)
  plt.xticks([])
  plt.yticks([])

  plt.imshow(img, cmap=plt.cm.binary)

  predicted_label = np.argmax(predictions_array)
  if predicted_label == true_label:
    color = 'blue'
  else:
    color = 'red'

  plt.xlabel("{} {:2.0f}% ({})".format(class_names[predicted_label],
                                100*np.max(predictions_array),
                                class_names[true_label]),
                                color=color)

def plot_value_array(i, predictions_array, true_label):
  predictions_array, true_label = predictions_array[i], true_label[i]
  plt.grid(False)
  plt.xticks([])
  plt.yticks([])
  thisplot = plt.bar(range(10), predictions_array, color="#777777")
  plt.ylim([0, 1])
  predicted_label = np.argmax(predictions_array)

  thisplot[predicted_label].set_color('red')
  thisplot[true_label].set_color('blue')

# Apresentação de informação
i = 0
plt.figure(figsize=(6,3))
plt.subplot(1,2,1)
plot_image(i, predictions, test_labels, test_images)
plt.subplot(1,2,2)
plot_value_array(i, predictions,  test_labels)
plt.show()

# Plotando a primeira previsão
num_rows = 5
num_cols = 3
num_images = num_rows*num_cols
plt.figure(figsize=(2*2*num_cols, 2*num_rows))
for i in range(num_images):
  plt.subplot(num_rows, 2*num_cols, 2*i+1)
  plot_image(i, predictions, test_labels, test_images)
  plt.subplot(num_rows, 2*num_cols, 2*i+2)
  plot_value_array(i, predictions, test_labels)
plt.show()

# Previsão para uma unica imagem
img = test_images[0]
print(img.shape)


# Copyright (C) 2010-2011 Richard Lincoln
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.

from CIM16.IEC61970.Core.PowerSystemResource import PowerSystemResource

class CAESPlant(PowerSystemResource):
    """Compressed air energy storage plantCompressed air energy storage plant
    """

    def __init__(self, ratedCapacityP=0.0, energyStorageCapacity=0.0, AirCompressor=None, ThermalGeneratingUnit=None, *args, **kw_args):
        """Initialises a new 'CAESPlant' instance.

        @param ratedCapacityP: The CAES plant's gross rated generating capacity
        @param energyStorageCapacity: The rated energy storage capacity.
        @param AirCompressor: An air compressor may be a member of a compressed air energy storage plant
        @param ThermalGeneratingUnit: A thermal generating unit may be a member of a compressed air energy storage plant
        """
        #: The CAES plant's gross rated generating capacity
        self.ratedCapacityP = ratedCapacityP

        #: The rated energy storage capacity.
        self.energyStorageCapacity = energyStorageCapacity

        self._AirCompressor = None
        self.AirCompressor = AirCompressor

        self._ThermalGeneratingUnit = None
        self.ThermalGeneratingUnit = ThermalGeneratingUnit

        super(CAESPlant, self).__init__(*args, **kw_args)

    _attrs = ["ratedCapacityP", "energyStorageCapacity"]
    _attr_types = {"ratedCapacityP": float, "energyStorageCapacity": float}
    _defaults = {"ratedCapacityP": 0.0, "energyStorageCapacity": 0.0}
    _enums = {}
    _refs = ["AirCompressor", "ThermalGeneratingUnit"]
    _many_refs = []

    def getAirCompressor(self):
        """An air compressor may be a member of a compressed air energy storage plant
        """
        return self._AirCompressor

    def setAirCompressor(self, value):
        if self._AirCompressor is not None:
            self._AirCompressor._CAESPlant = None

        self._AirCompressor = value
        if self._AirCompressor is not None:
            self._AirCompressor.CAESPlant = None
            self._AirCompressor._CAESPlant = self

    AirCompressor = property(getAirCompressor, setAirCompressor)

    def getThermalGeneratingUnit(self):
        """A thermal generating unit may be a member of a compressed air energy storage plant
        """
        return self._ThermalGeneratingUnit

    def setThermalGeneratingUnit(self, value):
        if self._ThermalGeneratingUnit is not None:
            self._ThermalGeneratingUnit._CAESPlant = None

        self._ThermalGeneratingUnit = value
        if self._ThermalGeneratingUnit is not None:
            self._ThermalGeneratingUnit.CAESPlant = None
            self._ThermalGeneratingUnit._CAESPlant = self

    ThermalGeneratingUnit = property(getThermalGeneratingUnit, setThermalGeneratingUnit)


import os
import sys
from pprint import pprint
from collections import defaultdict
from array import array
import argparse
import ROOT as r

'''
This script calculates the BX scale factors to be applied
on train MC for data comparison. They are binned by BX and E-rec regions.
There are different histograms for different eta regions:
1 = all endcap
2 = rings 18-21
3 = rings 22-25
4 = rings 26-28
'''

parser = argparse.ArgumentParser()
parser.add_argument("-i", "--inputfile", type=str, help="Input file", required=True)
parser.add_argument("-o", "--outputfile", type=str, help="Outputfile", required=True)
parser.add_argument("-n", "--nbx", type=int, help="Number of BX", required=True)
args = parser.parse_args()

regions_names = {
    "1": "18_28",
    "2": "18_21",
    "3": "22_25",
    "4": "26_28"
}

with open(args.inputfile) as fi:
    lines = fi.readlines()

regions = defaultdict(dict)

for li in lines:
    region_et, nevents_bx = li.split(" ")
    nevents_bx = int(nevents_bx)
    region , et = region_et.split("_")
    et = int(et)

    if et not in regions[region]:
        regions[region][et] = []

    regions[region][et].append(nevents_bx)

# Calculate Scale factors
sf_regions = {}
for region, etregs in regions.items():
    new_etrs = {}
    for k, etr in etregs.items():
        s = sum(etr)
        bxsf = list(map(lambda k: k/s, etr))
        new_etrs[k] = bxsf
    sf_regions[region] = new_etrs

# Create one 2D histogram for each region
output = r.TFile(args.outputfile, "recreate")

for region, data in sf_regions.items():
    sfhisto = r.TH2F("bxsf_"+ regions_names[region], "", args.nbx, 0, args.nbx-1, len(data), 1, len(data))

    for et, sfs in data.items():
        for bx,sf in enumerate(sfs):
            bn = sfhisto.GetBin(bx+1, et)
            sfhisto.SetBinContent(bn, sf)
            print(et,bx,sf)
    sfhisto.Write()

output.Close()


# summarize multiple confidence intervals on an ARIMA forecast
from pandas import read_csv
from statsmodels.tsa.arima.model import ARIMA
# load data
series = read_csv('daily-total-female-births.csv', header=0, index_col=0, parse_dates=True, squeeze=True)
# split data into train and test setes
X = series.values
X = X.astype('float32')
size = len(X) - 1
train, test = X[0:size], X[size:]
# fit an ARIMA model
model = ARIMA(train, order=(5,1,1))
model_fit = model.fit()
result = model_fit.get_forecast()
forecast = result.predicted_mean
# summarize confidence intervals
intervals = [0.2, 0.1, 0.05, 0.01]
for a in intervals:
	ci = result.conf_int(alpha=a)
	print('%.1f%% Confidence Interval: %.3f between %.3f and %.3f' % ((1-a)*100, forecast, ci[0,0], ci[0,1]))


#Import libraries
from PIL import Image
from matplotlib import pyplot as plt
import numpy as np
import face_recognition
import keras
from keras.models import load_model
import cv2

cap = cv2.VideoCapture(0)

img_counter = 1

while(True):
    # Capture frame-by-frame
    ret, frame = cap.read()

    # Display the resulting frame
    cv2.imshow('frame',frame)

    #If frame is read correctly, ret is True
    if not ret:
        break
    k = cv2.waitKey(1)

    if k%256 ==27:
        #Esc key pressed
        print("Escaped")
        break
    elif k%256 ==32:
        #Space key pressed
        #Format the image name
        img_name = "image_{}.png".format(img_counter)
        #Writing the file name and taking a snapshot of the current frame
        cv2.imwrite(img_name, frame)
        print("{} written".format(img_name))
        img_counter += 1

        #Set emotion dictionary
        emotion_dict= {'Angry': 0, 'Sad': 5, 'Neutral': 4, 'Disgust': 1, 'Surprise': 6, 'Fear': 2, 'Happy': 3}

        face_image = cv2.imread(r"/Users/poomkijoran/Downloads/Internship/face_and_emotion_detection-master/"+img_name)

        #resizing the image
        face_image = cv2.resize(face_image, (48,48))
        face_image = cv2.cvtColor(face_image, cv2.COLOR_BGR2GRAY)
        face_image = np.reshape(face_image, [1, face_image.shape[0], face_image.shape[1], 1])

        #Loading the model that has already been trained
        model = load_model(r"/Users/poomkijoran/Downloads/Internship/face_and_emotion_detection-master/Testing/model_v6_23.hdf5")

        #Predictions
        predicted_class = np.argmax(model.predict(face_image))
        label_map = dict((v,k) for k,v in emotion_dict.items())
        predicted_label = label_map[predicted_class]
        print(predicted_label)

    #Press q to stop the task
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

# When everything done, release the capture
cap.release()
cv2.destroyAllWindows()


import os
from django.template import Library, Node, TemplateSyntaxError
from google.appengine.ext import webapp
import logging

register = webapp.template.create_template_register()

def PageTree(pages, currentPage):
    return  { 'pageTree' : pages, 'currentPage': currentPage }

path = os.path.join(os.path.dirname(__file__), '../templates/edit/pageTree.html')
register.inclusion_tag(path)(PageTree)

def SitePageTree(pages, currentPage):
    return  { 'pageTree' : pages, 'currentPage': currentPage }

path = os.path.join(os.path.dirname(__file__), '../templates/pages/modules/sitePageTree.html')
register.inclusion_tag(path)(SitePageTree)

def Module(module, language):
    data = ''
    if module['data'].has_key(language):
        data = module['data'][language]
    return  { 'module' : module, 'data' : data, 'language' : language }

path = os.path.join(os.path.dirname(__file__), '../templates/edit/modules/module.html')
register.inclusion_tag(path)(Module)

def ifIn(value, list):
    if value in list:
        return True
    else:
        return False

register.filter('ifIn', ifIn)