#
# This file is part of pySMT.
#
#   Copyright 2014 Andrea Micheli and Marco Gario
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
#
from nose.plugins.attrib import attr
from pysmt.test import TestCase, skipIfSolverNotAvailable, main
from pysmt.test.examples import get_example_formulae
from pysmt.environment import get_env

class TestSimplify(TestCase):

    @attr("slow")
    @skipIfSolverNotAvailable("z3")
    def test_simplify_qf(self):
        simp = get_env().simplifier
        Iff = get_env().formula_manager.Iff
        for (f, _, _, logic) in get_example_formulae():
            if logic.is_quantified(): continue
            simp.validate_simplifications = True
            sf = f.simplify()
            simp.validate_simplifications = False
            self.assertValid(Iff(f, sf), solver_name='z3',
                             msg="Simplification did not provide equivalent "+
                                "result:\n f= %s\n sf = %s" % (f, sf))

    @attr("slow")
    @skipIfSolverNotAvailable("z3")
    def test_simplify_q(self):
        simp = get_env().simplifier
        Iff = get_env().formula_manager.Iff
        for (f, _, _, logic) in get_example_formulae():
            if logic.quantifier_free: continue
            simp.validate_simplifications = True
            sf = f.simplify()
            simp.validate_simplifications = False
            self.assertValid(Iff(f, sf), solver_name='z3',
                             msg="Simplification did not provide equivalent "+
                            "result:\n f= %s\n sf = %s" % (f, sf))

if __name__ == '__main__':
    main()


class Product (models.Model):
    id = models.PrimaryKey(
        unique = True,
        autoincrement = True
    )
    category_id = models.ForeignKey(
        null = False
    )
    brand = models.ForeignKey(
        max_length = 30
    )
    name = models.CharField(
        max_length = 30
    )
    model = models.CharField(
        max_length = 30
    )
    stack = models.IntegerField(

    )
    availability_from = models.DateField(
        default = ''
    )
    picture = ImageField(
        max_size = 1024
    )
    short_description = models.TextField(
        max_length = 350
    )
    description = models.TextField(
        max_length = 5000
    )

class Category (models.Model):
    id = models.PrimaryKey(
        unique = True
    )
    brand = models.ForeignKey(
         max_length = 30
    )
    image = models.ImageField (
        max_size = 1024
    )
    link = models.URLField (

    )


import matplotlib.pyplot as plt
from SolarSystem import solarSystem
from Particle import Particle
import numpy as np

#Data from 2016-11-30
sun = Particle(np.array([5.184e8,5.567e8,-2.360e7]),np.array([-4.092,1.166e1,8.348e-2]),'Sun',1.989e30)
earth = Particle(np.array([5.584e10,1.373e11,-2.945e7]),np.array([-2.812e4,1.107e4,-8.203e-2]),'Earth',5.972e24)
mars = Particle(np.array([2.077e11,-1.358e10,-5.404e9]),np.array([2.573e3,2.626e4,4.868e2]),'Mars',6.4171e23)
moon = Particle(np.array([5.572e10,1.369e11,5.603e6]),np.array([-2.718e4,1.081e4,-9.643]),'Moon',7.349e22)
mercury = Particle(np.array([4.215e10,-4.614e10,-7.659e9]),np.array([2.668e4,3.476e4,3.917e2]),'Mercury',3.302e23)
uranus = Particle(np.array([2.751e12,1.155e12,-3.136e10]),np.array([-2.686e3,5.962e3,5.709e1]),'Uranus',86.813e24)
pluto = Particle(np.array([1.431e12,-4.758e12,9.525e10]),np.array([5.319e3,4.254e2,-1.606e3]),'Pluto',1.307e22)
saturn = Particle(np.array([-3.038e11,-1.471e12,3.766e10]),np.array([8.929e3,-1.985e3,-3.206e2]),'Saturn',5.6834e26)
venus = Particle(np.array([1.090e11,-2.099e9,-6.320e9]),np.array([7.017e2,3.486e4,4.372e2]),'Venus',48.685e23)
jupiter = Particle(np.array([-8.068e11,-1.170e11,1.853e10]),np.array([1.723e3,-1.231e4,1.257e1]),'Jupiter',1898.13e24)
neptune = Particle(np.array([4.235e12,-1.464e12,-6.745e10]),np.array([1.739e3,5.169e3,-1.460e2]),'Neptune',102.413e24)

b = [sun, earth, mars, moon, mercury, saturn, venus, jupiter, pluto, uranus, neptune]    #Add all Particle class objects to a list
plt.xlabel('x')
plt.ylabel('y')

xDict = {}  #Defines 2 empty dictionaries
yDict = {}

timeStep = 180     #Time step of simulation in seconds
earthSun = solarSystem(b)

for planet in earthSun.planets:    #Add all particle class objects to a list
        xDict[planet.Name] = []     #Each planet has an empty list indexed to its name for both x and y coordinates
        yDict[planet.Name] = []

earthSun.livePlot(180,5000000,100,0.0005)

#for planet in earthSun.planets:
#    plt.plot(xDict[planet.Name], yDict[planet.Name], 'r-', label = planet.Name)     #Add functionality to give unique colour to each planet

"""
for n in range(6000):
    accelerationList = earthSun.acceleration()      #Calculates the ordered list of acceleration arrays
    for i, planet in enumerate(earthSun.planets):
        if n % 100 == 0:
            xDict[planet.Name].append(planet.coordinate(0))     #Adds current x coordinate to the end of the planet's xDict list
            yDict[planet.Name].append(planet.coordinate(1))     #Adds current y coordinate to the end of the planet's yDict list
        planet.update(accelerationList[i], timeStep)        #Updates planet's position and velocity arrays

for planet in earthSun.planets:
    plt.plot(xDict[planet.Name], yDict[planet.Name], 'r-', label = planet.Name)     #Add functionality to give unique colour to each planet

plt.legend()
plt.show()
"""


import numpy as np
from random import shuffle, sample
from astropy.io import fits
from numpy import vstack
from sklearn import preprocessing
from sklearn.datasets import make_moons, make_circles
from sklearn.preprocessing import StandardScaler
from matplotlib.colors import ListedColormap
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import VotingClassifier
from sklearn.model_selection import cross_val_score
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import pandas
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn.tree import DecisionTreeClassifier
from sklearn.naive_bayes import GaussianNB
from matplotlib import pyplot
from imblearn.over_sampling import RandomOverSampler
import imblearn as imbl
from collections import Counter
from sklearn.datasets import make_classification
from imblearn.over_sampling import SMOTE
from numpy import where

pyplot.rcParams['xtick.labelsize'] = 16
pyplot.rcParams['axes.labelsize'] = 16
pyplot.rcParams['axes.titlesize'] = 25
pyplot.rcParams['font.size'] = 15
pyplot.rcParams['ytick.labelsize'] = 16

se=0

rf=[]
bdt=[]
nn=[]
lr=[]

#dataframe = pandas.read_csv("./files/4fgldr2_all_newfeats.csv", header=None)
dataframe = pandas.read_csv("./files/3fgl_all_newfeats.csv", header=None)
dataset1 = dataframe.values
lenth=12

names_assoc=[dataset1[i,0] for i in range(len(dataset1)) if dataset1[i,lenth]=='AGN' or dataset1[i,lenth]=='PSR'or dataset1[i,lenth]=='OTHER']
unassocnames=[dataset1[i,0] for i in range(len(dataset1)) if dataset1[i,lenth]=='UNAS']

print(names_assoc)
###################
sourcename_ass=[]
probs_ass_bdt=[]
probs_un_bdt=[]
probs_ass_nn=[]
probs_un_nn=[]
probs_ass_rf=[]
probs_un_rf=[]
probs_ass_lr=[]
probs_un_lr=[]
#####################
times=1000

while se<times:
    #data:
    np.random.seed(se)
    #dataframe3 = pandas.read_csv("./files/4fgldr2_all_newfeats.csv", header=None)
    dataframe3 = pandas.read_csv("./files/3fgl_all_newfeats.csv", header=None)

    dataset3 = dataframe3.values
    k=dataset3[1:,1].astype(float)
    dataset3[1:,1]=np.cos(k)
    X2 = [dataset3[i,1:lenth].astype(float) for i in range(len(dataset3)) if dataset3[i,lenth]=='UNAS' ]
    np.random.shuffle(dataset3[1:])

    ##############################
    X = [dataset3[i,1:lenth].astype(float) for i in range(len(dataset3)) if dataset3[i,lenth]=='AGN' or dataset3[i,lenth]=='PSR'or dataset3[i,lenth]=='OTHER']
    Y = [dataset3[i,lenth] for i in range(len(dataset3)) if dataset3[i,lenth]=='AGN' or dataset3[i,lenth]=='PSR'or dataset3[i,lenth]=='OTHER']
    val_source1=[dataset3[i,0] for i in range(len(dataset3)) if dataset3[i,lenth]=='AGN' or dataset3[i,lenth]=='PSR'or dataset3[i,lenth]=='OTHER']

    encoder = preprocessing.LabelEncoder()
    encoder.fit(Y)
    Y = encoder.transform(Y)
    counter=Counter(Y)
    print(counter)
    '''
    for label, _ in counter.items():
        row_ix = where(Y == label)[0]
        pyplot.scatter(X1[row_ix,0], X1[row_ix,1], label=str(label))
    pyplot.legend()
    pyplot.show()
    '''

    ################################
    lenth2=len(Y)
    lenth2=int(0.7*lenth2)
    train1=X[0:lenth2]
    train_truth1=Y[0:lenth2]
    val_inp1=X[lenth2:]
    val_source=val_source1[lenth2:]
    val_out1=Y[lenth2:]

    val_out1=np.ravel(val_out1)                     #ravel is used since flattened label array required
    train_truth1=np.ravel(train_truth1)

    ################
    sourcename_ass.append(val_source)
    Y0=[i for i in range(len(train_truth1)) if train_truth1[i]==0]
    Y1=[i for i in range(len(train_truth1)) if train_truth1[i]==1]
    Y2=[i for i in range(len(train_truth1)) if train_truth1[i]==2]
    print('AGNS:',len(Y0))
    w1=int(np.sqrt(len(Y0)*len(Y1)))
    w2=int(np.sqrt(len(Y0)*len(Y2)))
    weight={0:len(Y0),1:w1,2:w2}
    ##################

    #X_over=train1
    #y_over=train_truth1
    oversample = SMOTE(sampling_strategy=weight)
    X_over, y_over = oversample.fit_resample(train1, train_truth1)
    counter=Counter(y_over)
    print(counter)

    #oversample = RandomOverSampler(sampling_strategy=0.5)
    clf= GradientBoostingClassifier(n_estimators=100, learning_rate=0.3,max_depth=2).fit(X_over, y_over)
    clf2= MLPClassifier(max_iter=600,hidden_layer_sizes=(16,), activation='tanh', solver='lbfgs').fit(X_over, y_over)
    clf3= LogisticRegression(max_iter=500, C=1,solver='lbfgs').fit(X_over, y_over)
    clf4 = RandomForestClassifier(n_estimators=50,max_depth=6,oob_score=True)
    clf4.fit(X_over, y_over)

    ##########################################
    fit1=clf.predict_proba(val_inp1)
    fit2=clf2.predict_proba(val_inp1)
    fit3=clf3.predict_proba(val_inp1)
    fit4=clf4.predict_proba(val_inp1)
    probs_ass_bdt.append(fit1)
    probs_un_bdt.append(clf.predict_proba(X2))
    probs_ass_nn.append(fit2)
    probs_un_nn.append(clf2.predict_proba(X2))
    probs_ass_rf.append(fit4)
    probs_un_rf.append(clf4.predict_proba(X2))
    probs_ass_lr.append(fit3)
    probs_un_lr.append(clf3.predict_proba(X2))
    ################################################
    fit5=clf.score(val_inp1,val_out1)
    fit6=clf2.score(val_inp1,val_out1)
    fit7=clf3.score(val_inp1,val_out1)
    fit8=clf4.score(val_inp1,val_out1)
    rf.append(fit8)
    lr.append(fit7)
    nn.append(fit6)
    bdt.append(fit5)
    print('Scores:',fit5,fit6,fit7,fit8)
    ##########################################
    other_lr=clf3.predict_proba(X2)
    number=[i for i in range(len(other_lr)) if (other_lr[i,1]>other_lr[i,0] and other_lr[i,1]>other_lr[i,2])]
    print('number of others in unassoc sources:',len(number))
    se=se+1

########################

print('meanstesting(rf,lr,bdt,nn):',np.mean(rf),np.mean(lr),np.mean(bdt),np.mean(nn))
print('stdo:',np.std(rf),np.std(lr),np.std(bdt),np.std(nn))
####################
unassoc_names=unassocnames
AGN_BDT_mean=np.zeros(len(unassoc_names))
AGN_BDT_std=np.zeros(len(unassoc_names))
PSR_BDT_mean=np.zeros(len(unassoc_names))
PSR_BDT_std=np.zeros(len(unassoc_names))
OTHER_BDT_mean=np.zeros(len(unassoc_names))
OTHER_BDT_std=np.zeros(len(unassoc_names))
AGN_NN_mean=np.zeros(len(unassoc_names))
AGN_NN_std=np.zeros(len(unassoc_names))
PSR_NN_mean=np.zeros(len(unassoc_names))
PSR_NN_std=np.zeros(len(unassoc_names))
OTHER_NN_mean=np.zeros(len(unassoc_names))
OTHER_NN_std=np.zeros(len(unassoc_names))
AGN_RF_mean=np.zeros(len(unassoc_names))
AGN_RF_std=np.zeros(len(unassoc_names))
PSR_RF_mean=np.zeros(len(unassoc_names))
PSR_RF_std=np.zeros(len(unassoc_names))
OTHER_RF_mean=np.zeros(len(unassoc_names))
OTHER_RF_std=np.zeros(len(unassoc_names))
AGN_LR_mean=np.zeros(len(unassoc_names))
AGN_LR_std=np.zeros(len(unassoc_names))
PSR_LR_mean=np.zeros(len(unassoc_names))
PSR_LR_std=np.zeros(len(unassoc_names))
OTHER_LR_mean=np.zeros(len(unassoc_names))
OTHER_LR_std=np.zeros(len(unassoc_names))
#########################

AGN_BDT_mean_as=np.zeros(len(names_assoc))
AGN_BDT_std_as=np.zeros(len(names_assoc))
PSR_BDT_mean_as=np.zeros(len(names_assoc))
PSR_BDT_std_as=np.zeros(len(names_assoc))
OTHER_BDT_mean_as=np.zeros(len(names_assoc))
OTHER_BDT_std_as=np.zeros(len(names_assoc))
AGN_NN_mean_as=np.zeros(len(names_assoc))
AGN_NN_std_as=np.zeros(len(names_assoc))
PSR_NN_mean_as=np.zeros(len(names_assoc))
PSR_NN_std_as=np.zeros(len(names_assoc))
OTHER_NN_mean_as=np.zeros(len(names_assoc))
OTHER_NN_std_as=np.zeros(len(names_assoc))
AGN_RF_mean_as=np.zeros(len(names_assoc))
AGN_RF_std_as=np.zeros(len(names_assoc))
PSR_RF_mean_as=np.zeros(len(names_assoc))
PSR_RF_std_as=np.zeros(len(names_assoc))
OTHER_RF_mean_as=np.zeros(len(names_assoc))
OTHER_RF_std_as=np.zeros(len(names_assoc))
AGN_LR_mean_as=np.zeros(len(names_assoc))
AGN_LR_std_as=np.zeros(len(names_assoc))
PSR_LR_mean_as=np.zeros(len(names_assoc))
PSR_LR_std_as=np.zeros(len(names_assoc))
OTHER_LR_mean_as=np.zeros(len(names_assoc))
OTHER_LR_std_as=np.zeros(len(names_assoc))
#########################

for i in range(len(names_assoc)):

    name=names_assoc[i]
    AGN_BDT=[]
    PSR_BDT=[]
    OTHER_BDT=[]
    AGN_NN=[]
    PSR_NN=[]
    OTHER_NN=[]
    AGN_RF=[]
    PSR_RF=[]
    OTHER_RF=[]
    AGN_LR=[]
    PSR_LR=[]
    OTHER_LR=[]
    for j in range(times):
        sourcenames1=sourcename_ass[j]
        for k in range(len(sourcenames1)):
            if sourcenames1[k]==name:
                prob=probs_ass_bdt[j]
                prob2=prob[k]
                AGN_BDT.append(prob2[0])
                PSR_BDT.append(prob2[2])
                OTHER_BDT.append(prob2[1])
                prob=probs_ass_nn[j]
                prob2=prob[k]
                AGN_NN.append(prob2[0])
                PSR_NN.append(prob2[2])
                OTHER_NN.append(prob2[1])
                prob=probs_ass_rf[j]
                prob2=prob[k]
                AGN_RF.append(prob2[0])
                PSR_RF.append(prob2[2])
                OTHER_RF.append(prob2[1])
                prob=probs_ass_lr[j]
                prob2=prob[k]
                AGN_LR.append(prob2[0])
                PSR_LR.append(prob2[2])
                OTHER_LR.append(prob2[1])

    AGN_BDT_mean_as[i]=np.mean(AGN_BDT)
    AGN_BDT_std_as[i]=np.std(AGN_BDT)
    PSR_BDT_mean_as[i]=np.mean(PSR_BDT)
    PSR_BDT_std_as[i]=np.std(PSR_BDT)
    OTHER_BDT_mean_as[i]=np.mean(OTHER_BDT)
    OTHER_BDT_std_as[i]=np.std(OTHER_BDT)
    AGN_NN_mean_as[i]=np.mean(AGN_NN)
    AGN_NN_std_as[i]=np.std(AGN_NN)
    PSR_NN_mean_as[i]=np.mean(PSR_NN)
    PSR_NN_std_as[i]=np.std(PSR_NN)
    OTHER_NN_mean_as[i]=np.mean(OTHER_NN)
    OTHER_NN_std_as[i]=np.std(OTHER_NN)
    AGN_RF_mean_as[i]=np.mean(AGN_RF)
    AGN_RF_std_as[i]=np.std(AGN_RF)
    PSR_RF_mean_as[i]=np.mean(PSR_RF)
    PSR_RF_std_as[i]=np.std(PSR_RF)
    OTHER_RF_mean_as[i]=np.mean(OTHER_RF)
    OTHER_RF_std_as[i]=np.std(OTHER_RF)
    AGN_LR_mean_as[i]=np.mean(AGN_LR)
    AGN_LR_std_as[i]=np.std(AGN_LR)
    PSR_LR_mean_as[i]=np.mean(PSR_LR)
    PSR_LR_std_as[i]=np.std(PSR_LR)
    OTHER_LR_mean_as[i]=np.mean(OTHER_LR)
    OTHER_LR_std_as[i]=np.std(OTHER_LR)

result_as=np.column_stack((names_assoc,AGN_BDT_mean_as,AGN_BDT_std_as,PSR_BDT_mean_as,PSR_BDT_std_as,OTHER_BDT_mean_as,OTHER_BDT_std_as,AGN_NN_mean_as,AGN_NN_std_as,PSR_NN_mean_as,PSR_NN_std_as,OTHER_NN_mean_as,OTHER_NN_std_as,AGN_RF_mean_as,AGN_RF_std_as,PSR_RF_mean_as,PSR_RF_std_as,OTHER_RF_mean_as,OTHER_RF_std_as,AGN_LR_mean_as,AGN_LR_std_as,PSR_LR_mean_as,PSR_LR_std_as,OTHER_LR_mean_as,OTHER_LR_std_as))
pro2=["Source_Name","AGN_BDT","AGN_BDT_STD","PSR_BDT","PSR_BDT_STD","OTHER_BDT","OTHER_BDT_STD","AGN_NN","AGN_NN_STD","PSR_NN","PSR_NN_STD","OTHER_NN","OTHER_NN_STD","AGN_RF","AGN_RF_STD","PSR_RF","PSR_RF_STD","OTHER_RF","OTHER_RF_STD","AGN_LR","AGN_LR_STD","PSR_LR","PSR_LR_STD","OTHER_LR","OTHER_LR_STD"]
result_As=np.vstack((pro2,result_as))
result_As=pandas.DataFrame(result_As)

result_As.to_csv(path_or_buf="./catas/try_3FGL_smote_as_3C.csv",index=False)

########################################
for i in range(len(unassoc_names)):
    AGN_BDT=[]
    PSR_BDT=[]
    OTHER_BDT=[]
    AGN_NN=[]
    PSR_NN=[]
    OTHER_NN=[]
    AGN_RF=[]
    PSR_RF=[]
    OTHER_RF=[]
    AGN_LR=[]
    PSR_LR=[]
    OTHER_LR=[]
    for j in range(times):
        prob=probs_un_bdt[j]
        prob2=prob[i]
        AGN_BDT.append(prob2[0])
        PSR_BDT.append(prob2[2])
        OTHER_BDT.append(prob2[1])
        prob=probs_un_nn[j]
        prob2=prob[i]
        AGN_NN.append(prob2[0])
        PSR_NN.append(prob2[2])
        OTHER_NN.append(prob2[1])
        prob=probs_un_rf[j]
        prob2=prob[i]
        AGN_RF.append(prob2[0])
        PSR_RF.append(prob2[2])
        OTHER_RF.append(prob2[1])
        prob=probs_un_lr[j]
        prob2=prob[i]
        AGN_LR.append(prob2[0])
        PSR_LR.append(prob2[2])
        OTHER_LR.append(prob2[1])

    AGN_BDT_mean[i]=np.mean(AGN_BDT)
    AGN_BDT_std[i]=np.std(AGN_BDT)
    PSR_BDT_mean[i]=np.mean(PSR_BDT)
    PSR_BDT_std[i]=np.std(PSR_BDT)
    OTHER_BDT_mean[i]=np.mean(OTHER_BDT)
    OTHER_BDT_std[i]=np.std(OTHER_BDT)
    AGN_NN_mean[i]=np.mean(AGN_NN)
    AGN_NN_std[i]=np.std(AGN_NN)
    PSR_NN_mean[i]=np.mean(PSR_NN)
    PSR_NN_std[i]=np.std(PSR_NN)
    OTHER_NN_mean[i]=np.mean(OTHER_NN)
    OTHER_NN_std[i]=np.std(OTHER_NN)
    AGN_RF_mean[i]=np.mean(AGN_RF)
    AGN_RF_std[i]=np.std(AGN_RF)
    PSR_RF_mean[i]=np.mean(PSR_RF)
    PSR_RF_std[i]=np.std(PSR_RF)
    OTHER_RF_mean[i]=np.mean(OTHER_RF)
    OTHER_RF_std[i]=np.std(OTHER_RF)
    AGN_LR_mean[i]=np.mean(AGN_LR)
    AGN_LR_std[i]=np.std(AGN_LR)
    PSR_LR_mean[i]=np.mean(PSR_LR)
    PSR_LR_std[i]=np.std(PSR_LR)
    OTHER_LR_mean[i]=np.mean(OTHER_LR)
    OTHER_LR_std[i]=np.std(OTHER_LR)

result2=np.column_stack((unassoc_names,AGN_BDT_mean,AGN_BDT_std,PSR_BDT_mean,PSR_BDT_std,OTHER_BDT_mean,OTHER_BDT_std,AGN_NN_mean,AGN_NN_std,PSR_NN_mean,PSR_NN_std,OTHER_NN_mean,OTHER_NN_std,AGN_RF_mean,AGN_RF_std,PSR_RF_mean,PSR_RF_std,OTHER_RF_mean,OTHER_RF_std,AGN_LR_mean,AGN_LR_std,PSR_LR_mean,PSR_LR_std,OTHER_LR_mean,OTHER_LR_std))
pro2=["Source_Name","AGN_BDT","AGN_BDT_STD","PSR_BDT","PSR_BDT_STD","OTHER_BDT","OTHER_BDT_STD","AGN_NN","AGN_NN_STD","PSR_NN","PSR_NN_STD","OTHER_NN","OTHER_NN_STD","AGN_RF","AGN_RF_STD","PSR_RF","PSR_RF_STD","OTHER_RF","OTHER_RF_STD","AGN_LR","AGN_LR_STD","PSR_LR","PSR_LR_STD","OTHER_LR","OTHER_LR_STD"]
result=np.vstack((pro2,result2))
result=pandas.DataFrame(result)

result.to_csv(path_or_buf="./catas/try_3FGL_smote_unas_3C.csv",index=False)

print('AGN',AGN_BDT)

first=[]
agn=[]
psr=[]
for i in range(2):
    k=probs_un_bdt[i]
    l=k[0]
    first.append(k[0])
    agn.append(l[0])
    psr.append(l[1])

#pro3=np.hstack((probs_un_rf,probs_un_bdt))
#print('pro3:',np.shape(pro3))
print(sourcename_ass)
print(probs_ass_bdt)
print('un bdt1',probs_un_bdt)
print('un bdt',probs_un_bdt[0])
print(first)
print('AGN',agn)
print('PSR',psr)

'''
scorerf=scorerf/1000
scoregb=scoregb/1000
scorelr=scorelr/1000
scorenn=scorenn/1000
'''

#print(scorerf,scoregb,scorenn,scorelr)

for i in range(3747):
    for j in range(8):
        if pro1[i,9]!=1:
            pro1[i,j+1]=pro1[i,j+1]/(pro1[i,9]-1)
pro1[:,9]=pro1[:,9]-1
#print(np.mean(pro1[:,9]))

#prop1=prop1/1000
#print(pro1)

#dataframe = pandas.read_csv("./files/3fgl_associated_AGNandPSR.csv", header=None)
dataframe = pandas.read_csv("./files/4fgldr2_assoc.csv", header=None)
dataset3 = dataframe.values
pro2=["Source_Name","AGN_BDT","PSR_BDT","AGN_NN","PSR_NN","AGN_LR","PSR_LR","AGN_RF","PSR_RF","Times in Testing"]
pro3=np.vstack((pro2,pro1))

result=np.hstack((dataset3[0:],pro3))

#print(clf.feature_importances_)
#print(result)
#print(valscore3/1000)
#print(feat2/1000)
#result=pandas.DataFrame(result)
result=pandas.DataFrame(result)

#result.to_csv(path_or_buf="./catas/4fgldr2_assoc_catalog_o.csv",index=False)


import json
from Tkinter import *
from collections import OrderedDict

from pycparser import c_ast

from config import JsonConfig
from config.JsonConfig import NativeTypeToSimulatorType, NativeTypeToSize, DefaultEnumByteSize, TypeOverrides, \
    Infer2ndByteExceptions, Infer2ndByte, Infer2ndByteSimulatorTypeToNextSmallerType, NativeTypeToSimulatorBitfieldType, \
    InferUpTo4thByte, InferUpTo4thByteExceptions
from parsing.EnumsToJson import EnumsToJson
from parsing.ParseCTypes import CTypesParser, StructDeclVisitor, EnumDeclVisitor

Verbose = True

def printLeaf(fieldPath, fieldMame, fullyQualifiedName, type, bitSize, dimension):
    if bitSize is not None and bitSize is not "":
        bitSize = ":" + bitSize
    else:
        bitSize = ""

    if dimension is not None and dimension is not "":
        dimension = "[" + dimension + "]"
    else:
        dimension = ""

    if Verbose:
        print("%s%s <%s%s>" % (fullyQualifiedName, dimension, type, bitSize))

# def traverseStructMembersDF(structs, nativeTypesToSize, enumTypesToValue, rootStructName, prefix, callback=printLeaf):
#
#     for structSubDecl in structs[rootStructName]:
#         structSubDeclTypeName = StructDeclVisitor.getStructTypeName(structSubDecl)
#         structSubDeclName = structSubDecl.name
#
#         dimension = None
#         if hasattr(structSubDecl.type, "dim"):
#             dimension = int(structSubDecl.type.dim.value)
#
#         isPointerType = type(structSubDecl.type) == c_ast.PtrDecl
#         if structSubDeclTypeName in nativeTypesToSize.keys() or \
#                         structSubDeclTypeName in enumTypesToValue.keys() or isPointerType == True:  # on enum, native or pointer type
#             bitSize = None
#             if structSubDecl.bitsize != None:
#                 bitSize = structSubDecl.bitsize.value
#
#             callback(prefix, structSubDeclName, prefix + "." + structSubDeclName, structSubDeclTypeName, bitSize,
#                      dimension, isPointerType)
#         else:  # on struct type
#             for dim in range(dimension):
#                 traverseStructMembersDF(structs, nativeTypesToSize, enumTypesToValue,
#                                         rootStructName=structSubDeclTypeName, callback=callback,
#                                         prefix=prefix + "." + structSubDeclName + "[" + str(dim) + "]")
#             else:
#                 traverseStructMembersDF(structs, nativeTypesToSize, enumTypesToValue,
#                                         rootStructName=structSubDeclTypeName, callback=callback,
#                                         prefix=prefix + "." + structSubDeclName)

def traverseStructMembersDF(structs, nativeTypesToSize, enumTypesToValue, rootStructName, prefix, callback=printLeaf):
    for structSubDecl in structs[rootStructName]:
        structSubDeclTypeName = StructDeclVisitor.getStructTypeName(structSubDecl)
        structSubDeclName = structSubDecl.name

        dimension = None
        if hasattr(structSubDecl.type, "dim"):
            dimension = int(structSubDecl.type.dim.value)

        isPointerType = isPointerType = type(structSubDecl.type) == c_ast.PtrDecl
        if structSubDeclTypeName in nativeTypesToSize.keys() or \
                        structSubDeclTypeName in enumTypesToValue.keys() or (
            isPointerType == True):  # on enum/native type
            bitSize = None
            if isPointerType:
                bitSize = None
            elif structSubDecl.bitsize != None:
                bitSize = structSubDecl.bitsize.value

            callback(prefix, structSubDeclName, prefix + "." + structSubDeclName, structSubDeclTypeName, bitSize,
                     dimension, isPointerType)
        else:  # on struct type
            if dimension != None:
                for dim in range(dimension):
                    traverseStructMembersDF(structs, nativeTypesToSize, enumTypesToValue,
                                            rootStructName=structSubDeclTypeName, callback=callback,
                                            prefix=prefix + "." + structSubDeclName + "[" + str(dim) + "]")
            else:
                traverseStructMembersDF(structs, nativeTypesToSize, enumTypesToValue,
                                        rootStructName=structSubDeclTypeName, callback=callback,
                                        prefix=prefix + "." + structSubDeclName)

def appendEnumToSize(enumToSize, typeToSizeMapping):
    for enum in enumToSize.keys():
        typeToSizeMapping[enum] = 2

def appendEnumNames(enumDict, nativeTypeToSimulatorTypeMaping):
    for enum in enumDict.keys():
        nativeTypeToSimulatorTypeMaping[enum] = enum

class FieldDescription:
    def __init__(self, description, fieldPath, fieldName, typeName, arrayIdx=None, isBitField=False, startBit=None,
                 bitsLength=None, isPointerType=False):
        self.startBit = startBit
        self.length = bitsLength
        self.endBit = None
        self.fieldPath = fieldPath
        self.fieldName = fieldName
        self.typeName = typeName
        self.arrayIndex = arrayIdx
        self.isPointerTyp = isPointerType
        self.__isBitField = isBitField

        if self.startBit != None:
            self.startBit = int(self.startBit)

        if self.length != None:
            self.length = int(self.length)
        else:
            self.length = 0

        if self.startBit != None and self.length != None:
            self.endBit = self.startBit + self.length - 1

        self.description = description

    def getArrayIdx(self):
        return self.arrayIndex

    def isArrayField(self):
        return self.arrayIndex != None

    def isPointerType(self):
        return self.isPointerTyp

    def isBitField(self):
        return self.__isBitField

    def getFieldName(self):
        return self.fieldName

    def getFieldPath(self):
        return self.fieldPath

    def getTypeName(self):
        return self.typeName

    def getStartBit(self):
        return self.startBit

    def getEndBit(self):
        return self.endBit

    def getLength(self):
        return self.length

    def getDescription(self):
        return self.description

    def __str__(self):
        return "st[%s]\tlen=%s\tend[%s]\tdesc=%s p=%s f=%s t=<%s>" % (
            self.startBit, self.length, self.endBit, self.description, self.fieldPath, self.fieldName, self.typeName)

class RegisterEntry:
    def __init__(self, name, type, bitSize=None, arraySize=None):
        self.name = name
        self.type = type
        self.fieldDescriptions = {}

        if bitSize != None:
            self.isBitField = True
            self.bitSize = bitSize

        if arraySize != None:
            self.isArray = True
            self.arraySize = arraySize

    def addFieldDescription(self, fieldDescription):
        self.fieldDescriptions[fieldDescription.getStartBit()] = fieldDescription

class LinearStructComposer:
    def __init__(self, typeToSizeMapping):
        self.linearFields = []
        self.typeToSize = typeToSizeMapping
        self.aggregatedLinearFields = OrderedDict()

    def consumeStructField(self, fieldPath, fieldName, fullyQualifiedName, type, bitSize, dimension, isPointerType):

        # on pointer type decl
        if isPointerType == True:
            fieldDescription = FieldDescription("%s*" % fullyQualifiedName, fieldPath, fieldName, type,
                                                arrayIdx=0, isBitField=False, startBit=0, bitsLength=8,
                                                isPointerType=isPointerType)
            self.linearFields.append(fieldDescription)
            return

        isBitField = True
        if bitSize == None:
            isBitField = False
            bitSize = self.typeToSize[type] * 8

        # on normal decl
        if dimension == None:
            startBit = 0
            if bitSize != None:
                if len(self.linearFields) > 0:
                    lastField = self.linearFields[-1]
                    if lastField.isPointerType():
                        startBit = (1 + lastField.getEndBit()) % (8)
                    else:
                        startBit = (1 + lastField.getEndBit()) % (self.typeToSize[lastField.getTypeName()] * 8)
            fieldDescription = FieldDescription(fullyQualifiedName, fieldPath, fieldName, type, arrayIdx=None,
                                                isBitField=isBitField,
                                                startBit=startBit,
                                                bitsLength=bitSize, isPointerType=isPointerType)
            self.linearFields.append(fieldDescription)
        else:
            # on array decl
            for i in range(dimension):
                fieldDescription = FieldDescription("%s[%i]" % (fullyQualifiedName, i), fieldPath, fieldName, type,
                                                    arrayIdx=i, isBitField=isBitField, startBit=0, bitsLength=8,
                                                    isPointerType=isPointerType)
                self.linearFields.append(fieldDescription)

    def __createDescriptionDictIfNotAvailable(self, key):
        if key not in self.aggregatedLinearFields:
            self.aggregatedLinearFields[key] = OrderedDict()

    def aggregateDescriptions(self):
        currentBitFieldId = 0
        for description in reversed(self.linearFields):
            if description.isBitField():
                bitFieldKey = description.getFieldPath() + "<" + str(currentBitFieldId) + ">"
                self.__createDescriptionDictIfNotAvailable(bitFieldKey)
                self.aggregatedLinearFields[bitFieldKey][description.getStartBit()] = description
                if description.getStartBit() == 0:
                    currentBitFieldId += 1
            else:
                bitFieldKey = description.getDescription() + "<" + str(currentBitFieldId) + ">"
                self.__createDescriptionDictIfNotAvailable(bitFieldKey)
                self.aggregatedLinearFields[bitFieldKey][description.getStartBit()] = description
                currentBitFieldId += 1

class LinearStructFieldsToJson:
    jsonSource = None
    doInfer2ndByteFromUint16_t = Infer2ndByte
    doInferUpTo4thByte = InferUpTo4thByte

    def __init__(self):
        pass

    def __compactBitFieldDescriptions(self, fieldDescriptions):
        desc = "("
        isFirst = True
        for fieldDescription in reversed(fieldDescriptions.keys()):
            d = fieldDescriptions[fieldDescription]
            if not isFirst:
                desc += " | "
            isFirst = False

            if d.getStartBit() == d.getEndBit():
                fromTo = str(d.getStartBit())
            else:
                fromTo = str(d.getEndBit()) + ":" + str(d.getStartBit())

            desc += d.getFieldName() + "[" + fromTo + "]"
        return desc + ")"

    def __infer2ndByte(self, startAddressLabel, addressOffet, property, fieldDescription, entries, type,
                       nativeTypeToSize):
        if self.doInfer2ndByteFromUint16_t and \
                        fieldDescription.getTypeName() in nativeTypeToSize and \
                        2 == nativeTypeToSize[fieldDescription.getTypeName()]:
            for exception in Infer2ndByteExceptions:
                if exception["property"] == property and exception["type"] == type:
                    return

            newProperty = property + "[1]"
            if startAddressLabel != "":
                newAddress = startAddressLabel + "+" + str(addressOffet + 1)
            else:
                newAddress = addressOffet + 1

            newType = Infer2ndByteSimulatorTypeToNextSmallerType[type]
            entries[fieldDescription.getFieldPath()].append(
                {"property": newProperty, "type": newType, "address": newAddress})

    def __inferUpTo4ThByte(self, startAddressLabel, addressOffet, property, fieldDescription, entries, type,
                       nativeTypeToSize):
        if self.doInfer2ndByteFromUint16_t and \
                    fieldDescription.getTypeName() in nativeTypeToSize and \
                    4 == nativeTypeToSize[fieldDescription.getTypeName()]:
            for exception in InferUpTo4thByteExceptions:
                if exception["property"] == property and exception["type"] == type:
                    return

            for idx in range(1,4):
                newProperty = property + "["+str(idx)+"]"
                if startAddressLabel != "":
                    newAddress = startAddressLabel + "+" + str(addressOffet + idx)
                else:
                    newAddress = addressOffet + idx

                newType = type + "[" + str(idx) + "]"
                entries[fieldDescription.getFieldPath()].append(
                    {"property": newProperty, "type": newType, "address": newAddress})

    def toJson(self, structFields, nativeTypeToSize={}, nativeTypeToSimulatorType={}, startAddressLabel=""):

        addressOffet = 0
        entries = OrderedDict()
        for field in structFields:

            fieldDescription = structFields[field][structFields[field].keys()[0]]
            if fieldDescription.getFieldPath() not in entries:
                entries[fieldDescription.getFieldPath()] = []

            fieldTypeName = None
            if fieldDescription.isPointerType():
                fieldTypeName = "<pointerType>"
            else:
                fieldTypeName = fieldDescription.getTypeName()

            if fieldDescription.isBitField():
                type = NativeTypeToSimulatorBitfieldType[fieldDescription.getTypeName()]
                property = self.__compactBitFieldDescriptions(structFields[field])
            else:

                if 0 not in structFields[field]:
                    pass

                if fieldDescription.isArrayField():
                    property = structFields[field][0].getFieldName() + "[" + str(fieldDescription.getArrayIdx()) + "]"
                else:
                    try:
                        property = structFields[field][0].getFieldName()
                    except KeyError as ke:
                        print("key %s does not exist" % field)
                        # raise ke
                        pass

                type = nativeTypeToSimulatorType[fieldTypeName]

            self.__infer2ndByte(startAddressLabel, addressOffet, property, fieldDescription, entries,
                                type, nativeTypeToSize)
            self.__inferUpTo4ThByte(startAddressLabel, addressOffet, property, fieldDescription, entries,
                                type, nativeTypeToSize)

            if startAddressLabel != "":
                address = startAddressLabel + "+" + str(addressOffet)
            else:
                address = addressOffet
            addressOffet += nativeTypeToSize[fieldTypeName]

            entries[fieldDescription.getFieldPath()].append({"property": property, "type": type, "address": address})
        self.jsonSource = OrderedDict({"structs": entries})

    def show(self):
        print(json.dumps(self.jsonSource, sort_keys=False, indent=2, separators=(',', ': ')))

    def getDump(self):
        return json.dumps(self.jsonSource, sort_keys=False, indent=2, separators=(',', ': '))

    def getObject(self):
        return self.jsonSource

def parseEnumTypesAndStructs():
    ctp = CTypesParser()
    sdv = StructDeclVisitor()
    structs = ctp.getEntities(sdv)

    if Verbose:
        print("parsed structs")

    if Verbose:
        sdv.show()
    edv = EnumDeclVisitor()
    parsedEnums = ctp.getEntities(edv)

    if Verbose:
        print("\nparsed enums")
        edv.show()
    return structs, parsedEnums

def aggregateFields(structs, enumTypesToByteSize):
    enumToSize = {}
    for enum in enumTypesToByteSize:
        enumToSize[enum] = DefaultEnumByteSize
    typeToSizeMapping = dict(NativeTypeToSize.viewitems() | enumToSize.viewitems())
    composer = LinearStructComposer(typeToSizeMapping)
    traverseStructMembersDF(structs, NativeTypeToSize, enumTypesToByteSize, "Particle", "Particle",
                            callback=composer.consumeStructField)

    if Verbose:
        print("\nlinear ordered struct member list")
        for d in composer.linearFields:
            print ("%s" % d)
    composer.aggregateDescriptions()
    structFieldToDescription = OrderedDict()

    if Verbose:
        print("\naggregated linear ordered struct member list")
    for fieldPath in reversed(composer.aggregatedLinearFields.keys()):
        if Verbose:
            print ("%s" % fieldPath)
        fieldDescriptions = composer.aggregatedLinearFields[fieldPath]
        structFieldToDescription[fieldPath] = OrderedDict()
        for startBitPosition in reversed(fieldDescriptions.keys()):
            field = fieldDescriptions[startBitPosition]
            if Verbose:
                print(" -- %s" % (field))
            structFieldToDescription[fieldPath][field.getStartBit()] = field
    return enumToSize, structFieldToDescription

def generateCStructJson(enumToSize, structFieldToDescription):
    appendEnumToSize(enumToSize, NativeTypeToSize)
    appendEnumNames(enumToSize, NativeTypeToSimulatorType)

    lsfj = LinearStructFieldsToJson()
    lsfj.toJson(structFieldToDescription, nativeTypeToSize=NativeTypeToSize,
                nativeTypeToSimulatorType=NativeTypeToSimulatorType, startAddressLabel="globalStateBase")
    # lsfj.show()
    return lsfj.getObject(), lsfj.getDump()

class Gui:
    def __init__(self, txt):
        self.rootWindow = Tk()
        self.rootWindow.resizable(True, False)
        self.rootWindow.title("content is copied to clipboard")

        self.rootWindow.clipboard_clear()
        self.rootWindow.clipboard_append(txt)

        sb = Scrollbar(self.rootWindow)
        sb.pack(side=RIGHT, fill=Y)
        textField = Text(self.rootWindow, height=30, width=74)
        textField.pack(side=LEFT, fill=Y)
        sb.config(command=textField.yview)
        textField.config(yscrollcommand=sb.set)
        textField.insert(END, "\n" + txt + "\n")

        self.rootWindow.mainloop()

def aggregateToJsonDumpableDescription(structsObject, enumsObject):
    jsonDescription["enums"] = enumsObject["enums"]

    # add sizeof structs
    jsonDescription["sizeofTypes"] = OrderedDict()
    for enumType in enumToSize:
        jsonDescription["sizeofTypes"][enumType] = DefaultEnumByteSize

    # add labels
    jsonDescription["labels"] = JsonConfig.Labels["labels"]

    jsonDescription["structs"] = OrderedDict()
    # append staticly defined register description
    for key in JsonConfig.HardwareRegisters["structs"]:
        jsonDescription["structs"][key] = JsonConfig.HardwareRegisters["structs"][key]

    # append parsed c structs
    for key in structsObject["structs"]:
        jsonDescription["structs"][key] = structsObject["structs"][key]

    if Verbose:
        print("\ntype overridings")
    # override types
    for descriptionSetKey in jsonDescription["structs"]:
        for description in jsonDescription["structs"][descriptionSetKey]:
            # jsonDescription["structs"][descriptionSet][key]
            for override in TypeOverrides:
                substring = override["property"]
                oldType = override["oldType"]
                newType = override["newType"]
                if substring in description["property"] and oldType == description["type"]:
                    if Verbose:
                        print(descriptionSetKey + "." + description["property"] + " " + description[
                            "type"] + " <== " + newType)
                    description["type"] = newType

    return jsonDescription

def simulatorToSize(simulatorType, nativeToSimulator, nativeToSize):
    for nativeType in nativeToSimulator:
        if nativeToSimulator[nativeType] == simulatorType:
            for nativeToSizeKey in nativeToSize:
                if nativeToSizeKey == nativeType:
                    return nativeToSize[nativeToSizeKey]

if __name__ == "__main__":
    structs, parsedEnums = parseEnumTypesAndStructs()
    enumToSize, structFieldToDescription = aggregateFields(structs, parsedEnums)

    jsonDescription = OrderedDict()

    etj = EnumsToJson()
    enumsObject = etj.getObject()

    structsObject, jsonDump = generateCStructJson(enumToSize, structFieldToDescription)
    # print(jsonDump)

    jsonDescription = aggregateToJsonDumpableDescription(structsObject, enumsObject)

    totalSize = 0
    for struct in jsonDescription["structs"]:
        for field in jsonDescription["structs"][struct]:
            totalSize += simulatorToSize("bit", JsonConfig.NativeTypeToSimulatorType, JsonConfig.NativeTypeToSize)
    print("Total struct size: %s Bytes" % totalSize)
    # print(json.dumps(jsonDescription, sort_keys=False, indent=2, separators=(',', ': ')))
    Gui(json.dumps(jsonDescription, sort_keys=False, indent=2, separators=(',', ': ')))


x = int(input())
y = int(input())
index = 0
if(x>=y):
    print(x-y)
else:
    while x<y:
        if(y%2==0):
            y = y//2
        else:
            y+=1
    print(index+x-y)


import numpy as np

# 手动实现计算细节
def cos_calculate_dingyi(ab, cd):
    sum_ab = 0
    sum_cd = 0
    for i in range(len(ab)):
        sum_ab += ab[i] ** 2
    sum_ab = np.sqrt(sum_ab)
    for i in range(len(cd)):
        sum_cd += cd[i] ** 2
    sum_cd = np.sqrt(sum_cd)
    cos_ab_cd = ab.dot(cd)/(sum_ab * sum_cd)
    return cos_ab_cd

#  直接调用numpy包中的相关计算公式
def cos_calculate_gongshi(ab, cd):

    return ab.dot(cd)/((np.linalg.norm(ab))*(np.linalg.norm(cd)))

# 定义坐标系中的两个向量a和b
ab = np.array([1, 2, 3, 8.8])
cd = np.array([3, 6, 7, 9.8])

cos_ab_cd_gongshi = cos_calculate_gongshi(ab, cd)
print("向量ab和cd之间的夹角余弦值为(公式计算)：", cos_ab_cd_gongshi)

cos_ab_cd_dingyi = cos_calculate_dingyi(ab, cd)
print("向量ab和cd之间的夹角余弦值为（手动实现）：", cos_ab_cd_dingyi)


#!/usr/bin/env python
import cv2
import numpy as np

if __name__ == '__main__':
    cap = cv2.VideoCapture(0)
    ret, frame = cap.read()
    cv2.imwrite('rpi_image.jpg', frame)
    print("Hello")
    cap.release()


import random
from urllib.parse import urlencode

import requests

from ..engine.decorators import Plugin
from ..plugins import BatchCheckBase, match1
from ..engine.download import DownloadBase

VALID_URL_BASE = r'(?:https?://)?(?:(?:www|go|m)\.)?twitch\.tv/(?P<id>[0-9_a-zA-Z]+)'
_OPERATION_HASHES = {
    'CollectionSideBar': '27111f1b382effad0b6def325caef1909c733fe6a4fbabf54f8d491ef2cf2f14',
    'FilterableVideoTower_Videos': 'a937f1d22e269e39a03b509f65a7490f9fc247d7f83d6ac1421523e3b68042cb',
    'ClipsCards__User': 'b73ad2bfaecfd30a9e6c28fada15bd97032c83ec77a0440766a56fe0bd632777',
    'ChannelCollectionsContent': '07e3691a1bad77a36aba590c351180439a40baefc1c275356f40fc7082419a84',
    'StreamMetadata': '1c719a40e481453e5c48d9bb585d971b8b372f8ebb105b17076722264dfa5b3e',
    'ComscoreStreamingQuery': 'e1edae8122517d013405f237ffcc124515dc6ded82480a88daef69c83b53ac01',
    'VideoPreviewOverlay': '3006e77e51b128d838fa4e835723ca4dc9a05c5efd4466c1085215c6e437e65c',
}
_CLIENT_ID = 'kimne78kx3ncx6brgo4mv6wki5h1ko'

@Plugin.download(regexp=VALID_URL_BASE)
class Twitch(DownloadBase):
    def __init__(self, fname, url, suffix='flv'):
        DownloadBase.__init__(self, fname, url, suffix=suffix, opt_args=['-ss', "00:00:16"])

    def check_stream(self):
        if not list(Twitch.BatchCheck([self.url]).check()):
            return
        channel_name = match1(self.url, VALID_URL_BASE)
        r = requests.get(f'https://api.twitch.tv/api/channels/{channel_name}/access_token',
                         headers={
                             'Accept': 'application/vnd.twitchtv.v5+json; charset=UTF-8',
                             'Client-ID': _CLIENT_ID,
                         }, timeout=10)
        r.close()
        access_token = r.json()
        token = access_token['token']
        query = {
            'allow_source': 'true',
            'allow_audio_only': 'true',
            'allow_spectre': 'true',
            'p': random.randint(1000000, 10000000),
            'player': 'twitchweb',
            'playlist_include_framerate': 'true',
            'segment_preference': '4',
            'sig': access_token['sig'],
            'token': token,
        }
        self.raw_stream_url = f'https://usher.ttvnw.net/api/channel/hls/{channel_name}.m3u8?{urlencode(query)}'
        return True

    class BatchCheck(BatchCheckBase):
        def __init__(self, urls):
            BatchCheckBase.__init__(self, pattern_id=VALID_URL_BASE, urls=urls)

        def check(self):
            gql = self.get_streamer()
            i = -1
            for data in gql:
                i += 1
                user = data['data'].get('user')
                if not user:
                    continue
                stream = user['stream']
                if not stream:
                    continue
                yield self.usr_dict.get(self.usr_list[i])

        def get_streamer(self):
            ops = []
            for channel_name in self.usr_list:
                op = {
                    'operationName': 'StreamMetadata',
                    'variables': {'channelLogin': channel_name.lower()}
                }
                op['extensions'] = {
                    'persistedQuery': {
                        'version': 1,
                        'sha256Hash': _OPERATION_HASHES[op['operationName']],
                    }
                }
                ops.append(op)
            gql = requests.post(
                'https://gql.twitch.tv/gql',
                json=ops,
                headers={
                    'Content-Type': 'text/plain;charset=UTF-8',
                    'Client-ID': _CLIENT_ID,
                }, timeout=15)
            gql.close()
            return gql.json()


import numpy as np
import scipy.io
from nn import *
from collections import Counter

train_data = scipy.io.loadmat('../data/nist36_train.mat')
valid_data = scipy.io.loadmat('../data/nist36_valid.mat')

# we don't need labels now!
train_x = train_data['train_data']
valid_x = valid_data['valid_data']

max_iters = 100
# pick a batch size, learning rate
batch_size = 36
learning_rate =  3e-5
hidden_size = 32
lr_rate = 20
batches = get_random_batches(train_x,np.ones((train_x.shape[0],1)),batch_size)
batch_num = len(batches)

params = Counter()

# Q5.1 & Q5.2
# initialize layers here
initialize_weights(train_x.shape[1], hidden_size , params,'layer1')
initialize_weights(hidden_size , hidden_size ,params,'hidden')
initialize_weights(hidden_size , hidden_size ,params,'hidden2')
initialize_weights(hidden_size , train_x.shape[1] ,params,'output')

keys = [key for key in params.keys()]
for k in keys:
    params['m_' + k] = np.zeros(params[k].shape)

train_loss = []
# should look like your previous training loops
for itr in range(max_iters):
    total_loss = 0
    for xb,_ in batches:
        # training loop can be exactly the same as q2!
        # your loss is now squared error
        # delta is the d/dx of (x-y)^2
        # to implement momentum
        #   just use 'm_'+name variables
        #   to keep a saved value over timestamps
        #   params is a Counter(), which returns a 0 if an element is missing
        #   so you should be able to write your loop without any special conditions

        h1 = forward(xb,params,'layer1',relu)
        h2 = forward(h1, params,'hidden',relu)
        h3 = forward(h2, params,'hidden2',relu)
        probs = forward(h3,params,'output',sigmoid)

        loss = np.sum(np.square(probs-xb))
        total_loss += loss

        # Implement backwards!
        delta1 = 2.0*(probs - xb)
        delta2 = backwards(delta1,params,'output',sigmoid_deriv)
        delta3 = backwards(delta2,params,'hidden2',relu_deriv)
        delta4 = backwards(delta3, params,'hidden',relu_deriv)
        backwards(delta4, params,'layer1',relu_deriv)

        for k in params.keys():
            if '_' in k:
                continue
            params['m_' + k] = 0.9*params['m_' + k] - learning_rate*params['grad_'+ k ]
            params[k] += params['m_' + k]

    train_loss.append(total_loss)
    if itr % 2 == 0:
        print("itr: {:02d} \t loss: {:.2f} ".format(itr,total_loss))

    if itr % lr_rate == lr_rate - 1:
        learning_rate *= 0.9

# 5.2
import matplotlib.pyplot as plt
plt.figure("Loss")
plt.plot(range(max_iters), train_loss)
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.show()

# Q5.3.1
import matplotlib.pyplot as plt
# visualize some results
h1 = forward(valid_x,params,'layer1',relu)
h2 = forward(h1, params,'hidden',relu)
h3 = forward(h2, params,'hidden2',relu)
valid_out = forward(h3,params,'output',sigmoid)

idx = np.random.choice(valid_x.shape[0],size=7,replace=False) # random indices
for i in idx:
    plt.subplot(2,1,1)
    plt.imshow(valid_x[i].reshape(hidden_size, hidden_size).T)
    plt.subplot(2,1,2)
    plt.imshow(valid_out[i].reshape(hidden_size, hidden_size).T)
    plt.show()

# Q5.3.2
from skimage.measure import compare_psnr as psnr
# evaluate PSNR
noise_sig = 0
for i in range(valid_out.shape[0]):
    noise_sig += psnr(vaild_x[i], valid_out[i])

noise_sig /= valid_out.shape[0]
print(noise_sig)


from __future__ import print_function
import re
try:
    from urllib.parse import urlparse, parse_qsl
except ImportError:
    from urlparse import urlparse, parse_qsl
from collections import defaultdict
from base64 import b64encode, b64decode
import platform
import os.path as op
import os
from os import makedirs
import stat
import tempfile
import shutil
import subprocess
import sys
import shlex
import pkg_resources
import pip

# TODO bundle own version of distlib. Perhaps
try:
    from pip._vendor.distlib import locators
except ImportError:
    from pip.vendor.distlib import locators

try:
    str_types = (basestring,)
except NameError:
    str_types = (str, bytes)

def print_message(*a, **kw):
    print(*a, **kw)

class PundleException(Exception):
    pass

def python_version_string():
    version_info = sys.pypy_version_info if platform.python_implementation() == 'PyPy' else sys.version_info
    version_string = '{v.major}.{v.minor}.{v.micro}'.format(v=version_info)
    build, _ = platform.python_build()
    return '{}-{}-{}'.format(platform.python_implementation(), version_string, build)

def parse_file(filename):
    res = []
    with open(filename) as f:
        for line in f:
            s = line.strip()
            if s and not s.startswith('#'):
                if s.startswith('-r'):
                    continue
                if s.startswith('-e '):
                    s = s[3:]
                if parse_vcs_requirement(s):
                    res.append(s)
                else:
                    req = shlex.split(s, comments=True)
                    res.append(req[0])
    return res

def test_vcs(req):
    return '+' in req and req.index('+') == 3

def parse_vcs_requirement(req):
    if '+' not in req:
        return None
    vcs, url = req.split('+', 1)
    if vcs not in ('git', 'svn', 'hg'):
        return None
    parsed_url = urlparse(url)
    parsed = dict(parse_qsl(parsed_url.fragment))
    if 'egg' not in parsed:
        return None
    egg = parsed['egg'].rsplit('-', 1)
    if len(egg) > 1:
        try:
            pkg_resources.SetuptoolsVersion(egg[1])
        except pkg_resources._vendor.packaging.version.InvalidVersion:
            return parsed['egg'].lower(), req, None
        return egg[0].lower(), req, egg[1]
    else:
        return parsed['egg'].lower(), req, None

def parse_frozen_vcs(req):
    res = parse_vcs_requirement(req)
    if not res:
        return
    return res[0], res[1]

class VCSDist(object):
    def __init__(self, directory):
        self.dir = directory
        name = op.split(directory)[-1]
        key, encoded = name.split('+', 1)
        self.key = key.lower()
        self.line = b64decode(encoded).decode('utf-8')
        egg, req, version = parse_vcs_requirement(self.line)
        version = version or '0.0.0'
        self.hashcmp = (pkg_resources.SetuptoolsVersion(version), -1, egg, self.dir)
        self.version = self.line
        self.pkg_resource = next(iter(pkg_resources.find_distributions(self.dir, True)), None)
        self.location = self.pkg_resource.location

    def requires(self, extras=[]):
        return self.pkg_resource.requires(extras=extras)

    def activate(self):
        return self.pkg_resource.activate()

    def __lt__(self, other):
        return self.hashcmp.__lt__(other.hashcmp)

class CustomReq(object):
    def __init__(self, line, env, source=None):
        self.line = line
        self.egg = None
        if isinstance(line, pkg_resources.Requirement):
            self.req = line
        elif test_vcs(line):
            res = parse_vcs_requirement(line)
            if not res:
                raise PundleException('Bad url %r' % line)
            egg, req, version = res
            self.egg = egg
            self.req = None  # pkg_resources.Requirement.parse(res)
        else:
            self.req = pkg_resources.Requirement.parse(line)
        self.sources = set([source])
        self.envs = set()
        self.add_env(env)

    def __contains__(self, something):
        if self.req:
            return (something in self.req)
        elif self.egg:
            return something == self.line
        else:
            return False

    def __repr__(self):
        return '<CustomReq %r>' % self.__dict__

    def why_str(self):
        if len(self.sources) < 2:
            return '{} << {}'.format(self.line, self.why_str_one(list(self.sources)[0]))
        causes = list(sorted(self.why_str_one(source) for source in self.sources))
        return '{} << [{}]'.format(self.line, ' | '.join(causes))

    def why_str_one(self, source):
        if isinstance(source, str_types):
            return source
        elif isinstance(source, CustomReq):
            return source.why_str()
        return '?'

    def adjust_with_req(self, req):
        if not self.req:
            return
            raise PundleException('VCS')
        versions = ','.join(''.join(t) for t in set(self.req.specs + req.req.specs))
        self.requirement = pkg_resources.Requirement.parse('{} {}'.format(
            self.req.project_name, versions
        ))
        self.sources.update(req.sources)
        self.add_env(req.envs)

    @property
    def key(self):
        return self.req.key if self.req else self.egg

    @property
    def extras(self):
        return self.req.extras if self.req else []

    def locate(self, suite, prereleases=False):
        # requirements can have somethin after `;` symbol that `locate` does not understand
        req = str(self.req).split(';', 1)[0]
        dist = suite.locate(req, prereleases=prereleases)
        if not dist:
            # have not find any releases so search for pre
            dist = suite.locate(req, prereleases=True)
        if not dist:
            raise PundleException('%s can not be located' % self.req)
        return dist

    def locate_and_install(self, suite, installed=None, prereleases=False):
        if self.egg:
            key = b64encode(self.line.encode('utf-8')).decode()
            target_dir = op.join(suite.parser.directory, '{}+{}'.format(self.egg, key))
            target_req = self.line
            ready = [
                installation
                for installation in (installed or [])
                if getattr(installation, 'line', None) == self.line
            ]
        else:
            loc_dist = self.locate(suite, prereleases=prereleases)
            ready = [
                installation
                for installation in (installed or [])
                if installation.version == loc_dist.version
            ]
            target_dir = op.join(suite.parser.directory, '{}-{}'.format(loc_dist.key, loc_dist.version))
            # DEL? target_req = '%s==%s' % (loc_dist.name, loc_dist.version)
            # If we use custom index, then we want not to configure PIP with it
            # and just give it URL
            target_req = loc_dist.download_url
        if ready:
            return ready[0]
        try:
            makedirs(target_dir)
        except OSError:
            pass
        tmp_dir = tempfile.mkdtemp()
        try:
            pip.main([
                'install',
                '--no-deps',
                '-t', tmp_dir,
                target_req
            ])
            for item in os.listdir(tmp_dir):
                shutil.move(op.join(tmp_dir, item), op.join(target_dir, item))
        except Exception as exc:
            raise PundleException('%s was not installed due error %s' % (self.egg or loc_dist.name, exc))
        finally:
            shutil.rmtree(tmp_dir, ignore_errors=True)
        return next(iter(pkg_resources.find_distributions(target_dir, True)), None)

    def add_env(self, env):
        if isinstance(env, str):
            self.envs.add(env)
        else:
            self.envs.update(env)

class RequirementState(object):
    def __init__(self, key, req=None, frozen=None, installed=None):
        self.key = key
        self.requirement = req
        self.frozen = frozen
        self.installed = installed or []
        self.installed.sort()
        self.installed.reverse()

    def __repr__(self):
        return '<RequirementState %r>' % self.__dict__

    def adjust_with_req(self, req):
        if self.requirement:
            self.requirement.adjust_with_req(req)
        else:
            self.requirement = req

    def has_correct_freeze(self):
        return self.requirement and self.frozen and self.frozen in self.requirement

    def check_installed_version(self, suite, install=False):
        # install version of package if not installed
        dist = None
        if self.has_correct_freeze():
            dist = [
                installation
                for installation in self.installed
                if pkg_resources.parse_version(installation.version) == pkg_resources.parse_version(self.frozen)
            ]
            dist = dist[0] if dist else None
            if install and not dist:
                dist = self.install_frozen(suite)
        if install and not dist:
            dist = self.requirement.locate_and_install(suite, installed=self.get_installed())
            self.frozen = dist.version
            self.installed.append(dist)
            self.frozen = dist.version
        return dist

    def get_installed(self):
        return [installation for installation in self.installed if installation.version in self.requirement]

    def upgrade(self, suite, prereleases=False):
        # check if we have fresh packages on PIPY
        dists = self.get_installed()
        dist = dists[0] if dists else None
        latest = self.requirement.locate(suite, prereleases=prereleases)
        if not dist or pkg_resources.parse_version(latest.version) > pkg_resources.parse_version(dist.version):
            print_message('Upgrade to', latest)
            dist = self.requirement.locate_and_install(suite, installed=self.get_installed(), prereleases=prereleases)
        # Anyway use latest available dist
        self.frozen = dist.version
        self.installed.append(dist)
        return dist

    def reveal_requirements(self, suite, install=False, upgrade=False, already_revealed=None, prereleases=False):
        already_revealed = already_revealed.copy() if already_revealed is not None else set()
        if self.key in already_revealed:
            return
        if upgrade:
            dist = self.upgrade(suite, prereleases=prereleases)
        else:
            dist = self.check_installed_version(suite, install=install)
        if not dist:
            return
        already_revealed.add(self.key)
        for req in dist.requires(extras=self.requirement.extras):
            suite.adjust_with_req(
                CustomReq(str(req), self.requirement.envs, source=self.requirement),
                install=install,
                upgrade=upgrade,
                already_revealed=already_revealed,
            )

    def frozen_dump(self):
        if self.requirement.egg:
            return self.requirement.line
        main = '{}=={}'.format(self.key, self.frozen)
        comment = self.requirement.why_str()
        return '{:20s} # {}'.format(main, comment)

    def frozen_dist(self):
        if not self.frozen:
            return
        for dist in self.installed:
            if pkg_resources.parse_version(dist.version) == pkg_resources.parse_version(self.frozen):
                return dist

    def install_frozen(self, suite):
        if self.frozen_dist() or not self.frozen:
            return
        envs = self.requirement.envs if self.requirement else ''
        frozen_req = CustomReq("{}=={}".format(self.key, self.frozen), envs)
        dist = frozen_req.locate_and_install(suite)
        self.installed.append(dist)
        return dist

    def activate(self):
        dist = self.frozen_dist()
        if not dist:
            raise PundleException('Distribution is not installed %s' % self.key)
        dist.activate()
        pkg_resources.working_set.add_entry(dist.location)
        # find end execute *.pth
        sitedir = dist.location  # noqa some PTH search for sitedir
        for filename in os.listdir(dist.location):
            if not filename.endswith('pth'):
                continue
            try:
                for line in open(op.join(dist.location, filename)):
                    if line.startswith('import '):
                        exec(line.strip())
            except Exception as e:
                print('Erroneous pth file %r' % op.join(dist.location, filename))
                print(e)

class AggregatingLocator(object):
    def __init__(self, locators):
        self.locators = locators

    def locate(self, req, **kw):
        for locator in self.locators:
            print('try', locator, 'for', req)
            revealed = locator.locate(req, **kw)
            if revealed:
                return revealed

class Suite(object):
    """
    Main object that represents current directory pundle state
    """
    def __init__(self, parser, envs=[], urls=None):
        self.states = {}
        self.parser = parser
        self.envs = envs
        self.urls = urls or ['https://pypi.python.org/simple/']
        if 'PIP_EXTRA_INDEX_URL' in os.environ:
            self.urls.append(os.environ['PIP_EXTRA_INDEX_URL'])
        self.locators = []
        for url in self.urls:
            self.locators.append(
                locators.SimpleScrapingLocator(url, timeout=3.0, scheme='legacy')
            )
        self.locators.append(locators.JSONLocator(scheme='legacy'))
        self.locator = AggregatingLocator(self.locators)

    def use(self, key):
        """For single mode
        You can call suite.use('arrow') and then `import arrow`

        :key: package name
        """
        self.adjust_with_req(CustomReq(key, ''))
        self.install()
        self.activate_all()

    def locate(self, *a, **kw):
        return self.locator.locate(*a, **kw)

    def add(self, key, state):
        self.states[key] = state

    def __repr__(self):
        return '<Suite %r>' % self.states

    def required_states(self):
        return [state for state in self.states.values() if state.requirement]

    def need_freeze(self, verbose=False):
        self.install(install=False)
        not_correct = not all(state.has_correct_freeze() for state in self.required_states())
        if not_correct and verbose:
            for state in self.required_states():
                if not state.has_correct_freeze():
                    print(
                        state.key,
                        'Need',
                        state.requirement,
                        'have not be frozen',
                        state.frozen,
                        ', installed',
                        state.installed
                    )
        # TODO
        # unneeded = any(state.frozen for state in self.states.values() if not state.requirement)
        # if unneeded:
        #     print('!!! Unneeded', [state.key for state in self.states.values() if not state.requirement])
        return not_correct  # or unneeded

    def adjust_with_req(self, req, install=False, upgrade=False, already_revealed=None):
        state = self.states.get(req.key)
        if not state:
            state = RequirementState(req.key, req=req)
            self.add(req.key, state)
        else:
            state.adjust_with_req(req)
        state.reveal_requirements(self, install=install, upgrade=upgrade, already_revealed=already_revealed or set())

    def install(self, install=True):
        for state in self.required_states():
            state.reveal_requirements(self, install=install)

    def upgrade(self, key=None, prereleases=False):
        states = [self.states[key]] if key else self.required_states()
        for state in states:
            print('Check', state.requirement.req)
            state.reveal_requirements(self, upgrade=True, prereleases=prereleases)

    def dump_frozen(self, env):
        return '\n'.join(sorted(
            state.frozen_dump()
            for state in self.required_states()
            if state.requirement and env in state.requirement.envs
        )) + '\n'

    def need_install(self):
        return not all(state.frozen_dist() for state in self.states.values() if state.frozen)

    def install_frozen(self):
        for state in self.states.values():
            state.install_frozen(self)

    def activate_all(self, env=''):
        for state in self.required_states():
            if '' in state.requirement.envs or env in state.requirement.envs:
                state.activate()

    def save_frozen(self):
        "Saves frozen files to disk"
        for env in self.parser.envs():
            frozen_file = self.parser.get_frozen_file(env)
            with open(frozen_file, 'w') as f:
                f.write(self.dump_frozen(env))

class Parser(object):
    """Gather environment info, requirements,
    frozen packages and create Suite object
    """
    def __init__(
            self,
            base_path=None,
            directory='Pundledir',
            requirements_files=None,
            frozen_files={'': 'frozen.txt'},
            package=None,
    ):
        self.base_path = base_path or '.'
        self.directory = directory
        self.requirements_files = requirements_files
        self.frozen_files = frozen_files
        self.package = package
        self.package_envs = set([''])

    def envs(self):
        if self.requirements_files:
            return list(self.requirements_files.keys()) or ['']
        elif self.package:
            return self.package_envs
        return ['']

    def get_frozen_file(self, env):
        if env in self.frozen_files:
            return self.frozen_files[env]
        else:
            return os.path.join(self.base_path, 'frozen_%s.txt' % env)

    def create_suite(self):
        reqs, freezy, diry = self.parse_requirements(), self.parse_frozen(), self.parse_directory()
        state_keys = set(list(reqs.keys()) + list(freezy.keys()) + list(diry.keys()))
        suite = Suite(self, envs=self.envs())
        for key in state_keys:
            suite.add(
                key,
                RequirementState(key, reqs.get(key), freezy.get(key), diry.get(key, [])),
            )
        return suite

    def parse_directory(self):
        if not op.exists(self.directory):
            return {}
        dists = [
            # this magic takes first element or None
            next(iter(
                pkg_resources.find_distributions(op.join(self.directory, item), True)
            ), None)
            for item in os.listdir(self.directory) if '-' in item
        ]
        dists.extend(
            VCSDist(op.join(self.directory, item))
            for item in os.listdir(self.directory) if '+' in item
        )
        dists = filter(None, dists)
        result = defaultdict(list)
        for dist in dists:
            result[dist.key].append(dist)
        return result

    def parse_frozen(self):
        frozen_versions = {}
        for env in self.envs():
            frozen_file = self.get_frozen_file(env)
            if op.exists(frozen_file):
                frozen = [
                    (parse_frozen_vcs(line) or line.split('=='))
                    for line in parse_file(frozen_file)
                ]
            else:
                frozen = []
            for name, version in frozen:
                frozen_versions[name.lower()] = version
        return frozen_versions

    def parse_requirements(self):
        if self.requirements_files:
            all_requirements = {}
            for env, req_file in self.requirements_files.items():
                requirements = parse_file(req_file)
                if env:
                    source = 'requirements %s file' % env
                else:
                    source = 'requirements file'
                for req in (CustomReq(line, env, source=source) for line in requirements):
                    if req.key in all_requirements:
                        # if requirements exists in other env, then add this env too
                        all_requirements[req.key].add_env(env)
                    else:
                        all_requirements[req.key] = req
            return all_requirements
        elif self.package:
            setup_info = get_info_from_setup(self.package)
            if setup_info is None:
                raise PundleException('There is no requirements.txt nor setup.py')
            install_requires = setup_info.get('install_requires') or []
            reqs = [
                CustomReq(str(req), '', source='setup.py')
                for req in install_requires
            ]
            requirements = dict((req.key, req) for req in reqs)
            # we use `feature` as environment for pundle
            extras_require = (setup_info.get('extras_require') or {})
            for feature, feature_requires in extras_require.items():
                for req_line in feature_requires:
                    req = CustomReq(req_line, feature, source='setup.py')
                    # if this req already in dict, then add our feature as env
                    if req.key in requirements:
                        requirements[req.key].add_env(feature)
                    else:
                        requirements[req.key] = req
                self.package_envs.add(feature)
            return requirements
        return {}

# Utilities
def get_info_from_setup(path):
    """Mock setuptools.setup(**kargs) to get
    package information about requirements and extras
    """
    preserve = {}

    def _save_info(**setup_args):
        preserve['args'] = setup_args

    import setuptools
    setuptools.setup = _save_info
    import runpy
    runpy.run_path(os.path.join(path, 'setup.py'), run_name='__main__')
    return preserve.get('args')

def search_files_upward(start_path=None):
    "Search for requirements.txt upward"
    if not start_path:
        start_path = op.abspath(op.curdir)
    if op.exists(op.join(start_path, 'requirements.txt')) or op.exists(op.join(start_path, 'setup.py')):
        return start_path
    up_path = op.abspath(op.join(start_path, '..'))
    if op.samefile(start_path, up_path):
        return None
    return search_files_upward(start_path=up_path)

def find_all_prefixed_files(directory, prefix):
    "find all requirements_*.txt files"
    envs = {}
    for entry in os.listdir(directory):
        if not entry.startswith(prefix):
            continue
        name, ext = op.splitext(entry)
        env = name[len(prefix):].lstrip('_')
        envs[env] = op.join(directory, entry)
    return envs

def create_parser_parameters():
    base_path = search_files_upward()
    if not base_path:
        raise PundleException('Can not find requirements.txt nor setup.py')
    py_version_path = python_version_string()
    pundledir_base = os.environ.get('PUNDLEDIR') or op.join(op.expanduser('~'), '.pundledir')
    if op.exists(op.join(base_path, 'requirements.txt')):
        requirements_files = find_all_prefixed_files(base_path, 'requirements')
    else:
        requirements_files = {}
    envs = list(requirements_files.keys()) or ['']
    params = {
        'base_path': base_path,
        'frozen_files': {
            env: op.join(base_path, 'frozen_%s.txt' % env if env else 'frozen.txt')
            for env in envs
        },
        'directory': op.join(pundledir_base, py_version_path),
    }
    if requirements_files:
        params['requirements_files'] = requirements_files
    elif op.exists(op.join(base_path, 'setup.py')):
        params['package'] = base_path
    else:
        return
    return params

def create_parser_or_exit():
    parser_kw = create_parser_parameters()
    if not parser_kw:
        print_message('You have not requirements.txt. Create it and run again.')
        exit(1)
    return parser_kw

# Commands
def upgrade_all(*a, **kw):
    key = kw.pop('key')
    prereleases = kw.pop('prereleases')
    suite = Parser(*a, **kw).create_suite()
    suite.need_freeze()
    suite.upgrade(key=key, prereleases=prereleases)
    suite.install()
    suite.save_frozen()

def install_all(*a, **kw):
    suite = Parser(*a, **kw).create_suite()
    if suite.need_freeze() or suite.need_install():
        print_message('Install some packages')
        suite.install()
    else:
        print_message('Nothing to do, all packages installed')
    suite.save_frozen()
    return suite

def activate():
    parser_kw = create_parser_parameters()
    if not parser_kw:
        raise PundleException('Can`t create parser parameters')
    suite = Parser(**parser_kw).create_suite()
    if suite.need_freeze(verbose=True):
        raise PundleException('frozen file is outdated')
    if suite.need_install():
        raise PundleException('Some dependencies not installed')
    suite.activate_all(env=os.environ.get('PUNDLEENV') or '')
    return suite

FIXATE_TEMPLATE = """
# pundle user customization start
import pundle; pundle.activate()
# pundle user customization end
"""

def fixate():
    "puts activation code to usercustomize.py for user"
    print_message('Fixate')
    import site
    userdir = site.getusersitepackages()
    if not userdir:
        raise PundleException('Can`t fixate due user have not site package directory')
    try:
        makedirs(userdir)
    except OSError:
        pass
    template = FIXATE_TEMPLATE.replace('op.dirname(__file__)', "'%s'" % op.abspath(op.dirname(__file__)))
    usercustomize_file = op.join(userdir, 'usercustomize.py')
    print_message('Will edit %s file' % usercustomize_file)
    if op.exists(usercustomize_file):
        content = open(usercustomize_file).read()
        if '# pundle user customization start' in content:
            regex = re.compile(r'\n# pundle user customization start.*# pundle user customization end\n', re.DOTALL)
            content, res = regex.subn(template, content)
            open(usercustomize_file, 'w').write(content)
        else:
            open(usercustomize_file, 'a').write(content)
    else:
        open(usercustomize_file, 'w').write(template)
    link_file = op.join(userdir, 'pundle.py')
    if op.lexists(link_file):
        print_message('Remove exist link to pundle')
        os.unlink(link_file)
    print_message('Create link to pundle %s' % link_file)
    os.symlink(op.abspath(__file__), link_file)
    print_message('Complete')

def entry_points():
    suite = activate()
    entries = {}
    for r in suite.states.values():
        d = r.frozen_dist()
        if not d:
            continue
        scripts = d.get_entry_map().get('console_scripts', {})
        for name in scripts:
            entries[name] = d
    return entries

def execute(interpreter, cmd, args):
    # TODO proof implementation
    # clean it
    entries = entry_points()
    if cmd not in entries:
        print('Script is not found. Check if package is installed, or look at the `pundle entry_points`')
        sys.exit(1)
    exc = entries[cmd].get_entry_info('console_scripts', cmd).load(require=False)
    sys.path.insert(0, '')
    sys.argv = [cmd] + args
    exc()

class CmdRegister:
    commands = {}
    ordered = []

    @classmethod
    def cmdline(cls, *cmd_aliases):
        def wrap(func):
            for alias in cmd_aliases:
                cls.commands[alias] = func
                cls.ordered.append(alias)
        return wrap

    @classmethod
    def help(cls):
        for alias in cls.ordered:
            if not alias:
                continue
            print("{:15s} {}".format(alias, cls.commands[alias].__doc__))

    @classmethod
    def main(cls):
        alias = '' if len(sys.argv) == 1 else sys.argv[1]
        if alias == 'help':
            cls.help()
            return
        if alias not in cls.commands:
            print('Unknown command\nTry this:')
            cls.help()
            sys.exit(1)
        cls.commands[alias]()

@CmdRegister.cmdline('', 'install')
def cmd_install():
    "Install packages by frozen.txt and resolve ones that was not frozen"
    install_all(**create_parser_or_exit())

@CmdRegister.cmdline('upgrade')
def cmd_upgrade():
    """
    [package [pre]] if package provided will upgrade it and dependencies or all packages from PyPI.
    If `pre` provided will look for prereleases.
    """
    key = sys.argv[2] if len(sys.argv) > 2 else None
    prereleases = sys.argv[3] == 'pre' if len(sys.argv) > 3 else False
    upgrade_all(key=key, prereleases=prereleases, **create_parser_or_exit())

CmdRegister.cmdline('fixate')(fixate)

@CmdRegister.cmdline('exec')
def cmd_exec():
    "executes setuptools entry"
    execute(sys.argv[0], sys.argv[2], sys.argv[3:])

@CmdRegister.cmdline('entry_points')
def cmd_entry_points():
    "prints available setuptools entries"
    for entry, package in entry_points().items():
        print('%s (%s)' % (entry, package))

@CmdRegister.cmdline('edit')
def cmd_edit():
    "prints directory path to package"
    parser_kw = create_parser_parameters()
    suite = Parser(**parser_kw).create_suite()
    if suite.need_freeze():
        raise PundleException('%s file is outdated' % suite.parser.frozen_file)
    print(suite.states[sys.argv[2]].frozen_dist().location)

@CmdRegister.cmdline('info')
def cmd_info():
    "prints info about Pundle state"
    parser_kw = create_parser_parameters()
    suite = Parser(**parser_kw).create_suite()
    if suite.need_freeze():
        print('frozen.txt is outdated')
    else:
        print('frozen.txt is up to date')
    for state in suite.required_states():
        print(
            'Requirement "{}", frozen {}, {}'.format(
                state.key,
                state.frozen,
                state.requirement.line if state.requirement else 'None'
            )
        )
        print('Installed versions:')
        for dist in state.installed:
            print('    ', repr(dist))
        if not state.installed:
            print('     None')

def run_console(glob):
    import readline
    import rlcompleter
    import atexit
    import code

    history_path = os.path.expanduser("~/.python_history")

    def save_history(history_path=history_path):
        readline.write_history_file(history_path)
    if os.path.exists(history_path):
        readline.read_history_file(history_path)

    atexit.register(save_history)

    readline.set_completer(rlcompleter.Completer(glob).complete)
    readline.parse_and_bind("tab: complete")
    code.InteractiveConsole(locals=glob).interact()

@CmdRegister.cmdline('console')
def cmd_console():
    "[ipython|bpython|ptpython] starts python console with activated pundle environment"
    suite = activate()
    glob = {
        'pundle_suite': suite,
    }
    interpreter = sys.argv[2] if len(sys.argv) > 2 else None
    if not interpreter:
        run_console(glob)
    elif interpreter == 'ipython':
        from IPython import embed
        embed()
    elif interpreter == 'ptpython':
        from ptpython.repl import embed
        embed(glob, {})
    elif interpreter == 'bpython':
        from bpython import embed
        embed(glob)
    else:
        raise PundleException('Unknown interpreter: {}. Choose one of None, ipython, bpython, ptpython.')

@CmdRegister.cmdline('run')
def cmd_run():
    "executes given script"
    activate()
    import runpy
    sys.path.insert(0, '')
    script = sys.argv[2]
    sys.argv = [sys.argv[2]] + sys.argv[3:]
    runpy.run_path(script, run_name='__main__')

@CmdRegister.cmdline('module')
def cmd_module():
    "executes module like `python -m`"
    activate()
    import runpy
    sys.path.insert(0, '')
    module = sys.argv[2]
    sys.argv = [sys.argv[2]] + sys.argv[3:]
    runpy.run_module(module, run_name='__main__')

@CmdRegister.cmdline('env')
def cmd_env():
    "populates PYTHONPATH with packages paths and executes command line in subprocess"
    activate()
    aug_env = os.environ.copy()
    aug_env['PYTHONPATH'] = ':'.join(sys.path)
    subprocess.call(sys.argv[2:], env=aug_env)

ENTRY_POINT_TEMPLATE = '''#! /usr/bin/env python
import pundle; pundle.activate()
pundle.entry_points()['{entry_point}'].get_entry_info('console_scripts', '{entry_point}').load(require=False)()
'''

@CmdRegister.cmdline('linkall')
def link_all():
    "links all packages to `.pundle_local` dir"
    local_dir = '.pundle_local'
    suite = activate()

    try:
        makedirs(local_dir)
    except OSError:
        pass
    local_dir_info = {de.name: de for de in os.scandir(local_dir)}
    for r in suite.states.values():
        d = r.frozen_dist()
        if not d:
            continue
        for dir_entry in os.scandir(d.location):
            if dir_entry.name.startswith('__') or dir_entry.name.startswith('.') or dir_entry.name == 'bin':
                continue
            dest_path = os.path.join(local_dir, dir_entry.name)
            if dir_entry.name in local_dir_info:
                sym = local_dir_info.pop(dir_entry.name)
                existed = op.realpath(sym.path)
                if existed == dir_entry.path:
                    continue
                os.remove(sym.path)
            os.symlink(dir_entry.path, dest_path)
    # create entry_points binaries
    try:
        makedirs(os.path.join(local_dir, 'bin'))
    except OSError:
        pass
    for bin_name, entry_point in entry_points().items():
        bin_filename = os.path.join(local_dir, 'bin', bin_name)
        open(bin_filename, 'w').write(ENTRY_POINT_TEMPLATE.format(entry_point=bin_name))
        file_stat = os.stat(bin_filename)
        os.chmod(bin_filename, file_stat.st_mode | stat.S_IEXEC)
    local_dir_info.pop('bin')

    # remove extra links
    for de in local_dir_info:
        os.remove(de.path)

@CmdRegister.cmdline('show_requirements')
def show_requirements():
    "shows details requirements info"
    suite = activate()
    for name, state in suite.states.items():
        if state.requirement:
            print(name, 'frozen:', state.frozen, 'required:', state.requirement.req if state.requirement.req else 'VCS')

# Single mode that you can use in console
_single_mode_suite = {}  # cache variable to keep current suite for single_mode

def single_mode():
    """ Create, cache and return Suite instance for single_mode.
    """
    if not _single_mode_suite:
        py_version_path = python_version_string()
        pundledir_base = os.environ.get('PUNDLEDIR') or op.join(op.expanduser('~'), '.pundledir')
        directory = op.join(pundledir_base, py_version_path)
        _single_mode_suite['cache'] = Parser(directory=directory).create_suite()
    return _single_mode_suite['cache']

def use(key):
    """ Installs `key` requirement, like `django==1.11` or just `django`
    """
    suite = single_mode()
    suite.use(key)

if __name__ == '__main__':
    CmdRegister.main()


# encoding: utf-8
from rest_framework import serializers
from taggit_serializer.serializers import (
    TaggitSerializer, TagListSerializerField)
from . import models

class MediaTypeSerializer(serializers.ModelSerializer):

    class Meta:
        model = models.MediaType
        fields = '__all__'

class MediaFileSerializer(TaggitSerializer, serializers.ModelSerializer):

    media_type = MediaTypeSerializer(many=False, read_only=True)
    tags = TagListSerializerField()

    class Meta:
        model = models.MediaFile
        fields = '__all__'
        read_only_fields = ('owner', )

    def create(self, validated_data):
        data = validated_data.get('data', None)
        filename = validated_data.get('filename', None)
        if data and filename:
            data.name = filename
        return super(MediaFileSerializer, self).create(validated_data)

    def update(self, instance, validated_data):
        new_filename = validated_data.get('filename', instance.filename)
        new_access = validated_data.get('access', instance.access)
        renamed = instance.filename != new_filename or \
            instance.access != new_access
        result = super(MediaFileSerializer, self).update(instance, validated_data)
        if renamed:
            result.set_new_name(instance.filename)
        return result

class AlbumSerializer(serializers.ModelSerializer):

    class Meta:
        model = models.Album
        fields = '__all__'
        read_only_fields = ('owner', )

class AlbumFileSerializer(serializers.ModelSerializer):
    mediafile = MediaFileSerializer(many=False)

    class Meta:
        model = models.AlbumFile
        # fields = ['id', 'album', 'mediafile', 'order']
        fields = '__all__'

    def create(self, validated_data):
        mediafile = validated_data.pop('mediafile')
        mediafile['owner'] = validated_data.pop('owner', None)
        validated_data['mediafile'] = \
            models.MediaFile.objects.create(**mediafile)
        res = super(AlbumFileSerializer, self).create(validated_data)
        return res

    def update(self, instance, validated_data):
        result = super(AlbumFileSerializer, self).update(instance, validated_data)
        return result


import pandas as pd

class TradeHandler(object):

    def __init__(self):
        self.trade_book = {}
        self.upnl = pd.DataFrame(columns = ['cob','upnl'])
        self.rpnl = pd.DataFrame(columns = ['cob', 'rpnl'])
        self.net_position = pd.DataFrame(columns = ['cob', 'position'])
        self.latest_trade = {}
        self.current_positions = {}
        # assign portfolio

    def updateTrades(self,tick,close_flag):
        upnl = 0; rpnl = 0
        active_trade_ids, active_trades = self.getActiveTrades()
        if active_trade_ids:
            for active_trade in active_trade_ids:
                trade = active_trades[active_trade]
                if trade.trade == self.latest_trade[trade.ticker].trade: #e.g. if latest is long and this trade is long
                    if(trade.trade + close_flag != 0):
                        trade.upnl = self.pnl(tick.data[trade.ticker],trade.trade_price,trade.size)
                        trade.trade_vector = trade.trade_vector.append({'cob':tick.cob,'tick':tick.data[trade.ticker],
                                                                        'upnl':trade.upnl,'rpnl':0}
                                                                       , ignore_index = True)
                        upnl += trade.upnl
                    else: #!!!! trade.trade + close_flag == 0 if trade and close are opposite e.g. long = +1 close_flag = -1
                        trade.rpnl = self.pnl(tick.data[trade.ticker], trade.trade_price, trade.size)
                        trade.trade_vector = trade.trade_vector.append(
                            {'cob': tick.cob, 'tick': tick.data[trade.ticker],
                             'upnl': 0, 'rpnl': trade.rpnl}
                            , ignore_index=True)
                        trade.active = False  # close trade
                        trade.close_date = tick.cob  # set close date
                        rpnl += trade.rpnl
                        print(f"{tick.cob} | trade : {trade.trade_id} | {trade.trade} closed | exit signal")
                else: #e.g. if latest trade is long and this is short we would need to close out and realize pnl.
                    trade.rpnl = self.pnl(tick.data[trade.ticker], trade.trade_price, trade.size)
                    trade.trade_vector = trade.trade_vector.append({'cob':tick.cob,'tick':tick.data[trade.ticker],
                                                                    'upnl':0,'rpnl':trade.rpnl}
                                                                   , ignore_index = True)
                    trade.active = False #close trade
                    trade.close_date = tick.cob #set close date
                    rpnl += trade.rpnl
                    print(f"{tick.cob} | trade : {trade.trade_id} | {trade.trade} closed | reversal")

            self.upnl = self.upnl.append({'cob':tick.cob,'upnl':upnl}, ignore_index = True)
            self.rpnl = self.rpnl.append({'cob': tick.cob, 'rpnl': rpnl}, ignore_index=True)
            self.net_position = self.net_position.append({'cob': tick.cob, 'position': sum([v.quantity * v.trade for v in active_trades.values()])}, ignore_index=True)

    @staticmethod
    def pnl(current_val,original_val,trade_size):
        try:
            return (current_val - original_val) * trade_size
        except Exception:
            pass

    def getActiveTrades(self):
        active_trades = {} #dict of active trades agnostic of ticker as we will each by trade_id
        active_trade_ids = [] #list of active trade ids
        if len(self.trade_book.items()):
            for ticker,trades in self.trade_book.items():
                active_trade_ids_by_ticker = []
                for trade in trades:
                    if trade.active:
                        active_trade_ids.append(trade.trade_id)
                        active_trade_ids_by_ticker.append(trade.trade_id)
                        active_trades[trade.trade_id] = trade
                    active_trade_ids_by_ticker.sort(reverse = True)
                if active_trade_ids_by_ticker:
                    self.latest_trade[ticker] = active_trades[max(active_trade_ids_by_ticker)] #get max trade id for a given ticker
            return active_trade_ids, active_trades
        else:
            return None,None

    def updatePositions(self):
        pass


# -*- coding: utf-8 -*-
"""
Created on Wed Oct 24 13:26:10 2018

@author: jbk48
"""

import numpy as np
import pandas as pd
import time
import pickle
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import ExtraTreesRegressor

train_data = pd.read_csv("./data_1_north_train.csv", engine='python')
test_data = pd.read_csv("./data_1_north_test.csv", engine='python')

def make_batch(data):

    station_name = list(set(data['Next.Station.code']))

    total_batch = []

    for name in station_name:

        sub_data = data[data['Next.Station.code']==name]

        index_list = [0]

        for i in range(len(sub_data['No'])-1):
            if(abs(sub_data['No'].values[i+1] - sub_data['No'].values[i]) > 3):
                index_list.append(i+1)

        index_list.append(len(sub_data['No']))

        split_list = []

        for j in range(len(index_list)-1):
            split_list.append(sub_data.iloc[index_list[j]:index_list[j+1], [2,5,6,7,9,10]].values)

        total_batch += split_list

    mask = np.random.permutation(len(total_batch))

    return np.array(total_batch)[mask]

def make_window(total_batch, window, mode = "speed"):

    batch_X = []
    batch_Y = []
    for batch in total_batch:
        for i in range(len(batch)-window):
            batch_X.append(batch[i:i+window,:6])
            if(mode == "speed"):
                batch_Y.append(batch[i+window][0])
            elif(mode == "power"):
                batch_Y.append(batch[i+window][3])

    batch_X = np.reshape(batch_X, (-1,6*window))
    return batch_X, batch_Y

def MSE(a,b):
    return(np.mean(np.power(a-b,2)))

window = 10
path_model = "./north_speed_model_rf.pkl"

train_batch = make_batch(train_data)
X_train, Y_train = make_window(train_batch, window, "speed")

test_batch = make_batch(test_data)
X_test, Y_test = make_window(test_batch, window, "speed")

start_time = time.time()

model = ExtraTreesRegressor(n_estimators = 198, max_features = 0.5248, max_depth = 27,
                            min_samples_split = 5 , min_samples_leaf=1)
model.fit(X_train, Y_train)

duration = time.time() - start_time
minute = int(duration / 60)
second = int(duration) % 60

print("%dminutes %dseconds" % (minute,second))

y_pred = model.predict(X_test)
y_real = Y_test

print("MSE : {}".format(MSE(y_real,y_pred)))

with open(path_model, 'wb') as file:
    pickle.dump(model, file)


import re

unknown_checker = re.compile('.\?.*')

class DB:
    def __init__(self, filename):
        self.file = open(filename, 'r')
        self.datas = self.file.readlines()
        self.clear_data = []
        for value in self.datas:
            if unknown_checker.match(value):
                continue
            self.clear_data.append(value)

    def get_longest_of(self, start_char: str or list, exception_list=None):
        if exception_list is None:
            exception_list = []
        if type(start_char) == str:
            checker = [re.compile('{}.*'.format(start_char))]
        elif type(start_char) == list:
            checker = [re.compile('{}.*'.format(start_char[0])), re.compile('{}.*'.format(start_char[1]))]
        current = ''
        for value in self.clear_data:
            if value.rstrip() in exception_list:
                continue
            if checker[0].match(value):
                if current == '':
                    current = value
                else:
                    if len(value) > len(current):
                        current = value
            if len(checker) == 2:
                if checker[1].match(value):
                    if current == '':
                        current = value
                    else:
                        if len(value) > len(current):
                            current = value
        if current == '':
            return False
        return current.rstrip()


from .pages.main_page import MainPage
from .pages.login_page import LoginPage

link = ('http://selenium1py.pythonanywhere.com')

def test_can_go_to_login_page(browser):
    page = MainPage(browser, link)

    page.open()
    page.go_to_login_page()

    login_page = LoginPage(browser, browser.current_url)
    login_page.should_be_correct_url()

def test_should_see_login_link(browser):
    page = MainPage(browser, link)

    page.open()
    page.should_be_login_link()


from os import mkdir, listdir, remove, path

from master import master, Instruction

def main():
    if not directory():
        return
    try:
        script = open("input/script.txt", 'r')
    except OSError:
        print("\nSCRIPT WAS NOT FOUND.\n")
        return
    signal_time = 10
    error_detection = "crc"
    try:
        config = open("input/config.txt", 'r')
    except OSError:
        print("\nCONFIG WAS NOT FOUND.\n")
    else:
        try:
            values = lines(config)
            for line in values:
                if len(line) < 1:
                    continue
                line = [x.strip() for x in line.split('=')]
                if line[0] == "signal_time":
                    signal_time = int(line[1])
                elif line[0] == "error_detection":
                    error_detection = line[1].lower()
                else:
                    print("\nWRONG CONFIG INSTRUCTION.")
                    raise Exception
        except Exception:
            print("\nCONFIG FORMAT ERROR.\n")
            signal_time = 10
            error_detection = "crc"
    try:  # Comment to debug
        print("\nCONNECTION STARTED.")
        print("(TIME = 0)\n")
        time = master(signal_time, error_detection, translator(script))
        script.close()
        print("\nCONNECTION ENDED.")
        print("(TIME = {})\n".format(time))
    except Exception:
        print("\nBAD FORMAT ERROR.\n")

def directory():
    try:
        mkdir("output")
    except OSError:
        try:
            for file in listdir("output"):
                remove(path.join("output", file))
        except OSError:
            print("\nOUTPUT DIRECTORY CLEANING FAILED.")
            return False
    return True

def lines(script):
    return [x.replace('\n', '').split('#')[0].strip() for x in script.readlines()]

def translator(script):
    values = lines(script)
    instructions = []
    for line in values:
        if len(line) > 0:
            line = line.split(' ')
            instructions.append(Instruction(int(line[0]), line[1], line[2:len(line)]))
    return instructions

if __name__ == '__main__':
    main()


n1 = float(input("Nota 1: "))
n2 = float(input("Nota 2: "))

media = (n1 + n2) / 2

if media < 5.0:
    print("REPROVADO")
elif 7 > media >= 5.0:
#elif media >= 5.0 and media <= 6.9:
    print("RECUPERAÇÃO")
else:
    print("APROVADO")

print("A media é {} ".format(media))