"'In this model we have taken two hidden two layers and 1 i/p layer, 1 o/p layer and here we train 28 x 28 pixels images'"
import cv2 as cv           #To get my example loaded in the program
import numpy as np        #This is used for matrix
import tensorflow as tf     #This is used to type the code
import matplotlib.pyplot as plt  #For non linear functions

# mnist = tf.keras.datasets.mnist #This is used to get the training data
mnist = tf.keras.datasets.mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data() #It splits the data into training data & test data
x_train = tf.keras.utils.normalize(x_train, axis=1)     #This is to normalize the o/p i.e to bring total output to 1
x_test = tf.keras.utils.normalize(x_test, axis=1)
model = tf.keras.models.Sequential()                  #Sequential model is loaded
model.add(tf.keras.layers.Flatten(input_shape=(28, 28)))    #i/p layer
model.add(tf.keras.layers.Dense(units = 128, activation = tf.nn.relu))  #hidden layer1
model.add(tf.keras.layers.Dense(units = 128, activation = tf.nn.relu))  #hidden layer2
model.add(tf.keras.layers.Dense(units=10, activation = tf.nn.softmax))  #o/p layer
model.compile(optimizer= 'adam', loss= 'sparse_categorical_crossentropy', metrics = ['accuracy'])   #here my loss and accuracy is calcutlated
model.fit(x_train, y_train, epochs = 10)                                                             #here we tell to train the network 3 times on the same data
loss, accuracy = model.evaluate(x_test, y_test)                                                     #here our model is trainded i.e accuracy and loss is calculated
print(accuracy*100)
print(loss*100)
model.save('digits.model')                  #here we store the data of our neural network

for x in range(1,6):
    img = cv.imread(f'{x}.png')[:,:,0]
    img =np.invert( np.array([img]))
    img = tf.keras.utils.normalize(img, axis=1)
    prediction = model.predict(img)
    print(np.argmax(prediction))
    # img = tf.keras.utils.normalize(img , axis = 1)
    # result = model.predict(img)
    # print(np.argmax(result))
    plt.imshow(img[0], cmap=plt.cm.binary)
    plt.show()


#!/usr/bin/env python
# coding: utf-8

# In[20]:

from sklearn.datasets import fetch_openml
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
import umap
get_ipython().run_line_magic('matplotlib', 'inline')
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import seaborn as sns

# In[2]:

import pandas as pd
import numpy as np

# In[3]:

import time

# In[4]:

mnist = fetch_openml('mnist_784')

# In[5]:

X = mnist.data / 255.0
y = mnist.target

# In[6]:

X.shape

# In[7]:

feat_cols = [ 'pixel'+str(i) for i in range(X.shape[1]) ]
df = pd.DataFrame(X,columns=feat_cols)
df['y'] = y
df['label'] = df['y'].apply(lambda i: str(i))
X, y = None, None

# In[8]:

pca = PCA(n_components=3)
pca_result = pca.fit_transform(df[feat_cols].values)
df['pca-one'] = pca_result[:,0]
df['pca-two'] = pca_result[:,1]
df['pca-three'] = pca_result[:,2]

# In[9]:

pca.explained_variance_ratio_

# In[10]:

rndperm = np.random.permutation(df.shape[0])

# In[11]:

plt.figure(figsize=(16,10))
sns.scatterplot(
    x="pca-one", y="pca-two",
    hue="y",
    palette=sns.color_palette("hls", 10),
    data=df.loc[rndperm,:],
    legend="full",
    alpha=0.3
).get_figure().savefig("pca_mnist.png")

# In[12]:

pca.noise_variance_

# In[13]:

pca.n_components_

# In[14]:

ax = plt.figure(figsize=(16,10)).gca(projection='3d')
ax.scatter(
    xs=df.loc[rndperm,:]["pca-one"].astype(float),
    ys=df.loc[rndperm,:]["pca-two"].astype(float),
    zs=df.loc[rndperm,:]["pca-three"].astype(float),
    c=df.loc[rndperm,:]["y"].astype(float),
    cmap='tab10'
)
ax.set_xlabel('pca-one')
ax.set_ylabel('pca-two')
ax.set_zlabel('pca-three')
plt.show()

# In[18]:

ax.get_figure().savefig("pca_3d_mnist.png")

# In[26]:

N = 10000
df_subset = df.loc[rndperm[:N],:].copy()
data_subset = df_subset[feat_cols].values

# In[27]:

time_start = time.time()
tsne = TSNE(n_components=2, verbose=1, perplexity=40, n_iter=300)
tsne_results = tsne.fit_transform(data_subset)
print('t-SNE done! Time elapsed: {} seconds'.format(time.time()-time_start))

# In[38]:

df_subset['tsne-2d-one'] = tsne_results[:,0]
df_subset['tsne-2d-two'] = tsne_results[:,1]
plt.figure(figsize=(16,10))
sns_plot = sns.scatterplot(
    x="tsne-2d-one", y="tsne-2d-two",
    hue="y",
    palette=sns.color_palette("hls", 10),
    data=df_subset,
    legend="full",
    alpha=0.3
)

# In[42]:

sns_plot.get_figure().savefig("tsne_mnist.png")

# In[21]:

reducer = umap.UMAP(random_state=42)
embedding = reducer.fit_transform(mnist.data)

fig, ax = plt.subplots(figsize=(12, 10))
color = mnist.target.astype(int)
plt.scatter(
    embedding[:, 0], embedding[:, 1], c=color, cmap="Spectral", s=0.1
)
plt.setp(ax, xticks=[], yticks=[])
plt.title("MNIST data embedded into two dimensions by UMAP", fontsize=18)

plt.show()

# In[23]:

fig.savefig("umap_mnist.png")

# In[ ]:


from typing import List

class ListNode:
    def __init__(self, val=0, next=None):
        self.val = val
        self.next = next

class LinkList:
    def __init__(self, head: ListNode):
        self.head = head

class Heap:
    def __init__(self, lists: List[LinkList]):
        self.lists = lists
        self.size = len(lists)
        self.makeHeap()

    def makeHeap(self):
        if (self.size > 1):
            for i in range(self.size // 2 - 1, -1, -1):
                self.heapifySubTree(i)

    def heapifySubTree(self, i: int):
        left = self.left(i)
        right = left + 1
        if (left < self.size):
            if (right >= self.size or self.prior(left, right)):
                if (self.prior(left, i)):
                    self.swap(i, left)
                    self.heapifySubTree(left)
            elif (self.prior(right, i)):
                self.swap(i, right)
                self.heapifySubTree(right)

    def pop(self):
        self.size -= 1
        if (self.size > 0):
            self.lists[0] = self.lists[self.size]
            self.heapifySubTree(0)

    def top(self):
        return self.lists[0]

    def count(self):
        return self.size

    def left(self, i: int):
        return i + i + 1

    def right(self, i: int):
        return i + i + 2

    def prior(self, i: int, j: int) -> bool:
        return self.lists[i].head.val < self.lists[j].head.val

    def swap(self, i: int, j: int):
        t = self.lists[i]
        self.lists[i] = self.lists[j]
        self.lists[j] = t

class Solution:
    def mergeKLists(self, lists: List[ListNode]) -> ListNode:
        lists2 = [ LinkList(l) for l in lists if l != None ]

        if (len(lists2) == 0):
            return None

        result = []
        heap = Heap(lists2)
        while (heap.count() > 0):
            topList = heap.top()
            result.append(topList.head.val)

            topList.head = topList.head.next
            if (topList.head == None):
                heap.pop()
            else:
                heap.heapifySubTree(0)

        return result

if __name__ == "__main__":
    s = Solution()

    lists = [
        ListNode(1),
        ListNode(0)
    ]

    result = s.mergeKLists(lists)
    print(result)


import h5py
import numpy as np
import pandas as pd
from scipy.sparse.linalg import svds
from scipy.sparse import coo_matrix
from sklearn import linear_model
import os
import gc
#import ggplot as gg

def als(rating_matrix, prepare_path, prepare_name, transpose_prepare_name, init_user_matrix=None, init_item_matrix=None,\
        factor_num=200, method='l2', lambda_user=0.001, lambda_item=0.001, \
        iteration_num=100, user_loop_num=500, item_loop_num=500, rmse_loop_num=3, latent_file_path = "",
        latentFilePlace = "C:\\Users\\22560\\PycharmProjects\\lastFM\\hetrec2011-lastfm-2k\\latentFactor.hdf5",
        validationFilePlace = "C:\\Users\\22560\\PycharmProjects\\lastFM\\hetrec2011-lastfm-2k\\trainAfterReg.csv"
        ):
    # 如果p或者q为空值的话，分解原始矩阵作为初始值
    if((init_user_matrix==None)|(init_item_matrix==None)):
        init_user_matrix, init_item_matrix = decomposed_rating_matrix(rating_matrix, factor_num)
    with h5py.File(latent_file_path+'latentFactor.hdf5', 'w') as latent_factor_file:
        latent_factor_file.create_dataset('userLatentFactor', data=init_user_matrix)
        latent_factor_file.create_dataset('itemLatentFactor', data=init_item_matrix)
    with h5py.File(prepare_path, 'r') as prepare_y, \
            h5py.File(latent_file_path+'latentFactor.hdf5', 'r+') as latent_factor_file:
        y = prepare_y[prepare_name]
        transpose_y = prepare_y[transpose_prepare_name]
        user_latent_factor_matrix = latent_factor_file['userLatentFactor']
        item_latent_factor_matrix = latent_factor_file['itemLatentFactor']
        user_num, item_num = y.shape
        user_loop_interval = np.linspace(0, user_num, user_loop_num+1, dtype=np.int)
        item_loop_interval = np.linspace(0, item_num, item_loop_num+1, dtype=np.int)
        est_rmse = [5]
        est_rmse_on_vali = [5]
        # if(method=='l2'):
        #     for iter in range(iteration_num):
        #         for i, j in enumerate(user_loop_interval[:-1]):
        #             next_node = user_loop_interval[i+1]
        #             user_latent_factor_matrix[j:next_node, ] = ridge_reg(item_latent_factor_matrix[:], y[j:next_node, ].transpose(), lambdas=lambda_user)
        #         for i, j in enumerate(item_loop_interval[:-1]):
        #             next_node = item_loop_interval[i+1]
        #             item_latent_factor_matrix[j:next_node, ] = ridge_reg(user_latent_factor_matrix[:], y[:, j:next_node], lambdas=lambda_item)
        #         new_rmse = get_RMSE(rating_matrix, user_latent_factor_matrix[:], item_latent_factor_matrix[:], rmse_loop_num)
        #         est_rmse.append(new_rmse)
        #         print('this is the %s th iteration and the change of RMSE is %s:' %(iter, np.abs(new_rmse-est_rmse[iter])))
        if (method == 'l2'):
            for iter in range(iteration_num):
                for i, j in enumerate(user_loop_interval[:-1]):
                    next_node = user_loop_interval[i + 1]
                    user_latent_factor_matrix[j:next_node, ] = ridge_reg(item_latent_factor_matrix[:],
                                                                         y[j:next_node, ].transpose(),
                                                                         lambdas=lambda_user)
                for i, j in enumerate(item_loop_interval[:-1]):
                    next_node = item_loop_interval[i + 1]
                    item_latent_factor_matrix[j:next_node, ] = ridge_reg(user_latent_factor_matrix[:],
                                                                         transpose_y[j:next_node, ].transpose(),
                                                                         lambdas=lambda_item)
                # calculate the RMSE
                new_rmse = get_RMSE(rating_matrix, user_latent_factor_matrix[:], item_latent_factor_matrix[:],
                                    rmse_loop_num)
                est_rmse.append(new_rmse)
                print('this is the %s th iteration ,the RMSE is: %s and the change of RMSE is: %s' % (
                iter, new_rmse, (new_rmse - est_rmse[iter])/est_rmse[iter]))

                new_rmse_on_vali = validation(latentFilePlace,validationFilePlace )
                est_rmse_on_vali.append(new_rmse_on_vali)
                print('this is the %s th iteration ,the RMSE_on_validataion is: %s and the change of RMSE_on_vali is: %s' % (
                    iter, new_rmse_on_vali, (new_rmse_on_vali - est_rmse_on_vali[iter]) / est_rmse_on_vali[iter]))

        elif(method=='l1'):
            for iter in range(iteration_num):
                for i, j in enumerate(user_loop_interval[:-1]):
                    next_node = user_loop_interval[i + 1]
                    user_latent_factor_matrix[j:next_node, ] = lasso_reg(item_latent_factor_matrix[:],
                                                                         y[j:next_node, ].transpose(),
                                                                         lambdas=lambda_user)
                for i, j in enumerate(item_loop_interval[:-1]):
                    next_node = item_loop_interval[i + 1]
                    item_latent_factor_matrix[j:next_node, ] = lasso_reg(user_latent_factor_matrix[:],
                                                                         y[:, j:next_node], lambdas=lambda_item)
                # calculate the RMSE
                new_rmse = get_RMSE(rating_matrix, user_latent_factor_matrix[:], item_latent_factor_matrix[:],
                                    rmse_loop_num)
                est_rmse.append(new_rmse)
                print('this is the %s th iteration ,the RMSE is: %s and the change of RMSE is: %s' % (
                    iter, new_rmse, (new_rmse - est_rmse[iter]) / est_rmse[iter]))

                new_rmse_on_vali = validation(latentFilePlace, validationFilePlace)
                est_rmse_on_vali.append(new_rmse_on_vali)
                print(
                    'this is the %s th iteration ,the RMSE_on_validataion is: %s and the change of RMSE_on_vali is: %s' % (
                        iter, new_rmse_on_vali, (new_rmse_on_vali - est_rmse_on_vali[iter]) / est_rmse_on_vali[iter]))

        else:
            print('no this method')
            print (method)
            raise Exception('error occur !!')

        return est_rmse,est_rmse_on_vali

def decomposed_rating_matrix(sparse_matrix, k):
    u, s, vt = svds(sparse_matrix, k)
    return u.dot(np.diag(s)), vt.transpose()

def ridge_reg(X , y, lambdas):
    reg = linear_model.Ridge(alpha=lambdas, fit_intercept=False)
    reg.fit(X, y)
    return reg.coef_

def lasso_reg(X, y, lambdas):
    reg = linear_model.Lasso(alpha=lambdas, fit_intercept=False)
    reg.fit(X, y)
    return reg.coef_

#supposed p and q are small enough to fit the memory
def get_RMSE(rating_matrix, user_matrix, item_matrix, loop_num):
    count = len(rating_matrix.data)
    #m, n = rating_matrix.shape
    result = 0
    interval = np.linspace(0, count, num=loop_num+1, dtype=np.int)
    for i, j in enumerate(interval[:-1]):
        next_node = interval[i+1]
        part_row_index = rating_matrix.row[j:next_node]
        part_col_index = rating_matrix.col[j:next_node]
        result+= calcu_RMSE(rating_matrix.data[j:next_node], np.sum(np.multiply(user_matrix[part_row_index, ], item_matrix[part_col_index, ]), axis=1))
    return np.sqrt(result/count)

def calcu_RMSE(observed_rating, estimated_rating):
    return np.sum(np.square(observed_rating-estimated_rating))

# def graph(y):
#     data = pd.DataFrame({'iteration':list(range(len(y))), 'RMSE':y})
#     p = gg.ggplot(gg.aes(x='iteration', y='RMSE'), data=data)+gg.geom_point()+gg.geom_line()
#     return p

def validation(latentFilePlace,validationFilePlace ,
               userID = 'userID',itemID = 'artistID' ,targetID = 'weight'):
    # USE FOR TEST :userID = 'userID', itemID = 'artistID', targetID = 'weight'
    with h5py.File(latentFilePlace,'r') as h5f:
        #h5f = h5py.File(latentFilePlace)
        item = h5f['itemLatentFactor'].value
        user = h5f['userLatentFactor'].value
        evaluate = user.dot(item.transpose())
        validationset = pd.read_csv(validationFilePlace)
        lenOfValidation = validationset.shape[0]
        userlist  = validationset[userID]
        itemlist  = validationset[itemID]
        target = validationset[targetID]
        target_hat = evaluate[userlist,itemlist]
        eps =np.sqrt( ((target_hat - target)**2).sum() / lenOfValidation)
        return(eps)

def trainAndValidation(trainFilePlace, prepare_path,
                prepare_name, transpose_prepare_name, factor_num, method, iteration_num,
                 user_loop_num,item_loop_num, lambda_user, lambda_item,
                 latentFilePlace,validationFilePlace):

    trainAfterDealing = pd.read_csv(trainFilePlace)
    row = trainAfterDealing['userID'].get_values()
    col = trainAfterDealing['artistID'].get_values()
    data = trainAfterDealing['weight'].get_values()

    rating_matrix = coo_matrix((data, (row, col)), dtype=np.float)

    del  row, col, data
    gc.collect()

    rmse,rmse_val = als(rating_matrix=rating_matrix,  prepare_path= prepare_path,
                        prepare_name=prepare_name, transpose_prepare_name=transpose_prepare_name,
                        factor_num=factor_num, method=method, iteration_num=iteration_num,
                        user_loop_num=user_loop_num,item_loop_num=item_loop_num,
                        lambda_user=lambda_user, lambda_item=lambda_item,
                    latentFilePlace=latentFilePlace,validationFilePlace=validationFilePlace
                 )
    return (rmse,rmse_val)

if __name__=='__main__':
   pass


from __future__ import annotations

from typing import TYPE_CHECKING

from psycopg2.errors import IntegrityError

from data_access.dao import ContractsDAO, ProjectsDAO
from data_access.dto import ProjectsDTO
from errors import ContractAlreadyExistError, IncorrectIdError, ValidationError
from validators import validate_active_contract_id, validate_entered_id

if TYPE_CHECKING:
    from data_access.interfaces import DBGatewayProtocol

class ProjectLogic:
    """Contains methods for displaying project data."""

    def __init__(self, db_gateway: DBGatewayProtocol) -> None:
        self._db_gateway = db_gateway
        self._dao = ProjectsDAO(db_gateway=self._db_gateway)
        self._contracts_dao = ContractsDAO(db_gateway=self._db_gateway)

    def get_all_data(self) -> list[ProjectsDTO]:
        """Gets all projects information from the database."""

        data: list[ProjectsDTO] = self._dao.get_all_projects_info()
        return data

    def get_active_contracts_ids(self) -> list[tuple[int]]:
        """Gets list ids of active contracts"""
        contract_dao = ContractsDAO(db_gateway=self._db_gateway)
        active_contracts_list = contract_dao.get_active_contracts_ids()
        return active_contracts_list

    def create_record(self, entered_name: str) -> None:
        """Creates new record in database with entered data."""

        project_dto = ProjectsDTO(name=entered_name)
        try:
            self._dao.create_record(data=project_dto)
        except IntegrityError:
            raise

    def get_record_by_id(self, project_id: str) -> ProjectsDTO:
        """Gets project by entered id from database."""

        try:
            validate_entered_id(entered_id=project_id)
        except ValidationError:
            raise
        else:
            project_id_int = int(project_id)
        project_info = self._dao.get_info_by_id(project_id_int)
        if not project_info:
            raise IncorrectIdError("[ERROR]: There isn't a contract with entered ID in the database.")
        else:
            return project_info

    def add_contract_to_project(self, project_id: str, contract_id: str) -> None:
        """Adds contract to project."""

        try:
            validate_active_contract_id(
                entered_id=contract_id,
                contracts_ids_list=self.get_active_contracts_ids(),
            )
            project_info = self.get_record_by_id(project_id=project_id)
        except ValidationError:
            raise
        except IncorrectIdError:
            raise
        else:
            try:
                if project_info.contract_id:
                    raise ContractAlreadyExistError(
                        f"Project {project_id} already has an active contract! "
                        f"Please select another contract to add."
                    )
                else:
                    int_contract_id = int(contract_id)
                    int_project_id = int(project_id)
                    project_info.contract_id = int_contract_id
                    self._dao.update_record(project_info)
                    contract_info = self._contracts_dao.get_contract_info_by_id(contract_id=int_contract_id)
                    if contract_info:
                        contract_info.project_id = int_project_id
                        self._contracts_dao.update_record(data=contract_info)
                    else:
                        raise IncorrectIdError("[ERROR]: There isn't a contract with entered ID in the database.")
            except IntegrityError:
                raise

    def remove_active_contract_from_project(self, contract_id: str) -> None:
        """Removes active contract_id."""

        try:
            validate_entered_id(entered_id=contract_id)
        except ValidationError:
            raise
        int_contract_id = int(contract_id)
        data = self._dao.get_project_info_by_active_contract(contract_id=int_contract_id)
        if not data:
            raise IncorrectIdError(f"Project with entered active_contract_id {contract_id} does not exist.")
        else:
            data.contract_id = None
            self._dao.update_record(data=data)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import math
import solfege.interval
from .interval import DiatonicInterval, ChromaticInterval,SolfegeInterval, Alteration, TooBigAlteration#, _Interval
from util import *
from .note import DiatonicNote,ChromaticNote,Note,_Note

class IntervalWithNoRole(MyException):
    """Raised when the difference between base note and an interval has no role. I.e. is 1 or 2."""
    pass

class _NoteWithBase(_Note):
    """
    Classes for notes with base. They are a note, with an additional information representing the tonic.
    """
    def __init__(self,toCopy=None,base=None, **kwargs):
        super().__init__(toCopy=toCopy,**kwargs)
        self.base=None
        if isinstance(toCopy,_NoteWithBase):
            base=toCopy.getBase()
        if base is not None:
            self.addBase(base)

    def __add__(self,other):
        sum_=super().__add__(other)
        base=self.getBase()
        if base is not None:
            sum_.addBase(base)
        return  sum_

    def __hash__(self):
        return super().__hash__()

    def __eq__(self,other):
        return super().__eq__(other) and self.getBase()==other.getBase()

    def addBase(self,base):
        assert (base is not None)
        self.base=base
        if self.__class__!=base.__class__:
            raise Exception("Adding a base of a type %s distinct from the class %s"%(base.__class__,self.__class__))

        # if self.hasNumber():
        #     role=self.__class__.role[(self.getNumber()-base.getNumber())%self.__class__.modulo]
        #     if role is None:
        #         raise IntervalWithNoRole#Exception("Adding base %s whose number is %d to note %s, (interval=%s)  the role is None, from %s[%d%%%d=%d]"%(base,base.getNumber(),self,self.interval,self.__class__.role,self.interval.getNumber(),self.__class__.modulo,self.interval.getNumber()%self.__class__.modulo))
        #     self.role= role
        #     #debug("The role of %s is %s"%(self,role))

    def getInterval(self):
        """Interval between the note and its base"""
        if "interval" not in self.dic:
            if self.base is  None or self.value is None:
                self.dic["interval"]=None
            else:
                self.dic["interval"]=self-self.base
        return self.dic["interval"]

    def getRole(self):
        """The role of this note, assuming its in a major scale"""
        if "role" not in self.dic:
            interval=self.getInterval()
            interval=interval.getNumber()%self.modulo
            role= self.role[interval]
            self.dic["role"]=role
        return self.dic["role"]

    def getBase(self):
        return self.base

class DiatonicNoteWithBase(_NoteWithBase,DiatonicNote):
    #Saved as the interval from middle C
    role=["Tonic", "supertonic","mediant", "subdominant","dominant","submediant","leading"]

class ChromaticNoteWithBase(_NoteWithBase,ChromaticNote):
    RelatedDiatonicClass=DiatonicNoteWithBase
    def getColor(self,color=True):
        if color:
            dic={"unison":"black","third": "violet","fifth": "red","interval": "green",None:None}
            return dic[self.getRole()]
        else:
            return "black"

    def getDiatonic(self):
        """Assuming no base is used"""
        if "diatonic" not in self.dic:
            if self.getNumber() is None:
                diatonic=None
            elif self.getBase() is None:
                raise Exception("Diatonic asked when the current note %s has no base"%self)
            elif self == self.getBase():
                #If we can't use the base to determine the diatonic note, we take the more likely one
                diatonic= super().getDiatonic()
                diatonic.addBase(diatonic)
            else:
                #Otherwise, we use the role to figure out which diatonic note to use
                role=self.getRole()
                diatonicNumber={"unison":0,"third": 2,"fifth":4 ,"interval": 6}[role]
                diatonicIntervalBaseOctave=DiatonicInterval(diatonic=diatonicNumber)
                octave=self.getInterval().getOctave()
                diatonicInterval=diatonicIntervalBaseOctave.addOctave(octave)
                diatonic=self.base.getDiatonic()+diatonicInterval
                diatonic.addBase(self.base.getDiatonic())
                debug("Note %s's diatonic is not base. Its interval is %s and its diatonic is %s"%(self,diatonicInterval,diatonic))
            self.dic["diatonic"]=diatonic
        return self.dic["diatonic"]

    def getNote(self):
        note=super().getNote(Class=NoteWithBase)
        base=self.getBase()
        if self is base:
            note.addBase(note)
        elif base is not None:
            note.addBase(base.getNote())
        return note

class NoteWithBase(ChromaticNoteWithBase,Note):
    IntervalClass= SolfegeInterval
    DiatonicClass=DiatonicNote
    ChromaticClass=ChromaticNote
    """A note of the scale, as an interval from middle C."""

    def getName(self,kind=None):
        diatonic = self.getDiatonic()
        try:
            alteration=self.getAlteration()
        except TooBigAlteration as tba:
            tba.addInformation("Note",self)
            raise
        diatonicName=diatonic.getName().upper()
        alterationName=alteration.getName(kind=kind)
        return "%s%s" %(diatonicName,alterationName)

    def correctAlteration(self):
        return  self.getAlteration().printable()

ChromaticNoteWithBase.RelatedSolfegeClass=NoteWithBase


"""
Produces Fig. 4 of Johnson & Weinberg (2019), a 1-panel plot showing the
IMF-averaged Sr yields from CCSNe as a funtion of total metallicity and
rotational velocity.
"""

import visuals # visuals.py -> matplotlib subroutines in this directory
import matplotlib.pyplot as plt
import math as m
import vice
import sys
import warnings
warnings.filterwarnings("ignore")

# The keywords accepted by VICE denoting the studies to plot yields from
_STUDIES_ = ["LC18", "CL04"]

# The rotational velocities of stars probed by a given study
_VROT_ = {
	"LC18":		[0, 150, 300],
	"CL04":		[0],
}

# The overall metallicities [M/H] probed by a given study
_MH_ = {
	"LC18":		[-3, -2, -1, 0],
	"CL04":		[-4, -2, -1, -0.37, 0.15],
}

# The colors to plot a given study in
_COLORS_ = {
	"LC18":		"blue",
	"CL04":		"crimson"
}

# The markers to use for each rotational velocity
_MARKERS_ = {
	0:		"circle",
	150:	"triangle_up",
	300:	"star"
}

# The full names of each study given the keyword
_NAMES_ = {
	"LC18":		"Limongi \& Chieffi (2018)",
	"CL04": 	"Chieffi \& Limongi (2004)"
}

def plot_all_yields(ax, element):
	"""
	Plots all yields for a given element on the axis given the studies and
	rotational velocities to determine yields from

	Parameters
	==========
	ax :: subplot
		The matplotlib axis object to plot on
	element :: str
		The element to plot the yield of
	"""
	for i in _STUDIES_:
		for j in _MH_[i]:
			for k in _VROT_[i]:
				plot_yield(ax, element, i, j, k, _MARKERS_[k], _COLORS_[i])
	plot_legend(ax)

def plot_yield(ax, element, study, MH, vrot, marker, color):
	"""
	Plots a single IMF-averaged CCSN yield of Sr on the axis

	Parameters
	==========
	ax :: subplot
		The matplotlib axis object to plot on
	element :: str
		The element to plot the yield of
	study :: str
		The study to adopt the yields from
	MH :: real number
		The value of [M/H] to calculate the yield for
	vrot :: real number
		The rotational velocity of the stars in km/s
	marker :: str
		The name of the marker to plot the point in
	color :: str
		The name of the color to plot the yield in
	"""
	y = vice.yields.ccsne.fractional(element, study = study, MoverH = MH,
		rotation = vrot)[0]
	ax.scatter(MH, y, c = visuals.colors()[color],
		marker = visuals.markers()[marker], s = 100)

def plot_functional(ax, func):
	"""
	Plots the yield as a function of metallicity on the axis

	Parameters
	==========
	ax :: subplot
		The matplotlib axis object to plot on
	func :: <function>
		The function to plot on the axis
	"""
	small, big = ax.get_xlim()
	xvals = [small + (big - small) / 1000 * i for i in range(1000)]
	ax.plot(xvals, list(map(func, xvals)), linestyle = ':',
		c = visuals.colors()["black"])

def plot_legend(ax):
	"""
	Draws the legend on the axis

	Parameters
	==========
	ax :: subplot
		The matplotlib axis object to put the legend on
	"""

	# First label the studies ...
	lines = len(_STUDIES_) * [None]
	for i in range(len(lines)):
		lines[i] = ax.plot([-1, -2], [1.e-8, 1.e-6],
			c = visuals.colors()["white"], label = _NAMES_[_STUDIES_[i]])[0]
	leg = ax.legend(loc = visuals.mpl_loc()["lower right"], ncol = 1,
		bbox_to_anchor = (0.99, 0.01), frameon = False, handlelength = 0,
		fontsize = 18)
	for i in range(len(lines)):
		lines[i].remove()
		leg.get_texts()[i].set_color(_COLORS_[_STUDIES_[i]])
	ax.add_artist(leg)

	# ... then label the rotational velocities
	points = 3 * [None]
	for i in range(len(points)):
		points[i] = ax.scatter([-1, -2], [1.e-8, 1.e-6],
			c = visuals.colors()["black"], s = 50,
			marker = visuals.markers()[_MARKERS_[[0, 150, 300][i]]],
			label = r"$v_\text{rot}$ = %g km s$^{-1}$" % ([0, 150, 300][i])
		)
	ax.legend(loc = visuals.mpl_loc()["upper left"], ncol = 1,
		bbox_to_anchor = (0.01, 0.99), frameon = False, fontsize = 18,
		handlelength = 1)
	for i in range(len(points)):
		points[i].remove()

def main():
	"""
	Produces the figure and saves it as a PDF.
	"""
	ax = visuals.subplots(1, 1)
	ax.set_xlabel(r"$\log_{10}(Z/Z_\odot)$")
	ax.set_ylabel(r"$y_\text{Sr}^\text{CC}$")
	ax.set_yscale("log")
	ax.set_xlim([-4.4, 0.6])
	ax.set_ylim([3.e-13, 3.e-6])
	plot_all_yields(ax, "sr")
	plot_functional(ax, lambda x: 3.5e-8 * 10**x)
	plot_functional(ax, lambda x: 1.e-7 * (1 - m.exp(-10**(x + 1))))
	plt.tight_layout()
	plt.savefig(sys.argv[1])
	plt.clf()

if __name__ == "__main__":
	main()


import ConfigParser
import unittest
import os
import tempfile

from moto import mock_ec2
import boto.ec2

import graphlab
from graphlab.connect.aws import _ec2 as EC2
from graphlab.connect.aws._ec2 import _ec2_factory

class Ec2Test(unittest.TestCase):
    """
    This is the beginning of tests for the EC2 class. VERY basic stuff for now, but we'll use moto and add more tests in
    the near future.
    """

    def setUp(self):
        self.access_key = os.environ.get('AWS_ACCESS_KEY_ID', None)
        self.secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY', None)

        if 'AWS_ACCESS_KEY_ID' in os.environ:
            del os.environ['AWS_ACCESS_KEY_ID']
        if 'AWS_SECRET_ACCESS_KEY' in os.environ:
            del os.environ['AWS_SECRET_ACCESS_KEY']

    def tearDown(self):
        if self.access_key: os.environ['AWS_ACCESS_KEY_ID'] = self.access_key
        if self.secret_key: os.environ['AWS_SECRET_ACCESS_KEY'] = self.secret_key

    def test_no_auth_exception(self):
        """
        Verify that no auth shows correct error message
        """

        # Arrange
        instance_type = 'not-a-real-type'
        region = 'us-west-2'

        # Act & Assert
        try:
            ec2 = EC2._Ec2GraphLabServer(instance_type, region, None, None, None, None, '', '')
        except Exception as e:
            self.assertTrue("AWS configuration not found." in e.message, e.message)
        else:
            self.fail('Expected an exception to be raised')

    def test_bad_auth(self):
        """
        Verify that bad auth shows correct error message
        """
        # Arrange
        graphlab.aws.set_credentials('foo', 'bar')

        # Act & Assert
        try:
            ec2 = EC2._Ec2GraphLabServer('not-real-type', 'us-west-2', None, None, None, None, '', '')
        except Exception as e:
            self.assertTrue('EC2 response error. Please verify your AWS credentials' in e.message, e.message)
        else:
            self.fail('Expected an exception to be raised')

    def test_bad_region(self):
        """
        Verify that bad region shows correct error message.
        """
        # Arrange
        graphlab.aws.set_credentials('foo', 'bar')

        # Act & Assert
        try:
            ec2 = EC2._Ec2GraphLabServer('not-real-type', 'not-real-region', None, None, None, None, '', '')
        except Exception as e:
            self.assertTrue('is not a valid region.' in e.message, e.message)
        else:
            self.fail('Expected an exception to be raised')

    def test_config_reading(self):
        # Test retrieving region
        REGION = 'Never Never Land'
        config = ConfigParser.ConfigParser()
        config.add_section(EC2.CONFIG_SECTION)
        config.set(EC2.CONFIG_SECTION, 'region', REGION)
        temp_file = tempfile.NamedTemporaryFile()
        config.write(temp_file)
        temp_file.flush()
        region = EC2._get_region_from_config(temp_file.name)
        self.assertEqual(region, REGION)

        # Test config with our section but not our option
        config = ConfigParser.ConfigParser()
        config.add_section(EC2.CONFIG_SECTION)
        temp_file = tempfile.NamedTemporaryFile()
        config.write(temp_file)
        temp_file.flush()
        region = EC2._get_region_from_config(temp_file.name)
        self.assertIsNone(region)

        # Test empty file
        temp_file = tempfile.NamedTemporaryFile()
        region = EC2._get_region_from_config(temp_file.name)
        self.assertIsNone(region)

        # Test non-existent file
        region = EC2._get_region_from_config('/foo/bar/noWayThereIsAFileCalledThis')
        self.assertIsNone(region)

    @mock_ec2
    def test_ec2_factory(self):
        os.environ['GRAPHLAB_TEST_AMI_ID'] = 'foo'
        os.environ['GRAPHLAB_TEST_ENGINE_URL'] = 'bar'
        os.environ['GRAPHLAB_TEST_EC2_KEYPAIR'] = 'biz'
        os.environ['AWS_ACCESS_KEY_ID'] = 'fake'
        os.environ['AWS_SECRET_ACCESS_KEY'] = 'totally fake'
        os.environ['GRAPHLAB_TEST_HASH_KEY'] = 'embarrassingly_fake'

        ret = _ec2_factory('m3.xlarge')

        self.assertIsNotNone(ret.instance)
        self.assertIsNotNone(ret.instance.private_dns_name)
        self.assertIsNotNone(ret.instance_id)
        self.assertIsNotNone(ret.public_dns_name)
        self.assertEqual(ret.region, 'us-west-2')

        self.assertEqual(ret.instance.image_id, 'foo')
        self.assertEqual(ret.instance.instance_type, 'm3.xlarge')
        self.assertEqual(ret.instance.key_name, 'biz')

        # Upgrading to moto 0.3.7 broke this test. It worked with moto 0.3.5
        # I've verified this functionality is correct in our 1.0 release.
        #ret.instance.update()   # tags were applied before the last update
        #self.assertEquals(ret.instance.tags, {u'GraphLab': u'None'})

        self.assertEquals(ret.instance.state, u'running')
        ret.stop()
        ret.instance.update()
        self.assertEquals(ret.instance.state, u'terminated')

    @mock_ec2
    def test_launching_multiple_hosts(self):
        os.environ['GRAPHLAB_TEST_AMI_ID'] = 'foo'
        os.environ['GRAPHLAB_TEST_ENGINE_URL'] = 'bar'
        os.environ['GRAPHLAB_TEST_EC2_KEYPAIR'] = 'biz'
        os.environ['AWS_ACCESS_KEY_ID'] = 'fake'
        os.environ['AWS_SECRET_ACCESS_KEY'] = 'totally fake'
        os.environ['GRAPHLAB_TEST_HASH_KEY'] = 'embarrassingly_fake'

        ret = _ec2_factory('m3.xlarge', num_hosts = 2)

        self.assertIsInstance(ret, list)
        self.assertEquals(len(ret), 2)
        self.assertIsInstance(ret[0], EC2._Ec2Instance)
        self.assertIsInstance(ret[1], EC2._Ec2Instance)


from functools import lru_cache

from google.oauth2 import service_account

from issue_tracker_bot.settings import CREDENTIALS_PATH

SCOPES = [
    "https://www.googleapis.com/auth/spreadsheets",
    "https://www.googleapis.com/auth/drive",
]

@lru_cache()
def get_credentials():
    credentials = service_account.Credentials.from_service_account_file(
        str(str(CREDENTIALS_PATH))
    )
    return credentials.with_scopes(SCOPES)


import os

import cv2
import math
import numpy as np

def preprocessing(img,num):
    print("pre processing...")
    image= img
    # gray convertion
    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
    grad_x = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=3)
    grad_y = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=3)

    abs_grad_x = cv2.convertScaleAbs(grad_x)
    abs_grad_y = cv2.convertScaleAbs(grad_y)

    # threshold
    thresh_x = cv2.threshold(abs_grad_x, 0, 255, cv2.THRESH_OTSU)[1]
    med_x = cv2.medianBlur(thresh_x, 3)
    thresh_y = cv2.threshold(abs_grad_y, 0, 255, cv2.THRESH_OTSU)[1]
    med_y = cv2.medianBlur(thresh_y, 3)
    #cv2.imwrite('x.jpg',abs_grad_x)
    #cv2.imwrite('y.jpg', abs_grad_y)

    # bluring
    kernel_size = 3
    blur_thresh_x = cv2.GaussianBlur(med_x, (kernel_size, kernel_size), 0)
    blur_thresh_y = cv2.GaussianBlur(med_y, (kernel_size, kernel_size), 0)
    #cv2.imwrite('xg.jpg', blur_thresh_x)
    #cv2.imwrite('yg.jpg', blur_thresh_y)
    # Run Hough on edge detected image
    width = image.shape[1]
    height = image.shape[0]
    rho = 1  # distance resolution in pixels of the Hough grid
    theta_vertical = np.pi  # angular resolution in radians of the Hough grid
    theta_horizonatal = np.pi / 2
    threshold_horizontal = math.floor(width * 0.2)  # minimum number of votes (intersections in Hough grid cell)
    threshold_vertical = math.floor(height * 0.2)
    min_line_length_horizontal = math.floor(width * 0.2)
    min_line_length_vertical = math.floor(height * 0.2)  # minimum number of pixels making up a line
    max_line_gap = 1  # maximum gap in pixels between connectable line segments
    line_image = np.copy(gray) * 0  # creating a blank to draw lines on

    # Vertical lines
    vertical_lines = cv2.HoughLinesP(blur_thresh_x, rho, theta_vertical, threshold_vertical, np.array([]),
                                     min_line_length_vertical, max_line_gap)

    if vertical_lines is not None:
        for line in vertical_lines:
            for x1, y1, x2, y2 in line:
                # here it's possible to add a selection of only vertical lines
                if np.abs(y1 - y2) > 0.1 * np.abs(x1 - x2):
                    cv2.line(line_image, (x1, y1), (x2, y2), 255, 5)

    # Horizontal lines
    horizontal_lines = cv2.HoughLinesP(blur_thresh_y, rho, theta_horizonatal, threshold_horizontal, np.array([]),
                                       min_line_length_horizontal, max_line_gap)

    if horizontal_lines is not None:
        for line in horizontal_lines:
            for x1, y1, x2, y2 in line:
                # here it's possible to add a selection of only horizontal lines
                if np.abs(x1 - x2) > 0.1 * np.abs(y1 - y2):
                    cv2.line(line_image, (x1, y1), (x2, y2), 255, 5)

    # threshold
    thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1]
    #thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY_INV, 61,18)

    # remove lines
    clean_thresh = cv2.subtract(thresh, line_image)
    clean_invert = cv2.bitwise_not(clean_thresh)
    #cv2.imwrite('l.jpg',line_image)
    cv2.namedWindow('detectedLines', cv2.WINDOW_NORMAL)
    cv2.imshow('detectedLines', line_image)
    cv2.namedWindow('preProcessed', cv2.WINDOW_NORMAL)
    cv2.imshow('preProcessed', clean_invert)
    cv2.waitKey(0)
    name = "C:/Users/Shashi/Documents/FYP/FYP_Final/Outputs/" + num
    cv2.imwrite(os.path.join(name,'pp.jpg'), clean_invert)
    return  clean_invert


"""
2016 Day 14
https://adventofcode.com/2016/day/14
"""

from dataclasses import dataclass
from hashlib import md5
from typing import Iterator, List
import re
import aocd  # type: ignore

RE_THREEPEATS = re.compile(r"(\w)\1{2,2}")

def first_threepeat(text: str) -> str:
    """
    Find the first 'threepeated' (i.e. repeated 3 times in a row) character in a string.
    """
    search = RE_THREEPEATS.search(text)
    if search:
        return search.group()[0]
    return ""

def md5hex(text: str) -> str:
    """
    Get the hex digest of the MD5 hash of the given string.
    """
    return md5(text.encode("utf-8")).hexdigest()

def stretched_hash(text: str) -> str:
    """
    Produce the stretched hash of a string by recursively running the MD5 hash algorithm 2017
    times.
    """
    for _ in range(2017):
        text = md5hex(text)
    return text

@dataclass
class KeyCandidate:
    """
    A candidate which could be a key (due to a triple character), but is potentially awaiting the
    examination of the following 1000 hashes.
    """

    hash: str
    char: str
    index: int
    matched: int

def find_keys(
    salt: str, quantity: int, stretched: bool = False
) -> Iterator[KeyCandidate]:
    """
    Search for an appropriate quantity of keys (i.e. exploring candidates with a triple character
    and checking for a hash in the next 1000 in the stream containing five of the character).
    """
    returned = 0
    index = 0
    candidates: List[KeyCandidate] = []
    create_hash = stretched_hash if stretched else md5hex

    while returned < quantity:
        hsh = create_hash(salt + str(index))
        candidates = [
            candidate for candidate in candidates if candidate.index >= (index - 1000)
        ]
        paired = [candidate for candidate in candidates if candidate.char * 5 in hsh]

        if paired:
            for candidate in paired:
                candidate.matched = index

        char = first_threepeat(hsh)
        if char:
            candidates.append(KeyCandidate(hsh, char, index, -1))

        if len(candidates) > 0 and candidates[0].matched > -1:
            candidate, *candidates = candidates
            yield candidate
            returned += 1

        index += 1

def main() -> None:
    """
    Calculate and output the solutions based on the real puzzle input.
    """
    data = aocd.get_data(year=2016, day=14)

    *_, solution1 = find_keys(data, 64)
    print(f"Part 1: {solution1.index}")

    *_, solution2 = find_keys(data, 64, True)
    print(f"Part 2: {solution2.index}")

if __name__ == "__main__":
    main()


# q2 Loop over your list from Part 1, and use requests to download the CSV files from.
import requests
#loop over a file
for line in open("ELECTION_ID"):
    addr=addr="http://historical.elections.virginia.gov/elections/download/{}/precincts_include:0".format(line[5:10])
    resp=requests.get(addr)
    s=resp.status_code
    print(s)
#print contents of response
    file_name = line[0:4] +".csv" #saving it as year.csv as it saves extra line of codes for me in the next question. For naming it per the question the code:file_name = "president_general_"+line[0:4] +".csv"
    with open(file_name, "w") as out:
        out.write(resp.text)


#!/usr/bin/env python3
# _*_ coding: utf-8 _*_

from collections import namedtuple
from xml.dom.minidom import Document

def readFile(filename, lines):
    with open(filename, 'r') as f:
        for line in f:
            line = line.rstrip(' \n')
            if line.startswith('//') or len(line) == 0:
                continue
            lines.append(line)

def writeXml(filename, lines, tagNames):
    # 创建doc
    doc = Document()
    # 创建根节点
    root = doc.createElement(tagNames[0])
    doc.appendChild(root)

    # 记录每层节点的最新元素
    nodes = {0: root}

    for line in lines:
        index = line.rfind(' ')
        level = (index + 1) / 4 + 1
        line = line.lstrip(' ')

        node = doc.createElement(tagNames[level])
        node.setAttribute('name', line)

        nodes[level - 1].appendChild(node)
        nodes[level] = node

    with open(filename, 'w') as f:
        f.write(doc.toprettyxml(indent='\t'))

def display(lines):
    for line in lines:
        print(line)

if __name__ == '__main__':
    lines = []
    readFile('./file/sector.txt', lines)

    tagNames = ['SectorFile', 'Sectors', 'sector', 'sector_second']
    writeXml('./file/sector.xml', lines, tagNames)


import asyncio
from enum import Enum, unique
import numpy as np
import struct
import xml.etree.ElementTree as ET
from message import Message
import random
import string

# Source: https://pynative.com/python-generate-random-string/

def randomString(stringLength=10):
    """Generate a random string of fixed length """

    letters = string.ascii_lowercase + string.ascii_uppercase + " ,;-_"
    return ''.join(random.choice(letters) for i in range(stringLength))

class WRPServer:

    DEFAULT_PORT = 8766
    SERVER_IP_ADDRESS = "127.0.0.1"

    class FakeCamera:
        def __init__(self):
            setattr(self, 'SerialNumber', randomString())
            setattr(self, 'ModelName', randomString(50))
            setattr(self, 'Width', random.randint(1, 1920))
            setattr(self, 'Height', random.randint(1, 1080))
            setattr(self, 'CameraMaxFPS', random.random() * 30)
            setattr(
                self,
                'Version',
                f"{random.randint(0, 10)}.{random.randint(0, 10)}.{random.randint(0, 10)}")
            setattr(self, 'ManufacturerInfo', randomString(100))
            setattr(self, 'VendorName', randomString(100))

    @unique
    class State(Enum):
        IDLE = 1
        CONNECTED = 2
        CAMERA_SELECTED = 3
        CONTINUOUS_GRABBING = 4

    def __init__(self, port=DEFAULT_PORT):
        self.ip_address = WRPServer.SERVER_IP_ADDRESS
        self.port = port
        self.camera_list = WRPServer.generate_random_valid_camera_list(5)
        self.state = WRPServer.State.IDLE

    async def __handle_server_exception__(self):
        async with self.server as s:
            try:
                await s.serve_forever()
            except asyncio.CancelledError:
                pass
            except BaseException as e:
                print("Catched exception from the server:", e)

    async def run(self):
        self.server = await asyncio.start_server(self.routine, self.ip_address, self.port)
        self.server_task = asyncio.create_task(
            self.__handle_server_exception__())

    async def routine(self, reader, writer):
        self.state = WRPServer.State.CONNECTED
        while(True):
            print("Server: Waiting for the message...")
            message = await WRPServer.receive_message(reader)
            message_to_send = self.handle_message(message)
            if(message_to_send is not None):
                await WRPServer.send_message(writer, message_to_send)
            else:
                print("No response to be sent was generated")

    async def stop(self):
        self.server.close()
        await self.server.wait_closed()
        del self.server
        self.server_task.cancel()
        await self.server_task
        del self.server_task

    def handle_message(self, message):
        to_send = None
        print(f"Server: Handling message {message} in state {self.state}")
        if(message.msg_type == Message.Type.GET_CAMERA_LIST):
            if(self.state == WRPServer.State.CONNECTED):
                to_send = Message.create_message(
                    Message.Type.CAMERA_LIST,
                    xml_camera_list=WRPServer.camera_list_to_xml(
                        self.camera_list))
            else:
                to_send = Message.create_message(
                    Message.Type.ERROR, error_code=Message.ErrorCode.UNEXPECTED_MESSAGE)

        elif(message.msg_type == Message.Type.OPEN_CAMERA):
            if(self.state == WRPServer.State.CONNECTED):
                serial_number = getattr(
                    message, Message.CAMERA_SERIAL_NUMBER_ATTR_NAME)
                try:
                    camera = [
                        c for c in self.camera_list if c.SerialNumber == serial_number][0]
                    self.state = WRPServer.State.CAMERA_SELECTED
                    self.camera_selected = camera
                    to_send = Message.create_message(Message.Type.OK)
                except IndexError:
                    to_send = Message.create_message(
                        Message.Type.ERROR, error_code=Message.ErrorCode.CAMERA_NOT_FOUND)
            else:
                to_send = Message.create_message(
                    Message.Type.ERROR, error_code=Message.ErrorCode.UNEXPECTED_MESSAGE)

        elif(message.msg_type == Message.Type.CLOSE_CAMERA):
            if(self.state == WRPServer.State.CAMERA_SELECTED):
                self.state = WRPServer.State.CONNECTED
                del self.camera_selected
                to_send = Message.create_message(Message.Type.OK)
            else:
                to_send = Message.create_message(
                    Message.Type.ERROR, error_code=Message.ErrorCode.UNEXPECTED_MESSAGE)

        elif(message.msg_type == Message.Type.GET_FRAME):
            if(self.state == WRPServer.State.CAMERA_SELECTED):
                print("Server: frame_dict:", self.frame_dict)
                to_send = Message.create_message(
                    Message.Type.FRAME, **self.frame_dict)
            else:
                to_send = Message.create_message(
                    Message.Type.ERROR, error_code=Message.ErrorCode.UNEXPECTED_MESSAGE)
        else:
            print(f"Unhandled message type {message.msg_type}")
        return to_send

    @staticmethod
    async def send_message(writer, message):
        print("Server: Message to send:", message)
        writer.write(message.encode())
        await writer.drain()

    @staticmethod
    async def receive_message(reader):
        message_type_value, payload_length = struct.unpack(">BI", await reader.readexactly(Message.MESSAGE_TYPE_LENGTH + Message.PAYLOAD_SIZE_LENGTH))
        print(
            f"Server: Received message type value: {message_type_value}, payload_length: {payload_length}")
        if(payload_length > 0):
            payload = await reader.readexactly(payload_length)
        else:
            payload = bytes()
        response = Message.create_message_from_buffer(
            message_type_value=message_type_value,
            payload=payload,
            payload_length=payload_length)
        print("Server: Received message:", response)
        return response

    @staticmethod
    def generate_random_valid_camera_list(number_of_cameras):
        return [WRPServer.FakeCamera() for i in range(number_of_cameras)]

    @staticmethod
    def camera_list_to_xml(camera_list):
        root = ET.Element("Cameras")
        for cam in camera_list:
            ET.SubElement(root, "Camera", **{key: str(val)
                                             for key, val in vars(cam).items()})
        return ET.tostring(root, encoding="unicode")

    @staticmethod
    def generate_random_frame(camera):
        width = getattr(camera, "width")
        height = getattr(camera, "height")
        frame_np = (np.random.random((100, 100)) * 100).astype(np.float32)
        return {
            Message.FRAME_NUMBER_ATTR_NAME: np.random.randint(0, 10),
            Message.FRAME_TIMESTAMP_ATTR_NAME: np.random.randint(0, 2**63),
            Message.FRAME_ATTR_NAME: frame_np
        }


import boto3
import json

class SqsManager:
    def __init__(self):
        self.sqs = boto3.client(
            service_name='sqs',
            region_name='eu-central-1'
        )

    def create_sqs_schedule_notifications_event(self, message: dict) -> None:
        self.sqs.send_message(
            QueueUrl='https://sqs.eu-central-1.amazonaws.com/007108578073/schedule-notifications-queue.fifo',
            MessageAttributes=message,
            MessageBody=json.dumps(message),
            MessageGroupId=(
                'CreateScheduleNotificationsEvent'
            )
        )


from datetime import datetime

from pydantic import BaseModel, Field, field_validator

class ContactPersonCreateSchema(BaseModel):
    fio: str
    phone: str = Field(max_length=10, min_length=10)

    @field_validator("fio")
    @classmethod
    def validate_fio(cls, value: str):
        try:
            surname, name, father_name = value.split()
        except IndexError:
            raise ValueError("Incorrect fio format!")
        else:
            return value

class ContactPersonSchema(ContactPersonCreateSchema):
    id: str

class ContactPersonUpdateSchema(ContactPersonSchema):
    fio: str | None = None
    phone: str | None = None

    @field_validator("phone")
    @classmethod
    def validate_phone(cls, value: str):
        if len(value) != 10:
            raise ValueError("Phone number must be length of 10!")
        return value

class WatchListSchema(BaseModel):
    id: str
    address: str
    time_start: datetime
    time_finished: datetime | None = None
    is_watched: bool
    is_finished: bool
    is_refused: bool

class WatchSchema(BaseModel):
    id: str
    address: str
    time_start: datetime
    time_finished: datetime | None = None
    is_watched: bool
    is_finished: bool
    is_refused: bool
    refuse_purpose: str | None = None
    note: str
    contact_persons: list[ContactPersonSchema] = Field(default_factory=list)

class WatchCreateSchema(BaseModel):
    address: str
    time_start: datetime
    time_finished: datetime | None = None

class WatchUpdateSchema(BaseModel):
    address: str | None = None
    time_start: datetime | None = None
    time_finished: datetime | None = None
    note: str | None = None

class WatchRefuseSchema(BaseModel):
    refuse_purpose: str

class WatchFinishSchema(BaseModel):
    finish_comment: str


# "pip install keras scikit-learn" if you have not already done so
from keras.models import Sequential
from keras.layers import Dense, Activation
from keras.datasets import boston_housing
from sklearn.metrics import mean_absolute_error

# load boston housing data. Uses 80%/20% train/test split by default.
(x_train, y_train), (x_test, y_test) = boston_housing.load_data()

# create our neural network. Will have two hidden layers of 64 units each
# and an output layer with one neuron because we are predicting a single
# scalar value. Use relu activation functions.
model = Sequential()
# add a fully-connected layer with 64 units
model.add(Dense(64, input_shape=(13,)))
# apply a relu activation function to "bend" the linear output of
# the fully connected layer
model.add(Activation('relu'))
# repeat...
model.add(Dense(64))
model.add(Activation('relu'))
model.add(Dense(1))

# use RMSProp for optimization and Mean Squared Error (MSE) as loss function.
model.compile(optimizer='rmsprop', loss='mse')
model.summary()

print('[*] training model...')
model.fit(x_train, y_train, epochs=150, verbose=2)

print('[*] predicting from test set...')
y_hat = model.predict(x_test)

for i in range(len(y_hat)):
	print ('[+] predicted: {:.1f}    real: {}     error: {:.1f}'\
		   .format(y_hat[i][0], y_test[i], abs(y_hat[i][0] - y_test[i])))

print('[+] the mean absolute error is {:.1f}'.format(mean_absolute_error(y_hat, y_test)))


# -*- coding: utf-8 -*-

import os
import re
import shutil
import typing

from PIL import Image

from preview_generator.manager import PreviewManager
from tests import test_utils

CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
CACHE_DIR = "/tmp/preview-generator-tests/cache"
IMAGE_FILE_PATH = os.path.join(CURRENT_DIR, "test_sketch.sketch")

def setup_function(function: typing.Callable) -> None:
    shutil.rmtree(CACHE_DIR, ignore_errors=True)

def test_sketch_to_jpeg() -> None:
    manager = PreviewManager(cache_folder_path=CACHE_DIR, create_folder=True)
    assert manager.has_jpeg_preview(file_path=IMAGE_FILE_PATH) is True
    path_to_file = manager.get_jpeg_preview(
        file_path=IMAGE_FILE_PATH, height=256, width=512, force=True
    )

    assert os.path.exists(path_to_file) is True
    assert os.path.getsize(path_to_file) > 0
    assert re.match(test_utils.CACHE_FILE_PATH_PATTERN__JPEG, path_to_file)

    with Image.open(path_to_file) as jpeg:
        assert jpeg.height == 256
        assert jpeg.width == 482


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
import django.core.validators

class Migration(migrations.Migration):

    dependencies = [
        ('database', '0001_initial'),
    ]

    operations = [
        migrations.AddField(
            model_name='databaseentry',
            name='related_incidents',
            field=models.ManyToManyField(to='database.DatabaseEntry', blank=True),
        ),
        migrations.AddField(
            model_name='databaseentry',
            name='reliability_score',
            field=models.IntegerField(default=1, validators=[django.core.validators.MaxValueValidator(100), django.core.validators.MinValueValidator(1)]),
        ),
    ]


import torch
import argparse
from tqdm import tqdm

import random
import time

from torch.utils.data import DataLoader
from dataset import Corpus, EncodedDataset, EncodeEvalData

from transformers import *
from detector import RobertaForTextGenClassification

from sklearn.metrics import roc_curve
from sklearn.metrics import roc_auc_score
from matplotlib import pyplot

import decimal

def float_range(start, stop, step):
  while start < stop:
    yield float(start)
    start += decimal.Decimal(step)

def load_datasets(data_dir, real_dataset, fake_dataset, tokenizer,
                  max_sequence_length, random_sequence_length):

    real_corpus = Corpus(real_dataset, data_dir=data_dir, single_file=True)

    if fake_dataset == "TWO":
        real_train, real_valid = real_corpus.train * 2, real_corpus.valid * 2
        fake_corpora = [Corpus(name, data_dir=data_dir) for name in ['grover_fake', 'gpt2_fake']]
        fake_train = sum([corpus.train for corpus in fake_corpora], [])
        fake_valid = sum([corpus.valid for corpus in fake_corpora], [])

    else:
        fake_corpus = Corpus(fake_dataset, data_dir=data_dir, single_file=True)

        real_valid = real_corpus.data
        fake_valid = fake_corpus.data

    min_sequence_length = 10 if random_sequence_length else None

    validation_dataset = EncodedDataset(real_valid, fake_valid, tokenizer, max_sequence_length, min_sequence_length)
    validation_loader = DataLoader(validation_dataset)

    return validation_loader

def direct_load_dataset(data_dir, dataset, tokenizer,
                  max_sequence_length, random_sequence_length=False):

    data_corpus = Corpus(dataset, data_dir=data_dir, single_file=True)

    data_list = data_corpus.data
    label_list = data_corpus.label

    validation_dataset = EncodedSingleDataset(data_list, label_list, tokenizer, max_sequence_length)

    validation_loader = DataLoader(validation_dataset)

    return validation_loader

class GeneratedTextDetection:
    """
    Artifact class
    """

    def __init__(self, args):
        torch.manual_seed(int(time.time()))

        self.args = args

        # Load the model from checkpoints
        self.model, self.tokenizer = self._init_detector()

    def _init_detector(self):

        model_name = 'roberta-large' if self.args.large else 'roberta-base'
        tokenization_utils.logger.setLevel('ERROR')
        tokenizer = RobertaTokenizer.from_pretrained(model_name)
        model = RobertaForTextGenClassification.from_pretrained(model_name).to(self.args.device)

        # Load the model from checkpoints
        if self.args.device == "cpu":
            model.load_state_dict(torch.load((self.args.check_point + '{}.pt').format(self.args.model_name),
                                             map_location='cpu')['model_state_dict'])
        else:
            model.load_state_dict(
                torch.load((self.args.check_point + '{}.pt').format(self.args.model_name))['model_state_dict'])

        return model, tokenizer

    def evaluate(self, input_text):
        """
           Method that runs the evaluation and generate scores and evidence
        """

        tp = 0
        tn = 0
        fp = 0
        fn = 0
        total = 0

        predict_prob = []
        y = []

        validation_loader = load_datasets(self.args.data_dir,
                                          self.args.real_dataset,
                                          self.args.fake_dataset,
                                          self.tokenizer,
                                          self.args.max_sequence_length,
                                          random_sequence_length=False)

        self.model.eval()

        with tqdm(validation_loader, desc="Eval") as loop:
            for texts, masks, labels in loop:

                total += labels.size(0)
                texts, masks, labels = texts.to(self.args.device), masks.to(self.args.device), labels.to(self.args.device)

                output_dic = self.model(texts, attention_mask=masks)
                disc_out = output_dic["logits"]

                _, predicted = torch.max(disc_out, 1)

                prob_values = [item[1] for item in disc_out.tolist()]

                predict_prob.extend(prob_values)

                y.extend(labels.tolist())

                # predict_prob.extend(predicted.tolist())

                tp += ((predicted == labels) & (labels == 1)).sum().item()
                tn += ((predicted == labels) & (labels == 0)).sum().item()
                fn += ((predicted != labels) & (labels == 1)).sum().item()
                fp += ((predicted != labels) & (labels == 0)).sum().item()

        recall = float(tp) / (tp+fn)
        precision = float(tp) / (tp+fp)
        f1_score = 2 * float(precision) * recall / (precision + recall)

        print('Accuracy of the discriminator: %d %%' % (
            100 * (tp+tn) / total))
        print('Recall of the discriminator: %d %%' % (
            100 * recall))
        print('Precision of the discriminator: %d %%' % (
            100 * precision))
        print('f1_score of the discriminator: %d %%' % (
            100 * f1_score))

        # calculate scores
        lr_auc = roc_auc_score(y, predict_prob)

        # summarize scores
        print('Classifier: ROC AUC=%.3f' % (lr_auc))

        # calculate roc curves
        lr_fpr, lr_tpr, _ = roc_curve(y, predict_prob)

        eq_fpr = list(float_range(0, 1, 1 / len(lr_fpr)))
        eq_tpr = [1 - item for item in eq_fpr]

        # plot the roc curve for the model
        pyplot.plot(lr_fpr, lr_tpr, marker='.', label='RobertaTextGen')
        pyplot.plot(eq_fpr, eq_tpr, marker='.', label='EER')
        # axis labels

        pyplot.xlabel('Probability of False Alarm')
        pyplot.ylabel('Probability of Detection')
        # show the legend
        pyplot.legend()
        # show the plot
        pyplot.show()
        pyplot.savefig('ROC.pdf')

def main():
    parser = argparse.ArgumentParser(
        description='Roberta: Discriminator'
    )

    # Input data and files
    parser.add_argument('--model_name', default="drfinetunedroberta", type=str,
                        help='name of the model')
    parser.add_argument('--check_point', default="/content/drive/Shareddrives/DARPA/Datasets/Eval1Sources/", type=str,
                        help='saved model checkpoint directory')
    parser.add_argument('--data-dir', type=str, default='/content/drive/Shareddrives/DARPA/Datasets/Eval1Sources')
    parser.add_argument('--real-dataset', type=str, default='dryrun_real_eval.valid')
    parser.add_argument('--fake-dataset', type=str, default='dryrun_fake_eval.valid')

    # Model parameters
    parser.add_argument('--device', type=str, default=None)
    parser.add_argument('--batch-size', type=int, default=1)
    parser.add_argument('--max-sequence-length', type=int, default=128)
    parser.add_argument('--large', action='store_true', help='use the roberta-large model instead of roberta-base')

    args = parser.parse_args()

    if args.device is None:
        args.device = f'cuda:{0}' if torch.cuda.is_available() else 'cpu'

    artifact = GeneratedTextDetection(args)

    sample_eval_data = ["Russian opposition politician Aleksei Navalny says he owes his life to the pilots who made an emergency landing when he collapsed on a flight last month, and to paramedics he said had quickly diagnosed poisoning and injected him with atropine.",
                        "To receive Steve Gutterman's Week In Russia each week via e-mail, subscribe by clicking here.Alyaksandr Lukashenka's rushed, hushed-up inauguration ceremony in Belarus may evoke memories for Vladimir Putin, and contain a warning about the future as Russia hurtles toward 2024. Also, a COVID-19 surge, Kremlin contortions over the poisoning of Aleksei Navalny, and a UN speech \"rehearsing the defense of a nation in decline.",
                        "As Americans anxiously watch the spread of coronavirus variants that were first identified in Britain and South Africa, scientists are finding a number of new variants that seem to have originated in the United States — and many of them may pose the same kind of extra-contagious threat."]

    artifact.evaluate(sample_eval_data)

if __name__ == "__main__":
    main()


import tensorflow as tf
from PIL import Image, ImageOps
import numpy as np
import matplotlib.pyplot as plt
from keras.datasets import mnist

model=tf.keras.models.load_model('MNIST/MNIST_Model')
(x_train,y_train),(x_test,y_test)=mnist.load_data()

image=Image.open() #'picture'
image_gray = ImageOps.grayscale(image)
image_small = image_gray.resize((28,28))

# n = np.random.randint(len(x_train))
# img = x_train[n]
# print(y_train[n])
# x_trial = x_train[y_train==2]
# y_trial = y_train[y_train ==2]

#img = x_trial[0]

img=np.asarray(image_small)
plt.imshow(img)
plt.show()
#print(img.shape)

tf.keras.utils.normalize(img)
img=img.reshape(1,28,28,1)
prediction = model.predict(img)[0]
prediction_list = prediction.tolist()
max_value = max(prediction_list)
max_index = prediction_list.index(max_value)
print("The Predicted number is {}".format(max_index))

#print(y_trial)
#print(model.weights)


import time

from django.conf import settings

from dashboard.utils import get_web3
from gas.utils import recommend_min_gas_price_to_confirm_in_time
from kudos.utils import kudos_abi
from web3 import Web3

#setup
w3 = get_web3('mainnet')
kudos_contract_address = Web3.toChecksumAddress(settings.KUDOS_CONTRACT_MAINNET)
kudos_owner_address = Web3.toChecksumAddress(settings.KUDOS_OWNER_ACCOUNT)
nonce = w3.eth.getTransactionCount(kudos_owner_address)

# pk range to re-send
min_pk = 1684
max_pk = 1740
TIME_SLEEP = 1

for kt in KudosTransfer.objects.filter(pk__gte=min_pk, pk__lte=max_pk):

    token_id = kt.kudos_token_cloned_from.token_id
    address = kt.receive_address
    price_finney = kt.kudos_token_cloned_from.price_finney

    contract = w3.eth.contract(Web3.toChecksumAddress(kudos_contract_address), abi=kudos_abi())
    tx = contract.functions.clone(address, token_id, 1).buildTransaction({
        'nonce': nonce,
        'gas': 500000,
        'gasPrice': int(recommend_min_gas_price_to_confirm_in_time(2) * 10**9),
        'value': int(price_finney / 1000.0 * 10**18),
    })

    signed = w3.eth.account.signTransaction(tx, settings.KUDOS_PRIVATE_KEY)
    time.sleep(TIME_SLEEP)

    try:
        txid = w3.eth.sendRawTransaction(signed.rawTransaction).hex()
        nonce += 1
        print(f'sent tx {nonce}')
        kt.txid = txid
        kt.save()

    except Exception as e:
        print(e)
        error = "Could not redeem your kudos.  Please try again soon."


import time
import math
import torch
import torch.nn as nn
import numpy as np
from data.utils import *
from data.datasets import STGODEDataset
from torch.utils.data import DataLoader
from trainer.base_trainer import BaseTrainer
from util.logging import *
from logger.logger import Logger

class STGODETrainer(BaseTrainer):
    def __init__(self, cls, config, args):
        self.config = config
        self.device = self.config.device
        self.cls = cls
        self.setup_save(args)
        self.logger = Logger()

    def load_dataset(self):
        if os.path.exists("{}/{}_train_{}_{}.npy".format(self.config.dataset_dir, self.config.dataset_name, self.config.train_ratio, self.config.test_ratio)) and \
            os.path.exists("{}/{}_test_{}_{}.npy".format(self.config.dataset_dir, self.config.dataset_name, self.config.train_ratio, self.config.test_ratio)) and \
                os.path.exists("{}/{}_val_{}_{}.npy".format(self.config.dataset_dir, self.config.dataset_name, self.config.train_ratio, self.config.test_ratio)) and \
                os.path.exists("{}/{}_dtw_distance.npy".format(self.config.dataset_dir, self.config.dataset_name)) and \
                os.path.exists("{}/{}_spatial_distance.npy".format(self.config.dataset_dir, self.config.dataset_name)):
            print(toGreen('Found generated dataset in '+self.config.dataset_dir))
        else:
            print(toGreen('Generating dataset...'))
            generate_data_matrix(self.config.dataset_dir, self.config.dataset_name, self.config.train_ratio,
             self.config.test_ratio, self.config.sigma1, self.config.sigma2, self.config.thres1, self.config.thres2, self.config.data_format)
        if self.config.data_format == 'h5':
            data = pd.read_hdf(os.path.join(self.config.dataset_dir, self.config.dataset_name)+".h5").values
            data = np.expand_dims(data, -1)
        elif self.config.data_format == 'npz':
            data = np.load(os.path.join(self.config.dataset_dir, self.config.dataset_name)+".npz")['data']
        self.config.num_nodes = data.shape[1]
        self.config.num_features = data.shape[2]
        datasets = {}

        for category in ['train', 'val', 'test']:
            data = np.load("{}{}_{}_{}_{}.npy".format(self.config.dataset_dir, self.config.dataset_name, category, self.config.train_ratio, self.config.test_ratio))
            #x, y = seq2instance(data, self.config.num_his, self.config.num_pred)
            '''
            if category == 'train':
                self.mean, self.std = np.mean(x), np.std(x)
            x = (x - self.mean) / self.std
            '''
            datasets[category] = data

        return datasets

    def get_matrices(self):
        # distance matrix
        dist_matrix = np.load("{}/{}_dtw_distance.npy".format(self.config.dataset_dir, self.config.dataset_name))
        mean = np.mean(dist_matrix)
        std = np.std(dist_matrix)
        dist_matrix = (dist_matrix - mean) / std
        sigma = self.config.sigma1
        dist_matrix = np.exp(-dist_matrix ** 2 / sigma ** 2)
        dtw_matrix = np.zeros_like(dist_matrix)
        dtw_matrix[dist_matrix > self.config.thres1] = 1
        # spatial matrix
        dist_matrix = np.load("{}/{}_spatial_distance.npy".format(self.config.dataset_dir, self.config.dataset_name))
        # normalization
        std = np.std(dist_matrix[dist_matrix != np.float('inf')])
        mean = np.mean(dist_matrix[dist_matrix != np.float('inf')])
        dist_matrix = (dist_matrix - mean) / std
        sigma = self.config.sigma2
        sp_matrix = np.exp(- dist_matrix**2 / sigma**2)
        sp_matrix[sp_matrix < self.config.thres2] = 0

        #print(f'average degree of spatial graph is {np.sum(sp_matrix > 0)/2/self.config.num_nodes}')
        #print(f'average degree of semantic graph is {np.sum(dtw_matrix > 0)/2/self.config.num_nodes}')

        A_sp_wave = get_normalized_adj(sp_matrix).to(self.device)
        A_se_wave = get_normalized_adj(dtw_matrix).to(self.device)
        return A_sp_wave, A_se_wave

    def setup_model(self):
        A_sp_wave, A_se_wave = self.get_matrices()
        print(self.config.num_nodes, self.config.num_features)
        self.model = self.cls(self.config, A_sp_wave, A_se_wave).to(self.device)

    def compose_dataset(self):
        datasets = self.load_dataset()
        for category in ['train', 'val', 'test']:
            datasets[category] = STGODEDataset(datasets[category], self.config.num_his, self.config.num_pred)
        self.num_train_iteration_per_epoch = math.ceil(len(datasets['train']) / self.config.batch_size)
        self.num_val_iteration_per_epoch = math.ceil(len(datasets['val']) / self.config.test_batch_size)
        self.num_test_iteration_per_epoch = math.ceil(len(datasets['test']) / self.config.test_batch_size)
        return datasets['train'], datasets['val'], datasets['test']

    def compose_loader(self):
        train_dataset, val_dataset, test_dataset = self.compose_dataset()
        print(toGreen('Dataset loaded successfully ') + '| Train [{}] Test [{}] Val [{}]'.format(toGreen(len(train_dataset)), toGreen(len(test_dataset)), toGreen(len(val_dataset))))
        self.train_loader = DataLoader(train_dataset, batch_size=self.config.batch_size, shuffle=True)
        self.val_loader = DataLoader(val_dataset, batch_size=self.config.test_batch_size, shuffle=False)
        self.test_loader = DataLoader(test_dataset, batch_size=self.config.test_batch_size, shuffle=False)

    def train_epoch(self, epoch):
        self.model.train()
        start_time = time.time()
        total_loss = 0
        total_metrics = np.zeros(len(self.metrics))

        for batch_idx, (data, target) in enumerate(self.train_loader):
            label = target[..., :self.config.output_dim]  # (..., 1)  supposed to be numpy array
            data, target = data.to(self.device), target.to(self.device)
            self.optimizer.zero_grad()

            output = self.model(data)

            output = output.cpu()
            loss = self.loss(output, label)  # loss is self-defined, need cpu input
            loss.backward()

            torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.config.max_grad_norm)
            self.optimizer.step()
            training_time = time.time() - start_time
            start_time = time.time()

            total_loss += loss.item()
            this_metrics = self._eval_metrics(output, label)
            total_metrics += this_metrics

            print_progress('TRAIN', epoch, self.config.total_epoch, batch_idx, self.num_train_iteration_per_epoch, training_time, self.config.loss, loss.item(), self.config.metrics, this_metrics)

        return total_loss, total_metrics

    def validate_epoch(self, epoch, is_test):
        self.model.eval()
        total_loss = 0
        total_metrics = np.zeros(len(self.metrics))
        start_time = time.time()

        for batch_idx, (data, target) in enumerate(self.test_loader if is_test else self.val_loader):
            label = target[..., :self.config.output_dim]
            data, target = data.to(self.device), target.to(self.device)
            self.optimizer.zero_grad()

            with torch.no_grad():
                output = self.model(data)

            output = output.cpu()
            loss = self.loss(output, label)  # loss is self-defined, need cpu input

            valid_time = time.time() - start_time
            start_time = time.time()

            total_loss += loss.item()
            this_metrics = self._eval_metrics(output, label)
            total_metrics += this_metrics

            print_progress('TEST' if is_test else 'VALID', epoch, self.config.total_epoch, batch_idx, \
                self.num_test_iteration_per_epoch if is_test else self.num_val_iteration_per_epoch, valid_time, \
                self.config.loss, loss.item(), self.config.metrics, this_metrics)

        return total_loss, total_metrics

    def _eval_metrics(self, output, target):
        acc_metrics = np.zeros(len(self.metrics))
        for i, metric in enumerate(self.metrics):
            acc_metrics[i] += metric(output, target)
            # break
        return acc_metrics


from langchain.embeddings.openai import OpenAIEmbeddings
from langchain.vectorstores.faiss import FAISS
import os,argparse
from dotenv import load_dotenv
from os import getenv
from util import get_answer

load_dotenv()
os.environ['OPENAI_API_KEY'] = os.getenv('OPENAI_API_KEY')

def main():

    """
    The main function that runs the program.

    This function takes a query from the command line arguments, uses OpenAI's GPT-3 language model to search for similar
    text in a pre-indexed dataset, and generates an answer based on the search results.

    Parameters: None
    Returns: None
    """
    # Parse command line arguments
    parser = argparse.ArgumentParser(description='Process arguments.')
    parser.add_argument('query', type=str, help='The query to search for.')
    args = parser.parse_args()
    query = args.query

    # Initialize OpenAIEmbeddings object with API key and chunk size
    embed = OpenAIEmbeddings(openai_api_key=os.getenv('OPENAI_API_KEY'), chunk_size=1000)

    # Load pre-indexed dataset using FAISS
    index = FAISS.load_local('index v2', embed)

    # Use FAISS to search for similar text in the dataset
    search = index.similarity_search(query, k=5)

    # Generate a prompt for the GPT-3 language model to use for generating an answer
    prompt = f"The text below is a group conversation in English language and some part is in pidgin english. From the text answer this question: \
    {query}: Based on the text, what are some of the mentions in the conversations, \
    make a list of the answers and you must provide links if they appear in the conversation where necessary.\
    Ignore adding source to the answer"
    answer = get_answer(search,prompt)
    print(answer['output_text'])
    return answer['output_text']

if __name__ == "__main__":
    main()


#!/usr/bin/env python
import argparse
import sys
import time
from bcc import BPF

kstime = time.time_ns() - time.monotonic_ns()

parser = argparse.ArgumentParser()
parser.add_argument("-p", "--pid", default=None)
args = parser.parse_args()

bpf_text = """
#include <uapi/linux/ptrace.h>
#include <net/sock.h>
#include <bcc/proto.h>

struct event {
        int pad;
        int size;
        u64 timestamp_ns;
};
BPF_RINGBUF_OUTPUT(events, 1 << 12);

int kretprobe__tcp_sendmsg(struct pt_regs *ctx)
{
    u64 id = bpf_get_current_pid_tgid();
    if (id >> 32 != __PID__) { return 0; }
    int size = PT_REGS_RC(ctx);
    u64 time = bpf_ktime_get_ns();
    struct event event = {
        .size = size,
        .timestamp_ns = time
    };
    events.ringbuf_output(&event, sizeof(event), 0);
    return 0;
}
"""

if args.pid is None:
    print("PID must set")
    exit(0)

bpf_text = bpf_text.replace("__PID__", args.pid)

bpf = BPF(text=bpf_text)

def callback(ctx, data, size):
    event = bpf['events'].event(data)
    print("%d,%d" % (event.timestamp_ns + kstime, event.size))

bpf['events'].open_ring_buffer(callback)

while True:
    try:
        bpf.ring_buffer_consume()
    except KeyboardInterrupt:
        exit()
    sys.stdout.flush()


import tempfile
import shutil

from nose.plugins.attrib import attr
from w3af.plugins.attack.payloads.payloads.tests.payload_test_helper import PayloadTestHelper
from w3af.plugins.attack.payloads.payload_handler import exec_payload

class test_get_source_code(PayloadTestHelper):

    EXPECTED_RESULT = {"https://moth/w3af/audit/local_file_read/local_file_read.php":
                       (
                       '/var/www/moth/w3af/audit/local_file_read/local_file_read.php',
                       'tmp__random__/var/www/moth/w3af/audit/local_file_read/local_file_read.php')
                       }

    CONTENT = "echo file_get_contents( $_REQUEST['file'] );"

    @attr('ci_fails')
    def test_get_source_code(self):
        temp_dir = tempfile.mkdtemp()
        result = exec_payload(self.shell, 'get_source_code', args=(temp_dir,),
                              use_api=True)

        self.assertEqual(len(self.EXPECTED_RESULT.keys()), 1)

        expected_url = self.EXPECTED_RESULT.keys()[0]
        downloaded_url = result.items()[0][0].url_string
        self.assertEquals(expected_url, downloaded_url)

        downloaded_file_path = result.items()[0][1][1]
        downloaded_file_content = file(downloaded_file_path).read()
        self.assertTrue(self.CONTENT in downloaded_file_content)

        shutil.rmtree(temp_dir)


import logging
from typing import Any, Dict, Tuple

import numpy as np
from scipy.sparse import csr_matrix

import cvxpy.settings as s
from cvxpy import Zero
from cvxpy.constraints import NonNeg
from cvxpy.reductions.dcp2cone.cone_matrix_stuffing import ParamConeProg
from cvxpy.reductions.solution import Solution, failure_solution
from cvxpy.reductions.solvers import utilities
from cvxpy.reductions.solvers.conic_solvers.conic_solver import ConicSolver
from cvxpy.utilities.versioning import Version

log = logging.getLogger(__name__)

class PDLP(ConicSolver):
    """An interface to PDLP via OR-Tools."""

    SUPPORTED_CONSTRAINTS = ConicSolver.SUPPORTED_CONSTRAINTS

    # The key that maps to the pdlp.QuadraticProgram in the data returned by
    # apply().
    PDLP_MODEL = "pdlp_model"

    def name(self) -> str:
        """The name of the solver."""
        return 'PDLP'

    def import_solver(self) -> None:
        """Imports the solver."""
        import ortools  # noqa F401
        if Version(ortools.__version__) < Version('9.7.0'):
            raise RuntimeError(f'Version of ortools ({ortools.__version__}) '
                               f'is too old. Expected >= 9.7.0.')
        if Version(ortools.__version__) >= Version('9.8.0'):
            raise RuntimeError('Unrecognized new version of ortools '
                               f'({ortools.__version__}). Expected < 9.8.0. '
                               'Please open a feature request on cvxpy to '
                               'enable support for this version.')

    def apply(self, problem: ParamConeProg) -> Tuple[Dict, Dict]:
        """Returns a new problem and data for inverting the new solution."""
        from ortools.pdlp.python import pdlp

        # Create data and inv_data objects
        data = {}
        inv_data = {self.VAR_ID: problem.x.id}
        if not problem.formatted:
            problem = self.format_constraints(problem, None)
        data[s.PARAM_PROB] = problem
        data[self.DIMS] = problem.cone_dims

        constr_map = problem.constr_map
        inv_data["constraints"] = constr_map[Zero] + constr_map[NonNeg]

        # Min c'x + d such that Ax + b = s, s \in cones.
        c, d, A, b = problem.apply_parameters()
        A = csr_matrix(A)
        data["num_constraints"], data["num_vars"] = A.shape

        model = pdlp.QuadraticProgram()
        model.objective_offset = d.item() if isinstance(d, np.ndarray) else d
        model.objective_vector = c
        model.variable_lower_bounds = np.full_like(c, -np.inf)
        model.variable_upper_bounds = np.full_like(c, np.inf)

        model.constraint_matrix = A
        constraint_lower_bounds = np.full_like(b, -np.inf)
        constraint_upper_bounds = np.full_like(b, np.inf)
        # Ax + b = 0
        num_eq = problem.cone_dims.zero
        constraint_lower_bounds[:num_eq] = -b[:num_eq]
        constraint_upper_bounds[:num_eq] = -b[:num_eq]
        # Ax + b >= 0
        constraint_lower_bounds[num_eq:] = -b[num_eq:]

        model.constraint_lower_bounds = constraint_lower_bounds
        model.constraint_upper_bounds = constraint_upper_bounds

        data[self.PDLP_MODEL] = model
        return data, inv_data

    def invert(self, solution: Dict[str, Any],
               inverse_data: Dict[str, Any]) -> Solution:
        """Returns the solution to the original problem."""
        status = solution["status"]

        if status in s.SOLUTION_PRESENT:
            primal_vars = {
                inverse_data[self.VAR_ID]: solution["primal"]
            }
            dual_vars = utilities.get_dual_values(
                result_vec=solution["dual"],
                parse_func=utilities.extract_dual_value,
                constraints=inverse_data["constraints"],
            )
            return Solution(status, solution["value"], primal_vars, dual_vars,
                            {})
        else:
            return failure_solution(status)

    def solve_via_data(
            self,
            data: Dict[str, Any],
            warm_start: bool,
            verbose: bool,
            solver_opts: Dict[str, Any],
            solver_cache: Dict = None,
    ) -> Solution:
        """Returns the result of the call to the solver."""
        from ortools.pdlp import solvers_pb2
        from ortools.pdlp.python import pdlp

        parameters = solvers_pb2.PrimalDualHybridGradientParams()
        parameters.verbosity_level = 3 if verbose else 0
        # CVXPY reductions can leave a messy problem (e.g., no variable
        # bounds), so we turn on presolving by default.
        parameters.presolve_options.use_glop = True
        if "parameters_proto" in solver_opts:
            proto = solver_opts["parameters_proto"]
            if not isinstance(proto, solvers_pb2.PrimalDualHybridGradientParams):
                log.error("parameters_proto must be a PrimalDualHybridGradientParams")
                return {"status": s.SOLVER_ERROR}
            parameters.MergeFrom(proto)
        if "time_limit_sec" in solver_opts:
            limit = float(solver_opts["time_limit_sec"])
            parameters.termination_criteria.time_sec_limit = limit

        result = pdlp.primal_dual_hybrid_gradient(data[self.PDLP_MODEL],
                                                  parameters)
        solution = {}
        solution["primal"] = result.primal_solution
        solution["dual"] = result.dual_solution
        solution["status"] = self._status_map(result.solve_log)

        convergence_info = self._get_convergence_info(
            result.solve_log.solution_stats,
            result.solve_log.solution_type
        )
        if convergence_info is not None:
            solution["value"] = convergence_info.primal_objective
        else:
            solution["value"] = -np.inf

        return solution

    @staticmethod
    def _get_convergence_info(stats, candidate_type):
        for convergence_info in stats.convergence_information:
            if convergence_info.candidate_type == candidate_type:
                return convergence_info
        return None

    def _status_map(self, solve_log):
        from ortools.pdlp import solve_log_pb2
        TerminationReason = solve_log_pb2.TerminationReason
        status = solve_log.termination_reason
        if status == TerminationReason.TERMINATION_REASON_OPTIMAL:
            return s.OPTIMAL
        elif status == TerminationReason.TERMINATION_REASON_PRIMAL_INFEASIBLE:
            return s.INFEASIBLE
        elif status == TerminationReason.TERMINATION_REASON_DUAL_INFEASIBLE:
            # Not technically correct, but this seems to be the convention.
            return s.UNBOUNDED
        elif status == TerminationReason.TERMINATION_REASON_TIME_LIMIT:
            return s.USER_LIMIT
        elif status == TerminationReason.TERMINATION_REASON_ITERATION_LIMIT:
            return s.USER_LIMIT
        elif status == TerminationReason.TERMINATION_REASON_KKT_MATRIX_PASS_LIMIT:
            return s.USER_LIMIT
        elif status == TerminationReason.TERMINATION_REASON_NUMERICAL_ERROR:
            log.warning('PDLP reported a numerical error.')
            return s.USER_LIMIT
        elif status == TerminationReason.TERMINATION_REASON_INVALID_PROBLEM:
            log.error('Invalid problem: %s', solve_log.termination_string)
            return s.SOLVER_ERROR
        elif status == TerminationReason.TERMINATION_REASON_INVALID_PARAMETER:
            log.error('Invalid parameter: %s', solve_log.termination_string)
            return s.SOLVER_ERROR
        elif status == TerminationReason.TERMINATION_REASON_PRIMAL_OR_DUAL_INFEASIBLE:
            return s.INFEASIBLE_OR_UNBOUNDED
        else:
            log.error("Unexpected status: %s Message: %s",
                      TerminationReason.Name(status),
                      solve_log.termination_string)
            return s.SOLVER_ERROR


import logging
import shutil
import time
from typing import Optional

import requests
from colorlog import ColoredFormatter
from urllib3.exceptions import ReadTimeoutError

LOGGER = logging.getLogger("MAIN")
# formatter = ColoredFormatter("  %(log_color)s%(levelname)s:%(reset)s %(message)s")
# stream = logging.StreamHandler()
# stream.setFormatter(formatter)
# LOGGER.addHandler(stream)
LOGGER.setLevel(logging.INFO)

def download_file(url: str, local_filename: str, timeout: Optional[int] = 5) -> int:
    read_counter = 0
    break_read_error_loop = False

    while (not break_read_error_loop) and (read_counter < 5):
        count = 0
        break_connection_error_loop = False
        while (not break_connection_error_loop) and (count < 5):
            try:
                resp = requests.get(url, stream=True, timeout=timeout)
                status_code = resp.status_code
            except requests.exceptions.ReadTimeout:
                LOGGER.warn("ReadTimeout from cmip6_utils.file.download_file")
                status_code = 408
            except requests.exceptions.ConnectTimeout:
                LOGGER.warn("ConnectTimeout from cmip6_utils.file.download_file")
                status_code = 408
            except requests.exceptions.ConnectionError:
                LOGGER.warn("ConnectionError from cmip6_utils.file.download_file")
                status_code = 491
            except Exception:
                LOGGER.warn("Unknown error from cmip6_utils.file.download_file")
                status_code = 499

            if status_code != 200:
                count += 1
                LOGGER.warn(f"Failed opening URL.... Retrying {count}")
                time.sleep(timeout)
            else:
                break_connection_error_loop = True

        if count == 5:
            break_read_error_loop = True

        if status_code == 200:
            try:
                with open(local_filename, "wb") as f:
                    shutil.copyfileobj(resp.raw, f)
                break_read_error_loop = True
            except ReadTimeoutError:
                read_counter += 1
                LOGGER.warn(f"Reading from stream failed.... Retrying {read_counter}")
                status_code = -1
                time.sleep(timeout)

    return status_code


# -*- coding: utf-8 -*-
from __future__ import unicode_literals, absolute_import

import codecs
import io
import logging
import re

from contextlib import contextmanager

from flask import send_file
from gridfs import GridFS
from pymongo import MongoClient

from . import BaseBackend

log = logging.getLogger(__name__)

class GridFsBackend(BaseBackend):
    '''
    A Mongo GridFS backend

    Expect the following settings:

    - `mongo_url`: The Mongo access URL
    - `mongo_db`: The database to store the file in.
    '''
    def __init__(self, name, config):
        super(GridFsBackend, self).__init__(name, config)

        self.client = MongoClient(config.mongo_url)
        self.db = self.client[config.mongo_db]
        self.fs = GridFS(self.db, self.name)

    def exists(self, filename):
        return self.fs.exists(filename=filename)

    @contextmanager
    def open(self, filename, mode='r', encoding='utf8'):
        if 'r' in mode:
            f = self.fs.get_last_version(filename)
            yield f if 'b' in mode else codecs.getreader(encoding)(f)
        else:  # mode == 'w'
            f = io.BytesIO() if 'b' in mode else io.StringIO()
            yield f
            params = {'filename': filename}
            if 'b' not in mode:
                params['encoding'] = encoding
            self.fs.put(f.getvalue(), **params)

    def read(self, filename):
        f = self.fs.get_last_version(filename)
        return f.read()

    def write(self, filename, content):
        kwargs = {
            'filename': filename
        }

        if hasattr(content, 'content_type') and content.content_type is not None:
            kwargs['content_type'] = content.content_type

        return self.fs.put(self.as_binary(content), **kwargs)

    def delete(self, filename):
        regex = '^{0}'.format(re.escape(filename))
        for version in self.fs.find({'filename': {'$regex': regex}}):
            self.fs.delete(version._id)

    def copy(self, filename, target):
        src = self.fs.get_last_version(filename)
        self.fs.put(src, filename=target, content_type=src.content_type)

    def list_files(self):
        for f in self.fs.list():
            yield f

    def serve(self, filename):
        file = self.fs.get_last_version(filename)
        return send_file(file, mimetype=file.content_type)

    def get_metadata(self, filename):
        f = self.fs.get_last_version(filename)
        return {
            'checksum': 'md5:{0}'.format(f.md5),
            'size': f.length,
            'mime': f.content_type,
            'modified': f.upload_date,
        }


import unittest
import random

from src.modelo.Competidor import Competidor
from src.modelo.Carrera import Carrera
from src.logica.Logica_mock import Logica_mock
from src.modelo.declarative_base import Session
from faker import Faker

class CompetidorTestCase(unittest.TestCase):

        def setUp(self):

                self.logica = Logica_mock()
                self.session = Session()
                self.data_factory = Faker()
                Faker.seed(1000)

                self.data = []
                self.datacar = []
                self.competidores = []
                self.carreras = []

                for j in range(0,4):
                        self.datacar.append((self.data_factory.name(),True))
                        self.carreras.append(Carrera(nombre = self.datacar[-1][0],abierta =True))
                        self.session.add(self.carreras[-1])
                self.session.add(self.carreras[-1])

                for i in range(0,10):
                        self.data.append((self.data_factory.unique.name(),
                        self.data_factory.pyfloat(0,1,positive=True),
                        self.data_factory.random_int(1,3)))
                        self.competidores.append(
                                Competidor(
                                        nombre = self.data[-1][0],probabilidad=self.data[-1][1],cuota=self.data[-1][1]/(1-self.data[-1][1]),ganador=False,id_carrera=self.data[-1][2]))
                        self.session.add(self.competidores[-1])

                self.session.commit()

        def test_constructor(self):
                for competidores, dato in zip(self.competidores, self.data):
                        self.assertEqual(competidores.nombre, dato[0])
                        self.assertEqual(competidores.probabilidad, dato[1])
                        self.assertEqual(competidores.id_carrera, dato[2])

        def tearDown(self):
                self.session = Session()
                busqueda = self.session.query(Competidor).all()
                busquedaCar = self.session.query(Carrera).all()
                for competidor in busqueda:
                        self.session.delete(competidor)
                for carrera in busquedaCar:
                        self.session.delete(carrera)
                self.session.commit()
                self.session.close()

        def test_aniadir_competidor(self):
                self.data.append((self.data_factory.unique.name(), self.data_factory.pyfloat(0,1,positive=True), self.data_factory.random_int(1,3)))
                carrera = self.session.query(Carrera).filter(Carrera.id == self.data[-1][2]).first()
                self.assertEqual(self.logica.aniadir_competidor(self.data[-1][2],self.data[-1][0],self.data[-1][1],carrera.nombre),True)

        def test_aniadir_competidor_repetida(self):
                self.data.append((self.data_factory.unique.name(), self.data_factory.pyfloat(0,1,positive=True), self.data_factory.random_int(1,3)))
                carrera = self.session.query(Carrera).filter(Carrera.id == self.data[-1][2]).first()
                self.logica.aniadir_competidor(self.data[-1][2],self.data[-1][0],self.data[-1][1],carrera.nombre)
                resultado=self.logica.aniadir_competidor(self.data[-1][2],self.data[-1][0],self.data[-1][1],carrera.nombre)
                self.assertNotEqual(resultado, True)

        def test_verificar_almacenamiento_aniadir_competidor(self):
                self.data.append((self.data_factory.unique.name(), self.data_factory.pyfloat(0,1,positive=True), self.data_factory.random_int(1,3)))
                carrera = self.session.query(Carrera).filter(Carrera.id == self.data[-1][2]).first()
                self.logica.aniadir_competidor(self.data[-1][2],self.data[-1][0],self.data[-1][1],carrera.nombre)
                self.session = Session()
                competidor = self.session.query(Competidor).filter(Competidor.nombre == self.data[-1][0]).first()
                self.assertEqual(competidor.nombre, self.data[-1][0])
                self.assertEqual(competidor.probabilidad, self.data[-1][1])


from django.shortcuts import render
from django.http import JsonResponse
from django.conf import settings
from . import utils
import json

# Create your views here.
def workflows(request):
    raw_list = utils.read_file()
    df_list = utils.create_df_list(raw_list)
    iteration_list = ['Iteration-' + str(i) for i in range(1, len(df_list)+1)]
    trust_iteration_list = df_list[0].TrustID.tolist()
    context = {'iteration_list': iteration_list, 'trust_iteration_list': trust_iteration_list}
    settings.result_df = df_list

    return render(request, 'workflows/workflows.html', context=context)

def iteration_chart(request):

    iteration_index = request.GET.get('iteration_index', None)
    iter_index = str(iteration_index).split('-')[1]
    df = settings.result_df[int(iter_index)-1]
    x = utils.make_chart(df)
    chart = x.to_json(indent=None)
    data = {'iteration_index': iteration_index, 'chart': chart}

    return JsonResponse(data)

def trust_chart(request):

    trust_id = request.GET.get('trust_id', None)
    trust_df = utils.trust_frame(settings.result_df, trust_id)
    x = utils.trust_chart(trust_df, trust_id)
    trust_chart = x.to_json(indent=None)
    data = {'trust_id': trust_id, 'trust_chart': trust_chart}
    # print(trust_id)

    return JsonResponse(data)

def examples(request):
    raw_list = utils.read_file()
    df_list = utils.create_df_list(raw_list)
    iteration_list = ['Iteration-' + str(i) for i in range(1, len(df_list)+1)]
    trust_iteration_list = df_list[0].TrustID.tolist()
    df = df_list[0]
    x = utils.make_chart(df)
    chart = x.to_json(indent=None)
    # print(chart)
    # spec = x.to_dict()
    context = {
        'iteration_list': iteration_list,
        'trust_iteration_list': trust_iteration_list,
        'chart': chart
    }

    return render(request, 'workflows/example.html', context=context)