from web3 import Web3
import brownie
from brownie import Contract as bContract
from brownie import accounts as bAccount
from utils.utility import get_w3, get_contract
from eth_account import Account
import numpy as np
from utils.snx_contract import SnxContracts
from utils.prices import Prices
import time
from web3.providers.base import JSONBaseProvider
base_provider = JSONBaseProvider()

class BrownieInteractions(SnxContracts,Prices):

    def __init__(self,conf):

        self.w3 = get_w3(conf)
        SnxContracts.__init__(self,conf)
        Prices.__init__(self,conf)
        self.conf=conf
        self.brownie_init()
        self.setup_contracts()

    def setup_contracts(self):

        self.contracts = {}

        wethContract = get_contract(self.conf,'0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2')
        self.contracts['weth'] = bContract.from_abi(name='weth',address=wethContract.address,abi=wethContract.abi)

        contract = get_contract(self.conf,'0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48')
        self.contracts['usdc'] = bContract.from_abi(name='usdc',address=contract.address,abi=wethContract.abi)

        contract = get_contract(self.conf,'0x68b3465833fb72A70ecDF485E0e4C7bD8665Fc45')
        self.contracts['uni'] = bContract.from_abi(name='uni',address=contract.address,abi=contract.abi)

        #setup synthetix contract
        proxyContract = self.get_snx_contract(contractNameAddress="ProxyERC20")
        snxContract   = self.get_snx_contract(contractNameAddress="Synthetix")
        self.contracts['snx'] = bContract.from_abi(name='snx', address=proxyContract.address, abi=snxContract.abi)

        #setup susd contract
        contract = self.get_snx_contract(contractNameAddress="ProxyERC20sUSD",contractNameAbi='ProxyERC20')
        self.contracts['susd'] = bContract.from_abi(name='susd', address=contract.address, abi=contract.abi)

        #setup seth contract
        contract = self.get_snx_contract(contractNameAddress="ProxysETH",contractNameAbi='ProxyERC20')
        self.contracts['seth'] = bContract.from_abi(name='seth', address=contract.address, abi=contract.abi)

        #setup sEUR contract
        contract = self.get_snx_contract(contractNameAddress="ProxysEUR",contractNameAbi='ProxyERC20')
        self.contracts['seur'] = bContract.from_abi(name='seur', address=contract.address, abi=contract.abi)

        #setup direct integration contract
        contract = self.get_snx_contract('DirectIntegrationManager')
        self.contracts['integration'] = bContract.from_abi(name='di', address=contract.address, abi=contract.abi)

        #setup collateral eth contract
        contract = self.get_snx_contract('CollateralEth')
        self.contracts['collateral_eth'] = bContract.from_abi(name='collateral_eth', address=contract.address, abi=contract.abi)

        #setup collateral eth contract
        contract = self.get_snx_contract('ExchangeRates')
        self.contracts['exchange_rates'] = bContract.from_abi(name='exchange_rates', address=contract.address, abi=contract.abi)

        #setup collateral eth contract
        contract = self.get_snx_contract('SystemSettings')
        self.contracts['settings'] = bContract.from_abi(name='settings', address=contract.address, abi=contract.abi)

        #dex aggregartors
        contract = get_contract(self.conf,'0xf120F029Ac143633d1942e48aE2Dfa2036C5786c')
        self.contracts['dex_agg'] = bContract.from_abi(name='dex_1',address=contract.address,abi=contract.abi)

        contract = get_contract(self.conf,self.conf["contracts"]["dexPriceAggregator"])
        self.contracts['dex_agg_mock'] = bContract.from_abi(name='dex_1',address=contract.address,abi=contract.abi)

        #setup mock aggergator
        address  = self.conf["contracts"]["mockAggregator"]
        contract = get_contract(conf=self.conf,address=address)
        self.contracts["mock"] = bContract.from_abi(name='mock', address=contract.address, abi=contract.abi)

        #Setup CircuitBreaker
        contract = self.get_snx_contract("CircuitBreaker")
        self.contracts['breaker'] = bContract.from_abi(name='breaker', address=contract.address, abi=contract.abi)

    def approve(self,approver,approvee,amount,contractName):
        return self.contracts[contractName].approve(approvee,amount,{"from":approver,'max_fee':int(1e9),'priority_fee':int(1e9)})

    def balance_of(self,contractName,account):
        contract = self.contracts[contractName]
        return contract.balanceOf(account.address)

    def set_integration(self,
                        targetAddress,
                        currencyKey,
                        dexPriceAggregator='0x0000000000000000000000000000000000000000',
                        atomicEquivalentForDexPricing='0x0000000000000000000000000000000000000000',
                        atomicExchangeFeeRate=0,
                        atomicTwapWindow=0,
                        atomicMaxTwapDelta=0,
                        atomicMaxVolumePerBlock=0,
                        atomicVolatilityConsiderationWindow=0,
                        atomicVolatilityTwapSeconds=0,
                        atomicVolatilityUpdateThreshold=0,
                        exchangeFeeRate=0,
                        exchangeMaxDynamicFee=0,
                        exchangeDynamicFeeRounds=0,
                        exchangeDynamicFeeThreshold=0,
                        exchangeDynamicFeeWeightDecay=0):
        currencyKeyHex = self.w3.toHex(text=currencyKey)
        currencyKeyHex = self.w3.toHex(text=currencyKey).ljust(66,"0")
        return self.contracts["integration"].setExchangeParameters(targetAddress,
                                                                   [currencyKeyHex],
                                                                   [currencyKeyHex,
                                                                    dexPriceAggregator,
                                                                    atomicEquivalentForDexPricing,
                                                                    atomicExchangeFeeRate,
                                                                    atomicTwapWindow,
                                                                    atomicMaxTwapDelta,
                                                                    atomicMaxVolumePerBlock,
                                                                    atomicVolatilityConsiderationWindow,
                                                                    atomicVolatilityTwapSeconds,
                                                                    atomicVolatilityUpdateThreshold,
                                                                    exchangeFeeRate,
                                                                    exchangeMaxDynamicFee,
                                                                    exchangeDynamicFeeRounds,
                                                                    exchangeDynamicFeeThreshold,
                                                                    exchangeDynamicFeeWeightDecay],
                                                                   {'from' : bAccount[-1],
                                                                    'max_fee':int(1e9),
                                                                    'priority_fee':int(1e9)})

    def get_integration(self,diAddress,currencyKey):
        headers = ['currencyKey',
                   'dexPriceAggregator',
                   'atomicEquivalentForDexPricing',
                   'atomicExchangeFeeRate',
                   'atomicTwapWindow',
                   'atomicMaxTwapDelta',
                   'atomicMaxVolumePerBlock',
                   'atomicVolatilityConsiderationWindow',
                   'atomicVolatilityTwapSeconds',
                   'atomicVolatilityUpdateThreshold',
                   'exchangeFeeRate',
                   'exchangeMaxDynamicFee',
                   'exchangeDynamicFeeRounds',
                   'exchangeDynamicFeeThreshold',
                   'exchangeDynamicFeeWeightDecay']
        currencyKeyHex = self.w3.toHex(text=currencyKey).ljust(66,"0")
        output = self.contracts["integration"].getExchangeParameters(diAddress,currencyKeyHex)
        outputDict = {header: parameter for header, parameter in zip(headers,output)}
        outputDict["currencyKey"] = self.w3.toText(outputDict["currencyKey"]).replace("\x00",'')
        return outputDict

    def set_exchange_fee(self,currencyKey,fee):
        currencyKeyHex = self.w3.toHex(text=currencyKey).ljust(66,"0")
        return self.contracts["settings"].setExchangeFeeRateForSynths([currencyKeyHex],[fee],{'from' : bAccount[-1],'max_fee':int(1e9),'priority_fee':int(1e9)})

    def set_atomic_exchange_fee(self,currencyKey,fee):
        currencyKeyHex = self.w3.toHex(text=currencyKey).ljust(66,"0")
        return self.contracts["settings"].setAtomicExchangeFeeRate(currencyKeyHex,fee,{'from' : bAccount[-1],'max_fee':int(1e9),'priority_fee':int(1e9)})

    def set_atomic_max_volume_per_block(self,maxVolumePerBlock):
        return self.contracts["settings"].setAtomicMaxVolumePerBlock(maxVolumePerBlock,{'from' : bAccount[-1],'max_fee':int(1e9),'priority_fee':int(1e9)})

    def set_atomic_volatility_threshold(self,currencyKey,volatilityThreshold):
        currencyKeyHex = self.w3.toHex(text=currencyKey).ljust(66,"0")
        return self.contracts["settings"].setAtomicVolatilityUpdateThreshold(currencyKeyHex ,volatilityThreshold,{'from' : bAccount[-1],'max_fee':int(1e9),'priority_fee':int(1e9)})

    def set_atomic_consideration_window(self,currencyKey,window):
        currencyKeyHex = self.w3.toHex(text=currencyKey).ljust(66,"0")
        return self.contracts["settings"].setAtomicVolatilityConsiderationWindow(currencyKeyHex,window,{'from' : bAccount[-1],'max_fee':int(1e9),'priority_fee':int(1e9)})

    def set_dynamic_parameters(self,rounds=None,threshold=None,decay=None,maxFee=None):
        if not rounds is None:
            self.contracts["settings"].setExchangeDynamicFeeRounds(rounds,{'from' : bAccount[-1],'max_fee':int(1e9),'priority_fee':int(1e9)})
        if not threshold is None:
            self.contracts["settings"].setExchangeDynamicFeeThreshold(threshold,{'from' : bAccount[-1],'max_fee':int(1e9),'priority_fee':int(1e9)})
        if not decay is None:
            self.contracts["settings"].setExchangeDynamicFeeWeightDecay(decay,{'from' : bAccount[-1],'max_fee':int(1e9),'priority_fee':int(1e9)})
        if not maxFee is None:
            self.contracts["settings"].setExchangeMaxDynamicFee(maxFee,{'from' : bAccount[-1],'max_fee':int(1e9),'priority_fee':int(1e9)})

    def swap_uni(self,account,fromToken,toToken,fromAmount,fee):
        #create some random account
        randomAccount = Account.create()
        #send random some eth
        bAccount[0].transfer(randomAccount.address,'1 ether',gas_price='1 gwei')

        #send the tokens to the random account
        self.contracts[fromToken].transfer(randomAccount.address,fromAmount,{'from':account,'max_fee':int(1e9),'priority_fee':int(1e9)})

        #have the suspense account approve uni to spend from tokens
        contract = self.w3.eth.contract(address = self.contracts[fromToken].address,
                                        abi = self.contracts[fromToken].abi)
        txPrep = contract.functions.approve(self.contracts["uni"].address,int(fromAmount))
        tx = txPrep.buildTransaction({'chainId': 1,
                                      'gas': int(1e6),
                                      'maxFeePerGas': int(1e9),
                                      'maxPriorityFeePerGas': int(1e9),
                                      'type': 2 ,
                                      'nonce': self.w3.eth.getTransactionCount(randomAccount.address)})
        signedTx = self.w3.eth.account.sign_transaction(tx, randomAccount.key)
        txHash = self.w3.eth.sendRawTransaction(signedTx.rawTransaction)
        time.sleep(10)
        txReceipt = self.w3.eth.getTransactionReceipt(txHash)
        assert txReceipt["status"] == 1,"approve failed"

        #swap the tokens on uniswap
        contract = self.w3.eth.contract(address = self.contracts["uni"].address,
                                        abi = self.contracts["uni"].abi)
        txPrep = contract.functions.exactInputSingle({'tokenIn':self.contracts[fromToken].address,
                                                      'tokenOut':self.contracts[toToken].address,
                                                      'fee':fee,
                                                      'recipient':randomAccount.address,
                                                      'deadline':int(time.time()+500),
                                                      'amountIn':int(fromAmount),
                                                      'amountOutMinimum':0,
                                                      'sqrtPriceLimitX96':0})
        tx = txPrep.buildTransaction({'chainId': 1,
                                      'gas': int(1e6),
                                      'maxFeePerGas': int(1e9),
                                      'maxPriorityFeePerGas': int(1e9),
                                      'type': 2 ,
                                      'nonce': self.w3.eth.getTransactionCount(randomAccount.address)})
        signedTx = self.w3.eth.account.sign_transaction(tx, randomAccount.key)
        txHash = self.w3.eth.sendRawTransaction(signedTx.rawTransaction)
        time.sleep(10)
        txReceipt = self.w3.eth.getTransactionReceipt(txHash)
        assert txReceipt["status"] == 1,"swap failed"

        #send the tokens to account
        contract = self.w3.eth.contract(address = self.contracts[toToken].address,
                                        abi = self.contracts[toToken].abi)

        txPrep = contract.functions.transfer(account.address,contract.functions.balanceOf(randomAccount.address).call())
        tx = txPrep.buildTransaction({'chainId': 1,
                                      'gas': int(1e6),
                                      'maxFeePerGas': int(1e9),
                                      'maxPriorityFeePerGas': int(1e9),
                                      'type': 2 ,
                                      'nonce': self.w3.eth.getTransactionCount(randomAccount.address)})
        signedTx = self.w3.eth.account.sign_transaction(tx, randomAccount.key)
        txHash = self.w3.eth.sendRawTransaction(signedTx.rawTransaction)
        time.sleep(10)
        txReceipt = self.w3.eth.getTransactionReceipt(txHash)
        assert txReceipt["status"] == 1,"sending tokens back to account failed"

    def eth_to_weth(self,account,amount):
        return self.contracts["weth"].deposit({'value':amount,
                                               'from':account,
                                               'max_fee':int(1e9),
                                               'priority_fee':int(1e9)})

    def eth_to_synth(self,account,ethAmount,currencyKey):
        currencyKeyHex = self.w3.toHex(text=currencyKey).ljust(66,"0")
        if currencyKey=='sUSD':
            atomicRate, linkRate = self.get_atomic_link_price('sETH','sUSD')
            amountOut = int(ethAmount*linkRate/1.31)
        elif currencyKey == 'sETH':
            amountOut = int(ethAmount/1.31)
        return self.contracts["collateral_eth"].open(\
                                                     amountOut,\
                                                         currencyKeyHex,\
                                                             {'value':ethAmount,
                                                              'from':account,
                                                              'max_fee':int(1e9),
                                                              'priority_fee':int(1e9)})

    def swap_atomically(self,fromCurrencyKey,toCurrencyKey,fromAmount,account):
        fromAmountPre = self.balance_of(fromCurrencyKey.lower(), account)
        toAmountPre   = self.balance_of(toCurrencyKey.lower(), account)
        atomicRate, linkRate  = self.get_atomic_link_price(fromCurrencyKey=fromCurrencyKey, toCurrencyKey=toCurrencyKey)
        fromHex   = Web3.toHex(text=fromCurrencyKey).ljust(66,"0")
        toHex     = Web3.toHex(text=toCurrencyKey).ljust(66,"0")
        self.contracts["snx"].exchangeAtomically(fromHex,
                                                 fromAmount,
                                                 toHex,
                                                 '0x0000000000000000000000000000000000000000',
                                                 0,
                                                 {'from':account,
                                                  'max_fee':int(1e9),
                                                  'priority_fee':int(1e9)})
        fromAmountPost = self.balance_of(fromCurrencyKey.lower(), account)
        toAmountPost   = self.balance_of(toCurrencyKey.lower(), account)
        return  1 - (toAmountPost-toAmountPre)/((fromAmountPre-fromAmountPost)*atomicRate)

    def get_atomic_swap_signed_tx(self,fromCurrencyKey,toCurrencyKey,fromAmount,account,incrementNonce=0):
        fromHex   = Web3.toHex(text=fromCurrencyKey).ljust(66,"0")
        toHex     = Web3.toHex(text=toCurrencyKey).ljust(66,"0")
        snxContract   = self.get_snx_contract(contractNameAddress="Synthetix")
        txPrep = snxContract.functions.exchangeAtomically(fromHex,
                                                          fromAmount,
                                                          toHex,
                                                          '0x0000000000000000000000000000000000000000',
                                                          0)

        tx = txPrep.buildTransaction({'chainId': 1,
                                      'gas': int(1e6),
                                      'maxFeePerGas': int(1e9),
                                      'maxPriorityFeePerGas': int(1e9),
                                      'type': 2 ,
                                      'nonce': self.w3.eth.getTransactionCount(account.address)+incrementNonce})

        tx = self.w3.eth.account.sign_transaction(tx, account.key)

        return base_provider.encode_rpc_request('eth_sendRawTransaction', [tx.rawTransaction.hex()])

    def swap_classic(self,fromCurrencyKey,toCurrencyKey,fromAmount,account):
        fromAmountPre = self.balance_of(fromCurrencyKey.lower(), account)
        toAmountPre   = self.balance_of(toCurrencyKey.lower(), account)
        atomicRate, linkRate = self.get_atomic_link_price(fromCurrencyKey=fromCurrencyKey, toCurrencyKey=toCurrencyKey)
        fromHex   = Web3.toHex(text=fromCurrencyKey).ljust(66,"0")
        toHex     = Web3.toHex(text=toCurrencyKey).ljust(66,"0")
        self.contracts["snx"].exchangeWithTracking(fromHex,
                                                   fromAmount,
                                                   toHex,
                                                   '0x0000000000000000000000000000000000000000',
                                                   '0x0000000000000000000000000000000000000000',
                                                   {'from':account,
                                                    'max_fee':int(1e9),
                                                    'priority_fee':int(1e9)})
        fromAmountPost = self.balance_of(fromCurrencyKey.lower(), account)
        toAmountPost   = self.balance_of(toCurrencyKey.lower(), account)
        return  1 - (toAmountPost-toAmountPre)/((fromAmountPre-fromAmountPost)*linkRate)

    def find_nearest_to_atomic(self,atomic,twap,spot,cl):
        rateNames = np.array(['twap','spot','cl'])
        deltas = np.array([abs(atomic/rate-1) for rate in [twap,spot,cl]])
        return rateNames[np.min(deltas) == deltas].item()

    def is_close(self,firstNumber,secondNumber,dof=10):
        return abs(firstNumber/secondNumber-1)<1/10**dof

    def set_oracle_to_mock(self,currencyKey):
        currencyKeyHex = self.w3.toHex(text=currencyKey).ljust(66,"0")
        return self.contracts["exchange_rates"].addAggregator(currencyKeyHex,self.contracts["mock"].address,{'from':bAccount[-1],'gas_price':'1 gwei'})

    def set_oracle_to_chainlink(self,currencyKey,address):
        currencyKeyHex = self.w3.toHex(text=currencyKey).ljust(66,"0")
        return self.contracts["exchange_rates"].addAggregator(currencyKeyHex,address,{'from':bAccount[-1],'gas_price':'1 gwei'})

    def set_mock_price(self,newPrice,timestamp):
        return self.contracts["mock"].setLatestAnswer(newPrice,timestamp,{'from':bAccount[0],'gas_price':'1 gwei'})

    def reset_circuit_breaker(self,address):
        return self.contracts["breaker"].resetLastValue([address],[0],{'from':bAccount[-1],'gas_price':'1 gwei'})

    def brownie_revert(self):
        brownie.chain.revert()

    def brownie_init(self):

        self.w3 = get_w3(conf=self.conf)
        self.specialAccounts = []

        if brownie.network.is_connected():
            self.disconnect_brownie()

        #connect
        brownie.network.connect('267-fork')

        #setup few random accounts and fund them
        for x in range(4):
            bAccount.add()
            bAccount[0].transfer(bAccount[x+1],"100 ether",gas_price="1 gwei")
            self.specialAccounts.append(Account.create())

        for account in self.specialAccounts:
            bAccount[0].transfer(account.address,"100 ether",gas_price="1 gwei")

        #unlock the owner
        contract = self.get_snx_contract(contractNameAddress='Issuer')
        ownerAddress = contract.functions.owner().call()
        bAccount.at(ownerAddress,force=True)
        brownie.chain.snapshot()

    def disconnect_brownie(self):
        brownie.network.disconnect()


from django.urls import path
from . import views

urlpatterns = [
    path('', views.index, name='index'),
    path('ranking/', views.ranking, name='ranking'),
    path('perfis/', views.perfis, name='perfis'),
    path('perfil_usuario/', views.perfil_usuario, name='perfil_usuario'),
    path('projeto/', views.projeto, name='projeto'),
    path('registro/', views.registro, name='registro'),
    path('login/', views.login, name='login'),
    path('mapa/', views.mapa, name='mapa'),
    path('chat/', views.chat, name='chat'),
    path('analises/', views.analises, name='analises'),
]


# %% [markdown]
"""
# Multivariate Distributions

Key concepts for this section:
1. Correlation / covariance
2. Contingency Table
3. Plotting joint/conditional/marginal distributions
4. Simpson's paradox, confounder?
"""

# %% tags=['hide-cell']
from IPython import get_ipython
if get_ipython() is not None:
    get_ipython().run_line_magic('load_ext', 'autoreload')
    get_ipython().run_line_magic('autoreload', '2')

# %%
import seaborn as sns; sns.set()
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from scipy import stats
sns.set(rc={'figure.figsize':(9,4)})

# %%
# Fake Data:
n=100
df = pd.DataFrame({
    "a":np.random.normal(10, 1, n),
})
df["c"] = df["a"] + np.random.normal(1, 0.5, n)

# %% [markdown]
"""
## Correlation / Covariance

**Covariance** indicates the level to which two variables vary together.

$$Cov(X,Y) = \frac{1}{n}\sum_{i=1}^n(x_i - E(X))(y_i-E(Y))$$
"""

# %%
# Compute covariance manually:
products = [
    (row["a"]-df["a"].mean()) * (row["c"]-df["c"].mean())
    for i,row in df.iterrows()
]
covariance = np.sum(products)/len(df)
print(covariance)

# %%
# Using numpy:
# if bias = False, computes "sample covariance" so denominator is N-1
np.cov(df["a"], df["c"], bias=True)

# %% [markdown]
"""
**Pearson correlation coefficient** is the covariance divided by product of standard deviations

$$\rho_{X,Y} = \frac{cov(X,Y)}{\sigma_X\sigma_Y}$$

also see: [spearman's rank correlation coefficient](https://en.wikipedia.org/wiki/Spearman%27s_rank_correlation_coefficient)
"""

# %%
corr = covariance / (np.std(df["a"]) * np.std(df["c"]))
corr

# %%
np.corrcoef(df["a"], df["c"])

# %% [markdown]
"""
## Contingency Table
"""

# %%
df_cat = pd.DataFrame({
    "colors":np.random.choice(["red","blue","green","yellow","orange"], size=30, replace=True),
    "names":np.random.choice(["jason","jorge","lisa","paul"], size=30, replace=True),
    "states":np.random.choice(["california","arizona","oregon"], size=30, replace=True),
})

# %%
pd.crosstab(
    [df_cat["states"],df_cat["names"]],
    df_cat["colors"],
    margins=True
)

# %%
pd.crosstab(
    [df_cat["states"],df_cat["names"]], df_cat["colors"],
    margins=True,
    normalize=True
)

# %% [markdown]
"""
## Joint / Conditional / Marginal Distributions
"""

# %% [markdown]
"""
joint distribution

$$f(a,c)$$
"""
# %%
sns.jointplot(df["a"],df["c"])

# %% [markdown]
"""
conditional distribution

$$f(a|c>0)$$
"""
# %%
sns.kdeplot(df.loc[df["c"]>0,"a"])

# %% [markdown]
"""
marginal distribution

$$f(a)$$
"""
# %%
sns.kdeplot(df["a"])

# %% [markdown]
"""
## Simpson's Paradox
"""

# %%
df2 = pd.DataFrame({
    "a":np.random.normal(8, 1, n),
})
df2["c"] = df2["a"] + np.random.normal(8, 0.5, n)
df["group"] = 0
df2["group"] = 1
df_simpson = pd.concat([df[["a","c","group"]],df2])

# %%
sns.regplot(df_simpson["a"], df_simpson["c"], scatter=False, color="red")
sns.scatterplot(df_simpson["a"], df_simpson["c"], hue=df_simpson["group"])

# %%


# Assignment - 2
# Name - Pallav Singla
# Roll No - 2020225

import json

'''
- This is the skeleton code, wherein you have to write the logic for each of the
functions defined below.

- DO NOT modify/delete the given functions.

- DO NOT import any python libraries, except for the ones already included.

- DO NOT create any global variables in this module.

- DO NOT add print statements anywhere in this module.

- Make sure to return value as specified in the function description.

- Remove the pass statement from the functions when you implement them.
'''

def read_data_from_file(file_path="data.json"):
	'''
	**** DO NOT modify this function ****
	Description: Reads the data.json file, and converts it into a dictionary.

	Parameters:
	- file_path (STRING): The path to the file (with .json extension) which contains the initial database. You can pass the file_path parameter as "data.json".

	Returns:
	- A dictionary containing the data read from the file
	'''

	with open(file_path, 'r') as data:
		records = json.load(data)

	return records

def filter_by_first_name(records, first_name):
	'''
	Description: Searches the records to find all persons with the given first name (case-insensitive)

	Parameters:
	- records (LIST): A list of person records (each of which is a dictionary)
	- first_name (STRING): The first name

	Returns:
	- A list of INTEGERS denoting the IDs of the persons with the given first name
		Case 1: No person found => Returns an empty list
		Case 2: At least one person found => Returns a list containing the IDs of all the persons found
	'''

	a=[]
	for i in range(len(records)):
		if (records[i]['first_name']).casefold() == first_name.casefold():
			a.append(records[i]['id'])
		else:
			continue
	return a

def filter_by_last_name(records, last_name):
	'''
	Description: Searches the records to find all persons with the given last name (case-insensitive)

	Parameters:
	- records (LIST): A list of person records (each of which is a dictionary)
	- last_name (STRING): The last name

	Returns:
	- A list of INTEGERS denoting the IDs of the persons with the given last name
		Case 1: No person found => Returns an empty list
		Case 2: At least one person found => Returns a list containing the IDs of all the persons found
	'''

	b=[]
	for i in range(len(records)):
		if (records[i]['last_name']).casefold() == (last_name).casefold():
			b.append(records[i]['id'])
		else:
			continue
	return b

def filter_by_full_name(records, full_name):
	'''
	Description: Searches the records to find all persons with the given full name (case-insensitive)

	Parameters:
	- records (LIST): A list of person records (each of which is a dictionary)
	- full_name (STRING): The full name (a single string with 2 space-separated words, the first name and the last name respectively)

	Returns:
	- A list of INTEGERS denoting the IDs of the persons with the given full name
		Case 1: No person found => Returns an empty list
		Case 2: At least one person found => Returns a list containing the IDs of all the persons found
	'''

	c=[]
	for i in range(len(records)):
		if (records[i]['first_name'] + " " + records[i]['last_name']).casefold() == full_name.casefold():
			c.append(records[i]['id'])
		else:
			continue
	return c

def filter_by_age_range(records, min_age, max_age):
	'''
	Description: Searches the records to find all persons whose age lies in the given age range [min_age, max_age]

	Parameters:
	- records (LIST): A list of person records (each of which is a dictionary)
	- min_age (INTEGER): The minimum age (inclusive)
	- max_age (INTEGER): The maximum age (inclusive)

	Note: 0 < min_age <= max_age

	Returns:
	- A list of INTEGERS denoting the IDs of the persons with the given full name
		Case 1: No person found => Returns an empty list
		Case 2: At least one person found => Returns a list containing the IDs of all the persons found
	'''

	d=[]
	for i in range(len(records)):
		if min_age > 0 and min_age <= max_age:
			if (min_age <= records[i]['age'] <= max_age):
				d.append(records[i]['id'])
			else:
				continue
		else:
			continue
	return d

def count_by_gender(records):
	'''
	Description: Counts the number of males and females

	Parameters:
	- records (LIST): A list of person records (each of which is a dictionary)

	Returns:
	- A dictionary with the following two key-value pairs:
		KEY        VALUE
		"male"     No of males (INTEGER)
		"female"   No of females (INTEGER)
	'''

	m = 0
	f = 0
	for i in range(len(records)):
		if records[i]['gender'] == 'female':
			f+=1
		else:
			m+=1
	Dict = {"male": m, "female": f}

	return Dict

def filter_by_address(records, address):
	'''
	Description: Filters the person records whose address matches the given address.

	Parameters:
	- records (LIST): A list of person records (each of which is a dictionary)
	- address (DICTIONARY): The keys are a subset of { "house_no", "block", "town", "city", "state", "pincode" } (case-insensitive)
		Some examples are:
			Case 1: {}
				=> All records match this case

			Case 2: { "block": "AD", "city": "Delhi" }
				=> All records where the block is "AD" and the city is "Delhi" (the remaining address fields can be anything)

			Case 3: { "house_no": 24, "block": "ABC", "town": "Vaishali", "city": "Ghaziabad", "state": "Uttar Pradesh", "pincode": 110020 }

	Returns:
	- A LIST of DICTIONARIES with the following two key-value pairs:
		KEY            VALUE
		"first_name"   first name (STRING)
		"last_name"    last name (STRING)
	'''

	e = []
	e = address.keys()
	List=[]
	for i in range(len(records)):
		flag = True
		for j in e:
			if j!= 'house_no' and j!= 'pincode':
				if address[j].casefold() != records[i]['address'][j].casefold():
					flag = False
			else:
				if address[j] != records[i]['address'][j]:
					flag = False
		if flag == True:
			List.append({"first_name": records[i]['first_name'], 'last_name': records[i]['last_name']})
	return List

def find_alumni(records, institute_name):
	'''
	Description: Find all the alumni of the given institute name (case-insensitive).

	Note: A person is an alumnus of an institute only if the value of the "ongoing" field for that particular institute is False.

	Parameters:
	- records (LIST): A list of person records (each of which is a dictionary)
	- institute_name (STRING): Name of the institute (case-insensitive)

	Returns:
	- A LIST of DICTIONARIES with the following three key-value pairs:
		KEY            VALUE
		"first_name"   first name (STRING)
		"last_name"    last name (STRING)
		"percentage"   percentage (FLOAT)
	'''

	List2=[]
	for i in range(len(records)):
		for j in range(len(records[i]['education'])):
			if (records[i]['education'][j]["institute"]).casefold() == institute_name.casefold():
				if records[i]['education'][j]['ongoing'] == False:
					List2.append({'first_name':records[i]['first_name'],'last_name':records[i]['last_name'],'percentage':records[i]['education'][j]['percentage']})
				else:
					continue
			else:
				continue
	return List2

def find_topper_of_each_institute(records):
	'''
	Description: Find topper of each institute

	Parameters:
	- records (LIST): A list of person records (each of which is a dictionary)

	Returns:
	- A DICTIONARY with the institute name (STRING) as the keys and the ID (INTEGER) of the topper of that institute.

	Note: If there are `N` distinct institutes in records, the dictionary will contain `N` key-value pairs. The ongoing status does NOT matter. It is guaranteed that each institute will have exactly one topper.
	'''

	Dict2 = {}
	Dict3={}
	for i in range(len(records)):
		L = {}
		for j in range(len(records[i]['education'])-1,-1,-1):
			if records[i]['education'][j]['ongoing'] == False :
				if records[i]['education'][j]['institute'] in L:
					continue
				if (records[i]['education'][j]["institute"] not in Dict2):
					L[records[i]['education'][j]["institute"]] = 1
					Dict2[records[i]['education'][j]["institute"]] = [records[i]['id'],records[i]['education'][j]['percentage']]
				elif Dict2[records[i]['education'][j]["institute"]][1] < records[i]['education'][j]['percentage']:
					Dict2[records[i]['education'][j]["institute"]] = [records[i]['id'],records[i]['education'][j]['percentage']]
					L[records[i]['education'][j]["institute"]] = 1

	for k in (Dict2):
		Dict3[k]=Dict2[k][0]

	return Dict3

def find_blood_donors(records, receiver_person_id):
	'''
	Description: Find all donors who can donate blood to the person with the given receiver ID.

		Note:
		- Possible blood groups are "A", "B", "AB" and "O".

		Rules:
		BLOOD GROUP      CAN DONATE TO
		A                A, AB
		B                B, AB
		AB               AB
		O                A, B, AB, O

	Parameters:
	- records (LIST): A list of person records (each of which is a dictionary)
	- receiver_person_id (INTEGER): The ID of the donee
		Note: It is guaranteed that exactly one person in records will have the ID as receiver_person_id

	Returns:
	- A DICTIONARY with keys as the IDs of potential donors and values as a list of strings, denoting the contact numbers of the donor
	'''

	List3 = {}
	if receiver_person_id < 0:
		return List3
	for i in range(len(records)):
		if records[i]['id'] != receiver_person_id:
			if records[receiver_person_id]['blood_group'] == 'A':
				if records[i]['blood_group'] == 'A' or records[i]['blood_group'] =='O':
					List3[records[i]['id']] = records[i]["contacts"]
				else:
					continue
			elif records[receiver_person_id]['blood_group'] == 'B':
				if records[i]['blood_group'] == 'B' or records[i]['blood_group'] =='O':
					List3[records[i]['id']] = records[i]["contacts"]
				else:
					continue
			elif records[receiver_person_id]['blood_group'] == 'AB':
				if records[i]['blood_group'] == 'A' or records[i]['blood_group'] =='B' or records[i]['blood_group'] =='AB' or records[i]['blood_group'] =='O':
					List3[records[i]['id']] = records[i]["contacts"]
				else:
					continue
			elif records[receiver_person_id]['blood_group'] == 'O':
				if records[i]['blood_group'] == 'O':
					List3[records[i]['id']] = records[i]["contacts"]
				else:
					continue

	return List3

def get_common_friends(records, list_of_ids):
	'''
	Description: Find the common friends of all the people with the given IDs

	Parameters:
	- records (LIST): A list of person records (each of which is a dictionary)
	- list_of_ids (LIST): A list of IDs (INTEGER) of all the people whose common friends are to be found

	Returns:
	- A LIST of INTEGERS containing the IDs of all the common friends of the specified list of people
	'''

	ab = []
	List4 = []
	id_1 = []
	for p in range(len(records)):
		id_1.append(records[p]['id'])
	for j in list_of_ids:
		if j in id_1:
			ab.extend(records[j]["friend_ids"])
		else:
			continue
	for i in ab:
		if ab.count(i)==len(list_of_ids):
			if i not in List4:
				List4.append(i)
		else:
			continue
	return List4

def is_related(records, person_id_1, person_id_2):
	'''
	**** BONUS QUESTION ****
	Description: Check if 2 persons are friends

	Parameters:
	- records (LIST): A list of person records (each of which is a dictionary)
	- person_id_1 (INTEGER): first person ID
	- person_id_2 (INTEGER): second person ID

	Returns:
	- A BOOLEAN denoting if the persons are friends of each other, directly or indirectly (if A knows B, B knows C and C knows D, then A knows B, C and D).
	'''

	# Alternste Code (if time increase pls run this one)
	# for i in range(len(records)):
	# 	if person_id_1 in records[i]["friend_ids"] and person_id_2 in records[i]["friend_ids"]:
	# 		return True
	# 	elif person_id_1 in records[i]["friend_ids"]:
	# 		for j in records[i]["friend_ids"]:
	# 			if person_id_2 in records[j]["friend_ids"]:
	# 				return True
	# 			else:
	# 				continue
	# 	else:
	# 		continue
	# return False

	List_6 = []
	for i in range(len(records)):
		List_6.extend(records[i]['friend_ids'])
	for j in List_6:
		for k in records[j]['friend_ids']:
			if k not in List_6:
				List_6.append(k)
			else:
				continue

	if person_id_1 == person_id_2:
		return False
	elif person_id_1 in List_6 and person_id_2 in List_6:
		return True
	else:
		return False

def delete_by_id(records, person_id):
	'''
	Description: Given a person ID, this function deletes them from the records. Note that the given person can also be a friend of any other person(s), so also delete the given person ID from other persons friend list. If the person ID is not available in the records, you can ignore that case.

	Parameters:
	- records (LIST): A list of person records (each of which is a dictionary)
	- person_id (INTEGER): The person id

	Returns:
	- A LIST of Dictionaries representing all the records (the updated version).
	In case there were no updates, return the original records.
	'''

	for i in records:
		if i['id'] == person_id:
			records.remove(i)
		else:
			continue
		for j in records:
			if person_id in j["friend_ids"]:
				j["friend_ids"].remove(person_id)
			else:
				continue

	return records

def add_friend(records, person_id, friend_id):
	'''
	Description: Given a person ID and a friend ID, this function makes them friends of each other. If any of the IDs are not available, you can ignore that case.

	Parameters:
	- records (LIST): A list of person records (each of which is a dictionary)
	- person_id (INTEGER): The person id
	- friend_id (INTEGER): The friend id

	Returns:
	- A LIST of Dictionaries representing all the records (the updated version).
	In case there were no updates, return the original records.
	'''

	for i in range(len(records)):
		if 0<=person_id < len(records) and 0<=friend_id < len(records) and person_id != friend_id:
			if records[i]['id'] == person_id:
				if friend_id not in records[i]['friend_ids']:
					records[i]['friend_ids'].append(friend_id)
					records[i]['friend_ids'].sort()
			if records[i]['id'] == friend_id:
				if person_id not in records[i]['friend_ids']:
					records[i]['friend_ids'].append(person_id)
					records[i]['friend_ids'].sort()
		else:
			continue

	return records

def remove_friend(records, person_id, friend_id):
	'''
	Description: Given a person ID and a friend ID, this function removes them as friends of each other. If any of the IDs are not available, you can ignore that case.

	Parameters:
	- records (LIST): A list of person records (each of which is a dictionary)
	- person_id (INTEGER): The person id
	- friend_id (INTEGER): The friend id

	Returns:
	- A LIST of Dictionaries representing all the records (the updated version).
	In case there were no updates, return the original records.
	'''

	for i in range(len(records)):
		if person_id!=friend_id:
			if records[i]['id'] == person_id:
				if friend_id in records[i]['friend_ids']:
					records[i]['friend_ids'].remove(friend_id)
			elif records[i]['id'] == friend_id:
				if person_id in records[i]['friend_ids']:
					records[i]['friend_ids'].remove(person_id)
			else:
				continue

	return records

def add_education(records, person_id, institute_name, ongoing, percentage):
	'''
	Description: Adds an education record for the person with the given person ID. The education record constitutes of insitute name, the person's percentage and if that education is currently ongoing or not. Please look at the format of an education field from the PDF. If the person ID is not available in the records, you can ignore that case.

	Parameters:
	- records (LIST): A list of person records (each of which is a dictionary)
	- person_id (INTEGER): The person id
	- institute_name (STRING): The institute name (case-insensitive)
	- ongoing (BOOLEAN): The ongoing value representing if the education is currently ongoing or not
	- percentage (FLOAT): The person's score in percentage

	Returns:
	- A LIST of Dictionaries representing all the records (the updated version).
	In case there were no updates, return the original records.
	'''

	if ongoing == "False":
		ongoing = False
	elif ongoing == "True":
		ongoing = True
	for i in range(len(records)):
		if records[i]['id'] == person_id:
			if ongoing == False:
				records[i]['education'].append({"institute": institute_name, "ongoing": False, "percentage": percentage})
			else:
				records[i]['education'].append({"institute": institute_name, "ongoing": True})

	return records


from django.shortcuts import redirect, render
from . models import Convert
from . forms import ConvertForm
from django.contrib import messages
from pdf2image import convert_from_path
from pathlib import Path
import os

BASE_DIR = Path(__file__).resolve().parent.parent
from pdf2image.exceptions import (
    PDFInfoNotInstalledError,
    PDFPageCountError,
    PDFSyntaxError
)

def converter(p):
    images = convert_from_path(f"C:/Users/91747/Desktop/Projects/pdftopng/{p}",output_folder="C:/Users/91747/Desktop/Projects/pdftopng/output_images/")

    for i, image in enumerate(images):
        fname = 'image'+str(i)+'.png'
        image.save(fname, "PNG")
        return

# Create your views here.
def task(request):
    print(os.path.join(BASE_DIR,'media' ))
    form = ConvertForm()
    if request.method == "POST":
        form = ConvertForm(request.POST,request.FILES)
        if form.is_valid():
            messages.success(request, 'File Uploaded')
            instance = form.save(commit=False)
            instance.save()
            print(instance.file.name)
            converter(instance.file.url)
            return render(request, 'index.html',{'form':form})
        else:
            messages.error(request, 'Something went wrong!')
            return render(request, 'index.html',{'form':form})
    return render(request, 'index.html',{'form':form})


# transactionBuilder.py
# HMC E85 8 December 2017
# hfang@hmc.edu, mjenrungrot@hmc.edu

import secrets
import Transaction
import json
import hashlib
import binascii
import ed25519 # API Documentation is available at: https://github.com/warner/python-ed25519

class TransactionBuilder:
    def __init__(self):
        self.listOfUTXO = None
        self.outputAddresses = None
        self.totalAmount = None
        self.changeAddress = None
        self.feeAmount = 0
        self.secretKey = None
        self.typeW = 'regular'

    def __repr__(self):
        """
        String representation for a transaction
        """
        # jsonDumper is used for recursively converting an object to correct JSON output format
        def jsonDumper(obj):
            return obj.__dict__
        return json.dumps(self, default=jsonDumper)

    def fromAddress(self, listOfUTXO):
        """ Change the from address of the transaction"""
        self.listOfUTXO = listOfUTXO
        return self

    def to(self, address, amount):
        """ Change the outputAddress and total amount of the transaction"""
        self.outputAddress = address
        self.totalAmount = amount
        return self

    def change(self, changeAddress):
        """ Change the changeAddress """
        self.changeAddress = changeAddress
        return self

    def fee(self, amount):
        """ Change the fee of the transaction"""
        self.feeAmount = amount
        return self

    def sign(self, secretKey):
        """ Change secret key of the transaction"""
        self.secretKey = secretKey
        return self

    def type(self, typeT):
        """ Change the type of the transaction """
        self.typeW = typeT

    def build(self):
        """ Build a transaction from the internal parameters"""

        # check for valid transactions by checking
        if self.listOfUTXO == None:
            raise ValueError(" Unspent Output Transactions ")
        elif self.outputAddress == None:
            raise ValueError(" No Output Address ")
        elif self.totalAmount == None:
            raise ValueError(" No Total Amount ")

        # Add up all the amounts from the utxo
        totalAmountOfUTXO = 0
        for output in self.listOfUTXO:
            totalAmountOfUTXO += output["amount"]

        # Remove the transaction amount from the total utxo amounts
        changeAmount =  int(totalAmountOfUTXO) - int(self.totalAmount) - int(self.feeAmount)

        # Generate the inputs
        inputs = []
        for utxo in self.listOfUTXO:
            # Generate the transaction input and calculate the hash/sign the data
            inputStr  = str(utxo["transaction"]) + str(utxo["index"]) + str(utxo["address"])
            txiHash = hashlib.sha256(inputStr.encode('utf-8')).digest()

            # Generate the signing key to sign the transaction
            signing_key, verify_key = self.generateKeyPair(self.secretKey)
            utxo["signature"] = binascii.hexlify(signing_key.sign(txiHash)).decode("utf-8")

            inputs.append(utxo)

        # Generate the outputs
        outputs = []
        outputs.append({"amount":   int(self.totalAmount),
                        "address":  self.outputAddress})

        if changeAmount > 0:
            outputs.append({"amount":   changeAmount,
                            "address":  self.changeAddress})
        else:
            raise ValueError("Sender does not have enough money to  send transaction")

        # return a dictionary of all the values to the create transaction function in Transaction
        buildData = {}
        buildData["id"] = secrets.token_hex(32)
        buildData["hash"] = None
        buildData["type"] = self.typeW
        inOut = {}
        inOut["inputs"] = inputs
        inOut["outputs"] = outputs
        buildData["data"] = inOut

        return Transaction.createTransaction(buildData)

    def generateKeyPair(self, seed):
            """
            Generate Key Public and Private Key Pairs using the EdDSA algorithm library
            """
            keys = {}

            # Note that seed is expected to be 64 hexadecimal characters
            # Convert 64 hexadecimal characters to 32 bytes
            seed_bytes = binascii.a2b_hex(seed)

            # Create the signing key from 32 bytes
            signing_key = ed25519.SigningKey(seed_bytes)

            # Obtain the verify key for a given signing key
            verify_key = signing_key.get_verifying_key()

            return signing_key, verify_key


import numpy as np
import math, time
from drone import DroneEnv

env = DroneEnv()
dt = env.sim.model.opt.timestep

g = 9.81
m = 0.3

def toEuler(q):
    w,x,y,z = q
    ysqr = y * y
    # roll (x-axis rotation)
    t0 = +2.0 * (w * x + y * z)
    t1 = +1.0 - 2.0 * (x * x + ysqr)
    X = math.atan2(t0, t1)
    # pitch (y-axis rotation)
    t2 = +2.0 * (w * y - z * x)
    t2 = +1.0 if t2 > +1.0 else t2
    t2 = -1.0 if t2 < -1.0 else t2
    Y = math.asin(t2)
    # yaw (z-axis rotation)
    t3 = +2.0 * (w * z + x * y)
    t4 = +1.0 - 2.0 * (ysqr + z * z)
    Z = math.atan2(t3, t4)
    return np.array([X, Y, Z])

class Controller:
    def __init__(self,Kp,Ki,Kd,Kp_angle,Ki_angle,Kd_angle):
        self.Kp = Kp
        self.Ki = Ki
        self.Kd = Kd
        self.Kp_angle = Kp_angle
        self.Ki_angle = Ki_angle
        self.Kd_angle = Kd_angle
        self.motor_limits = [0,800]
        self.reset()

    def control(self,pos,vel):
        e = self.target# - pos[:3]
        self.integral += dt*e
        derivative = (self.target-self.old_target)/dt-vel[:3]
        dest_dot = self.Kp*e + self.Kd*derivative + self.Ki*self.integral
        throttle = dest_dot[2]
        yaw = pos[5]
        dest_angle = np.zeros(3)
        dest_angle[:2] = np.array([[np.sin(yaw),-np.cos(yaw)],[np.cos(yaw),np.sin(yaw)]]).dot(dest_dot[:2])
        dest_angle[2] = pos[5] + self.yaw_target
        e_angle = dest_angle - pos[3:]
        self.integral_angle += dt*e_angle
        derivative_angle = 0*(dest_angle-self.dest_angle_old)/dt-vel[3:]
        dest_dot_angle = self.Kp_angle*e_angle + self.Kd_angle*derivative_angle + self.Ki_angle*self.integral_angle
        a = throttle*np.ones(4) + np.array([ dest_dot_angle[0]+dest_dot_angle[2],
                                            -dest_dot_angle[1]-dest_dot_angle[2],
                                            -dest_dot_angle[0]+dest_dot_angle[2],
                                             dest_dot_angle[1]-dest_dot_angle[2]])
        self.integral = np.where(np.abs(self.integral)>1,np.zeros(3),self.integral)
        self.integral_angle = np.where(np.abs(self.integral_angle)>1,np.zeros(3),self.integral_angle)
        print(a)
        # print("e:",e)
        self.old_target = self.target
        self.dest_angle_old = dest_angle
        self.yaw_target = self.yaw_target - dt*vel[5]
        self.target = self.target - dt*vel[:3]
        # print("ie:",(self.target-self.old_target)/dt)
        return np.clip(a,self.motor_limits[0],self.motor_limits[1])

    def addTarget(self,d,amount):
        self.target[d] = amount

    def addYaw_target(self,amount):
        self.yaw_target = amount

    def stop(self,d=-1):
        if d == -1:
            self.yaw_target = 0
        else:
            self.target[d] = 0

    def reset(self):
        self.integral = np.zeros(3)
        self.integral_angle = np.zeros(3)
        self.target = np.zeros(3)
        self.old_target = np.zeros(3)
        self.target[2] = 2
        self.yaw_target = 0
        self.dest_angle_old = np.zeros(3)

Kp = np.array([250,250,15000])
Ki = np.array([0.1,0.1,4.5])
Kd = np.array([990,990,8000])

Kp_angle = np.array([1,1,120])
Ki_angle = np.array([1,1,3])
Kd_angle = np.array([500,500,100])

controller = Controller(Kp,Ki,Kd,Kp_angle,Ki_angle,Kd_angle)

from threading import Thread, Event
import keyboard

def KeyboardListen(cont,env):
    esc = False
    amount = 1
    while not esc:
        key = keyboard.read_event()
        if key.name == 'esc':
            esc = True
        if key.name == 'up':
            cont.addTarget(2,amount)
            if key.event_type == 'up':
                cont.stop(2)
        if key.name == 'down':
            cont.addTarget(2,-amount)
            if key.event_type == 'up':
                cont.stop(2)
        if key.name == 'left':
            cont.addTarget(0,-amount)
            if key.event_type == 'up':
                cont.stop(0)
        if key.name == 'right':
            cont.addTarget(0,amount)
            if key.event_type == 'up':
                cont.stop(0)
        if key.name == 'w':
            cont.addTarget(1,amount)
            if key.event_type == 'up':
                cont.stop(1)
        if key.name == 's':
            cont.addTarget(-amount)
            if key.event_type == 'up':
                cont.stop(1)
        if key.name == 'y':
            cont.addYaw_target(2*amount)
            if key.event_type == 'up':
                cont.stop()
        if key.name == 'x':
            cont.addYaw_target(-2*amount)
            if key.event_type == 'up':
                cont.stop()
        if key.name == 'q':
            env.reset()
            cont.reset()

t = Thread(target=KeyboardListen, args=(controller,env))
t.start()

render = True
n = 100
for i in range(n):
    env.reset()
    done = False
    t = 0
    controller.reset()
    while not done:
        qpos_new = env.sim.data.qpos
        qvel_new = env.sim.data.qvel
        # PID controller
        pos = np.append(env.sim.data.qpos[:3],toEuler(env.sim.data.qpos[3:7]))
        vel = env.sim.data.qvel[:6]
        a = controller.control(pos,vel)
        s_, r, done, _ = env.step(a)
        if render:
            env.render()
        t += dt

# def KeyControl(key):
#     print(key)
#
# lis = keyboard.Listener(on_press=KeyControl)
# lis.start()
# lis.join()

# except Exception as e:
#     print(e)
#     print("End simulation..")


import pygame
from var import scores
from var import screen

clock = pygame.time.Clock()

def print_text(message, x, y, font_color=(0, 0, 0), font_type=None, font_size=30):
    font_type = pygame.font.Font(font_type, font_size)
    text = font_type.render(message, True, font_color)
    screen.blit(text, (x, y))

def pause():
    paused = True
    while paused:
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                pygame.quit()
                quit()
        print_text('Paused. Press enter', 300, 300)

        keys = pygame.key.get_pressed()
        if keys[pygame.K_RETURN]:
            paused = False

        pygame.display.update()
        clock.tick(15)


import pygame
import numpy as np  # For multidimensional arrays, used as grid and rows
import time
import random  # Will be used later to set the cell to alive or dead

# Creating an object called game

class Game:
    def __init__(self, width, height, area, offset):
        self.area = area  # Area of each square, the bigger the area the bigger the square
        # Numbers of row and cols is relative to the width and height and the area
        self.col = int(width/area)  # Number of columns relative to width
        self.row = int(height/area)  # Number of rows relative to the height
        self.size = (self.row, self.col)  # Tuple
        # Shape is a parameter in the ndarray() function that shapes !! Must be a tuple !!
        self.grid_array = np.ndarray(shape=(self.size))
        self.offset = offset  # Border width

    # This function is used to define new cells from 0
    def init_cells(self):
        # Looping in each cell
        for x in range(self.row):
            for y in range(self.col):
                self.grid_array[x][y] = random.randint(
                    0, 1)  # Return an integer, 0 or 1

    # This function will be used to get the total number of cousins arround each cell
    def get_cousins(self, x, y):
        total_cousins = 0
        for n in range(-1, 2):
            for m in range(-1, 2):
                # Calculationg from the current position, x and y the position of their cousins
                calculate_surrounding = lambda n,m,l : (n+m+l)%l
                surroundings_X = calculate_surrounding(x,n,self.row)
                surroundings_Y = calculate_surrounding(y,m,self.col)
                total_cousins += self.grid_array[surroundings_X][surroundings_Y]
        # We romved he current x and y
        total_cousins -= self.grid_array[x][y]
        return total_cousins

    # This function will define each cell's color depending on it status
    # This function will also set the
    def this_cell(self, aliveCol, deadCol, bg_Color):
        for x in range(self.row):
            for y in range(self.col):
                # Get the coordinance of the current cell
                calculate_position= lambda n,m : n * m
                position_Y = calculate_position(y,  self.area)
                position_X = calculate_position(x,  self.area)
                # If the cell is alive == 1 we give it the alive color
                if self.grid_array[x][y] == 1:
                    pygame.draw.rect(bg_Color, aliveCol, [position_X, position_Y, self.area-self.offset, self.area-self.offset])
                # If the cell is dead == 0 we will give it the dead color
                else:
                    pygame.draw.rect(bg_Color, deadCol, [position_X, position_Y, self.area-self.offset, self.area-self.offset])

        # Initialize a new 2 dimensional array that will contain the next move
        next = np.ndarray(shape=(self.size))  # The dimension won't change
        # Looping in each cell
        for x in range(self.row):
            for y in range(self.col):
                # Get the state of the current cell AND the state of the cousins
                # Will return the total number of cousins for the cell at x ,y and will store it in cousins
                cousins = self.get_cousins(x, y)
                state = self.grid_array[x][y]

                # Rules
                if state == 0 and cousins == 3:
                    next[x][y] = 1
                elif state == 1 and (cousins < 2 or cousins > 3):
                    next[x][y] = 0
                else:
                    next[x][y] = state
        self.grid_array = next  # Update the new changes

#The end of this class


import os
import pygame
from arctia.bfont import BitmapFont
from arctia.resources import load_image

def test_bfont_load():
    pygame.init()
    font_img = load_image('gfx/fawnt.png')
    chars = 'ABC'

    expected = [(1, 0, 7, 12),
                (9, 0, 7, 12),
                (17, 0, 7, 12)]

    bfont = BitmapFont(chars, font_img)

    assert bfont is not None
    for i, cell in enumerate(bfont.cells[:2]):
        assert cell == expected[i]
    assert len(bfont.cells) == len(chars)

def test_bfont_measure():
    pygame.init()
    font_img = load_image('gfx/fawnt.png')
    chars = 'ABC'
    bfont = BitmapFont(chars, font_img)
    result = bfont.measure('BACAB\nAB')
    assert(bfont.measure('BACAB\nAB') == (35, 24))


"""
This file measures the delay/performance of requests
"""

import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from Parameters import start_time, max_wait_time, max_pickup_time

scenario = '10_16_ml'

### requests' delay ###
request = pd.read_csv("H:\\EAV Taxi Data\\"+scenario+"\\request_output.csv")
request.dtypes
num_request = request.shape[0]

#percent of completed requests
request['served'].describe()
request[request['served']==False].shape[0]/request.shape[0]
request[request['served']==True].shape[0]/request.shape[0]

#delete False
request_rej = request[request['served']==False]
request = request[request['served']==True]

#delay time
request['wait_time'].describe()
np.percentile(request['wait_time'], 95)

# =============================================================================
# bins = np.linspace(0, 45, 46)
# fig = plt.figure()
# plt.hist(request['wait_time'], bins, color='y')
# plt.xlabel("Delay time (minutes)")
# plt.ylabel("Frequency")
# plt.xticks(np.linspace(0, 45, 10))
# fig.savefig("H:\\EAV Taxi Data\\"+scenario+"\\delay_time"+".jpg", figsize=(5, 3), dpi=300)
# plt.show()
# =============================================================================

#taxi pickup time
request['taxi_pickup_time'].describe()
np.percentile(request['taxi_pickup_time'], 90)
#% that can be picked up within x minutes
[request[request['taxi_pickup_time']<=x].shape[0] / request.shape[0] for x in [1,3,5,7,10,15]]

bins = np.linspace(0, 30, 31)
fig = plt.figure(figsize=(5, 3))
plt.hist(request['taxi_pickup_time'], bins, color='g', alpha=0.5)
plt.xlabel("Taxi pickup time (min)")
plt.ylabel("Frequency")
plt.yticks(np.linspace(0, 8000, 9))
plt.tight_layout()
plt.show()
#fig.savefig("H:\\EAV Taxi Data\\"+scenario+"\\pickup_time"+".jpg",  dpi=300)

#wait time before accepted
request['wait_time_before_accepted'] = request['wait_time']-request['taxi_pickup_time']
request['wait_time_before_accepted'].describe()
np.percentile(request['wait_time_before_accepted'], 95)

# =============================================================================
# bins = np.linspace(0, 15, 16)
# fig = plt.figure()
# plt.hist(request['wait_time_before_accepted'], bins, color='C1')
# plt.xlabel("Wait time until accepted (minutes)")
# plt.ylabel("Frequency")
# plt.xticks(np.linspace(0, 15, 16))
# fig.savefig("H:\\EAV Taxi Data\\"+scenario+"\\wait_time_accepted"+".jpg", figsize=(5, 3), dpi=300)
# plt.show()
# =============================================================================

def CompareTaxiPickupTime():
    op = pd.read_csv("H:\\EAV Taxi Data\\"+'10_16_op'+"\\request_output.csv")
    op = op[op['served']==True]
    op = [op[op['taxi_pickup_time']<=x].shape[0] / op.shape[0] for x in range(1,31)]
    ml = pd.read_csv("H:\\EAV Taxi Data\\"+'10_16_ml'+"\\request_output.csv")
    ml = ml[ml['served']==True]
    ml = [ml[ml['taxi_pickup_time']<=x].shape[0] / ml.shape[0] for x in range(1,31)]

    fig = plt.figure(figsize=(5, 3))
    plt.plot(op, color='b', alpha=0.5, label='EAV taxis \n(optimization dispatch)', linestyle='dashed')
    plt.plot(ml, color='g', alpha=0.5, label='EAV taxis \n(NN dispatch)', linestyle='dashed')
    plt.legend(loc='lower right')
    plt.xlabel("Taxi pickup time (min)")
    plt.ylabel("Prob. of occurrence")
    x_tick = ['\u2264'+'1', '', '\u2264'+'3', '','\u2264'+'5',
              '', '\u2264'+'7', '', '', '\u2264'+'10',
              '','','','','\u2264'+'15',
              '','','','','\u2264'+'20',
              '','','','','\u2264'+'25',
              '','','','','\u2264'+'30']
    plt.xticks(range(0,30), x_tick)
    plt.yticks([0,0.2,0.4,0.6,0.8,1])
    plt.tight_layout()
    plt.show()
    fig.savefig("H:\\EAV Taxi Data\\"+'10_16_ml'+"\\compare_pickup"+".jpg", dpi=300)

CompareTaxiPickupTime()

def CompareAcceptOrRejTime():
    op = pd.read_csv("H:\\EAV Taxi Data\\"+'10_16_op'+"\\request_output.csv")
    op = op[op['served']==True]
    op['wait_time_before_accepted'] = op['wait_time']-op['taxi_pickup_time']
    op = [op[op['wait_time_before_accepted']<=x].shape[0] / num_request for x in range(0,16)]
    ml = pd.read_csv("H:\\EAV Taxi Data\\"+'10_16_ml'+"\\request_output.csv")
    ml = ml[ml['served']==True]
    ml['wait_time_before_accepted'] = ml['wait_time']-ml['taxi_pickup_time']
    ml = [ml[ml['wait_time_before_accepted']<=x].shape[0] / num_request for x in range(0,16)]

    fig = plt.figure(figsize=(5, 3))
    plt.plot(op, color='b', alpha=0.5, label='EAV taxis \n(optimization dispatch)', linestyle='dashed')
    plt.plot(ml, color='g', alpha=0.5, label='EAV taxis \n(NN dispatch)', linestyle='dashed')
    plt.legend(loc='lower right')
    plt.xlabel("Waiting time before accepted (min)")
    plt.ylabel("Prob. of occurrence")
    x_tick = ['0', '\u2264'+'1', '', '\u2264'+'3', '','\u2264'+'5',
              '', '\u2264'+'7', '', '', '\u2264'+'10',
              '','','','','\u2264'+'15']
    plt.xticks(range(0,16), x_tick)
    plt.yticks([0.90,0.92,0.94,0.96,0.98,1])
    plt.tight_layout()
    plt.show()
    fig.savefig("H:\\EAV Taxi Data\\"+'10_16_ml'+"\\compare_accept_rej"+".jpg", dpi=300)

CompareAcceptOrRejTime()

#%%for all request

#% that can be accepted 0-x+1 min
[request[request['wait_time_before_accepted']<=x].shape[0] / num_request for x in [0,1,2,3,4]]

#wait time before accepted or rejected
combine_wait = request['wait_time_before_accepted'].append(request_rej['wait_time'])
combine_wait[combine_wait==16] = 15
combine_wait.describe()
np.percentile(combine_wait, 90)

# =============================================================================
# bins = np.linspace(0, 15, 16)
# fig = plt.figure()
# plt.hist(combine_wait, bins, color='C1')
# plt.xlabel("Wait time until accepted/rejected (minutes)")
# plt.ylabel("Frequency")
# plt.xticks(np.linspace(0, 15, 16))
# fig.savefig("H:\\EAV Taxi Data\\"+scenario+"\\wait_time_accepted_rejected"+".jpg", figsize=(5, 3), dpi=300)
# plt.show()
# =============================================================================

#total cost of system
temp = request_rej['wait_time']
temp[temp==16] = 15
sum(request['taxi_pickup_time']) + (sum(temp) + max_pickup_time*len(temp))
sum(request['wait_time']) + (sum(temp) + max_pickup_time*len(temp))

#plot GPS of rejected requests
fig = plt.figure(figsize=(5, 3))
plt.scatter(x=request_rej['origin_longitude'], y=request_rej['origin_latitude'], color='r')
plt.xlabel("Longitude")
plt.ylabel("Latitude")
plt.tight_layout()
fig.savefig("H:\\EAV Taxi Data\\"+scenario+"\\GPS_rejected"+".jpg", dpi=300)

#%%for all rejected requestes
#convert origin_timestamp into hours
request_rej['origin_hour'] = (request_rej['origin_timestamp']-start_time)/3600
#plot time distribution histogram
plt.hist(request_rej['origin_hour'], bins=72)

#%%for taxis at the end of simulation
### taxi SOC ###
#taxi end SOC
taxi = pd.read_csv("H:\\EAV Taxi Data\\"+scenario+"\\taxi_output.csv")
taxi.dtypes

#summary
taxi['end_range'].describe()
np.percentile(taxi['end_range'], 90)
np.percentile(taxi['end_range'], 95)

#histogram
plt.hist(taxi['end_range'])
plt.xlabel("Remaining range (mi)")
plt.ylabel("Frequency")

#boxplot
plt.boxplot(taxi['end_range'])

#the end location of taxis
plt.scatter(x=taxi['end_longitude'], y=taxi['end_latitude'])
plt.scatter(x=request_rej['origin_longitude'], y=request_rej['origin_latitude'])


#this is for post method
import requests
import json
import jsonpath

url = "https://reqres.in/api/users"

#Read input json file
file = open('post.json')

json_input = file.read()

#json.loads = type cast
request_json = json.loads(json_input)
response = requests.post(url,request_json)

#validating Responce
assert response.status_code == 201

#fetch header from the responce
#print(response.headers)
print(response.headers.get('Content-Type'))

#parse response into the json format
response_json = json.loads(response.text)


from PyQt5 import QtGui
from PyQt5.QtWidgets import QApplication,QMainWindow,QPushButton,QStatusBar
from PyQt5.QtCore import QDateTime
import sys

class Window(QMainWindow):

    def __init__(self):
        super().__init__()

        self.setGeometry(100,100,680,540)
        self.datetime = QDateTime.currentDateTime().toString()

        self.setWindowTitle("Pritam "+self.datetime)

        self.InitUI()

    def InitUI(self):

        self.setStyleSheet("background-color:grey;")
        self.statusbar = self.statusBar()
        self.statusbar.showMessage("Status Bar")
        self.statusbar.setStyleSheet("background-color:blue;")

        self.show()

def main():
    app = QApplication(sys.argv)
    window = Window()
    sys.exit(app.exec())

if __name__=="__main__":
    main()


import sqlite3

from misc import Models

class DataBase:
    def __init__(self, connection: sqlite3.Connection):
        self.__db = connection
        self.__cur = connection.cursor()
        self.__cur.execute(
            '''CREATE TABLE IF NOT EXISTS "books" (
                "id" INTEGER,
                "name"	TEXT,
                "author"    TEXT,
                "year"  INTEGER,
                PRIMARY KEY("id")
            )'''
        )

    def add_book(self, name: str, author: str, year: int) -> Models.Book:
        """
        - Добавление книги: Пользователь вводит информацию о книге: название, автор, год издания.
        Данные сохраняются в базу.

        Adds a new book to database

        :param name: book's name
        :param author: book's author
        :param year: year when book was published
        :return: New instance of Models.Book with given data
        """

        last_book_id = self.__cur.execute("SELECT id FROM books ORDER BY id DESC LIMIT 1").fetchone()  # get last id
        if not last_book_id:  # empy database, id = 1
            new_id = 1
        else:
            new_id = last_book_id[0] + 1  # last book's id + 1

        query = "INSERT INTO books (id, name, author, year) VALUES (?, ?, ?, ?)"
        self.__cur.execute(query, (new_id, name, author, year))
        self.__db.commit()

        return Models.Book(new_id, name, author, year)

    def search_books(self, name: str = None, author: str = None, year: int or str = None) -> list[Models.Book] or []:
        """
        - Поиск книги: По заданным параметрам (название, автор, год издания) производится поиск
        соответствующих книг в базе.

        Searchs for all books that match query

        :param name: keyword that book's name must contain
        :param author: author of book
        :param year: year of book's publications
        :raises ValueError if bad query given
        :return: list of [Models.Book] or empty list
        """

        if not name and not author and not year:
            raise ValueError("Bad searching query was given. can't find anything. Pass some information about book")

        query = "SELECT id, name, author, year FROM books WHERE 1"
        query_params = []

        # conditions
        if name:
            query += " AND name LIKE ?"
            query_params.append(f"%{name}%")
        if year:
            query += " AND year LIKE ?"
            query_params.append(year)  # exact year
        if author:
            query += " AND author LIKE ?"
            query_params.append(f"%{author}%")

        search_results = self.__cur.execute(query, query_params).fetchall()
        books_list = [Models.Book(**book_data) for book_data in search_results]
        return books_list

    def delete_book(self, book_id: int) -> None:
        """
        - Удаление книги: Пользователь может удалить книгу из базы по заданным параметрам.

        Deletes book from database

        :param book_id: id of book to delete
        :return: None
        """

        query = "DELETE FROM books WHERE id = ?"
        self.__cur.execute(query, (book_id, ))
        self.__db.commit()

    def edit_book(self, book_id: int, **kwargs) -> Models.Book:
        """
        - Редактирование данных о книге: Пользователь может изменить данные о книге в базе.

        Edits book genral information

        :param book_id: id of book to edit
        :param kwargs: new information: name, author, year
        :raises ValueError if book id was not found
        :return: new instance of Models.Book of edited book
        """

        query = "SELECT id, name, author, year FROM books WHERE id = ?"  # get actual info about book
        search_results = self.__cur.execute(query, (book_id, )).fetchone()

        if not search_results:
            raise ValueError("No such id in database")

        # create Book model from actual info
        actual_info = Models.Book(**search_results)

        # or use search_results[0], search_results[1], search_results[2]
        name = kwargs.get("name", actual_info.name)  # if name not passed - take current
        author = kwargs.get("author", actual_info.author)  # if author not passed - take current
        year = kwargs.get("year", actual_info.year)  # if year not passed - take current

        query = "UPDATE books SET name = ?, author = ?, year = ? WHERE id = ?"  # edit
        self.__cur.execute(query, (name, author, year, book_id)).fetchone()
        self.__db.commit()

        return Models.Book(
            id=book_id,
            name=name,
            author=author,
            year=year
        )

    def get_all_books(self) -> list[Models.Book] or []:
        """
        - Вывод списка всех книг: Выводит список всех книг в базе.

        Returns list of all books in database

        :return: list of [Models.Book] or empty list
        """

        query = "SELECT id, name, author, year FROM books"
        search_results = self.__cur.execute(query).fetchall()
        books_list = [Models.Book(**book_data) for book_data in search_results]
        return books_list


# -*- coding: utf-8 -*-

"""
Sub-command for Changes.

In all cases ``data = ctx.params`` when calling the appropriate action method
on ``ctx.obj``. (e.g. ``ctx.obj.add(ctx.params)``)

Also, ``action = ctx.info_name`` *might* reliably contain the name of the
action function, but still not sure about that. If so, every function could be
fundamentally simplified to this::

    getattr(ctx.obj, ctx.info_name)(ctx.params)
"""

from __future__ import unicode_literals

from __future__ import absolute_import
import click
from . import callbacks

__author__ = 'Jathan McCollum'
__maintainer__ = 'Jathan McCollum'
__email__ = 'jathan@dropbox.com'
__copyright__ = 'Copyright (c) 2015 Dropbox, Inc.'

# Ordered list of 2-tuples of (field, display_name) used to translate object
# field names oto their human-readable form when calling .print_list().
DISPLAY_FIELDS = (
    ('id', 'ID'),
    ('change_at', 'Change At'),
    ('user', 'User'),
    ('event', 'Event'),
    ('resource_name', 'Resource'),
    ('resource_id', 'Obj'),
    # ('site_id', 'Site ID'),
    # ('site', 'Site'),
)

# Fields to display when viewing a single record.
VERBOSE_FIELDS = (
    ('change_at', 'Change At'),
    ('user', 'User'),
    ('event', 'Event'),
    ('resource_name', 'Resource'),
    ('resource_id', 'ID'),
    ('resource', 'Data'),
)

# Main group
@click.group()
@click.pass_context
def cli(ctx):
    """
    Change events.

    All add, remove, and update events are logged as a Change. A Change
    includes information such as the change time, user, and the full resource
    after modification. Changes are immutable and can only be removed by
    deleting the entire Site.
    """

# List
@cli.command()
@click.option(
    '-e',
    '--event',
    metavar='EVENT',
    help='Filter result to specific event.',
    callback=callbacks.transform_event,
)
@click.option(
    '-i',
    '--id',
    metavar='ID',
    type=int,
    help='Unique ID of the Change being retrieved.',
)
@click.option(
    '-l',
    '--limit',
    metavar='LIMIT',
    type=int,
    help='Limit result to N resources.',
)
@click.option(
    '-o',
    '--offset',
    metavar='OFFSET',
    type=int,
    help='Skip the first N resources.',
)
@click.option(
    '-R',
    '--resource-id',
    metavar='RESOURCE_ID',
    type=int,
    help='Filter to Changes for a specific resource ID (e.g. Network ID 1)',
)
@click.option(
    '-r',
    '--resource-name',
    metavar='RESOURCE_NAME',
    help='Filter to Changes for a specific resource name (e.g. Network)',
    callback=callbacks.transform_resource_name,
)
@click.option(
    '-s',
    '--site-id',
    metavar='SITE_ID',
    type=int,
    help='Unique ID of the Site this Change is under.  [required]',
    callback=callbacks.process_site_id,
)
@click.pass_context
def list(ctx, event, id, limit, offset, resource_id, resource_name, site_id):
    """
    List Change events for a Site.

    You must provide a Site ID using the -s/--site-id option.

    When listing Changes, all events are displayed by default. You may
    optionally lookup a single Change by ID using the -i/--id option.

    You may limit the number of results using the -l/--limit option.
    """
    data = ctx.params

    # If we provide ID, be verbose
    if id is not None:
        display_fields = VERBOSE_FIELDS
    else:
        display_fields = DISPLAY_FIELDS

    ctx.obj.list(data, display_fields=display_fields)


import pandas as pd
import glob
import csv
import os
import pickle
from datetime import datetime, timedelta
from dataclasses import dataclass

#########################################################################################
# The class that takes all the attributes related to a specific identifier.
@dataclass
class Identifier:
    def __init__(self, id_hex, occurrences):
        self.id_hex = id_hex
        self.occurrences = occurrences
        self.id_int = 0
        self.payload_changes = -1
        self.last_payload = "xyz"
        self.unique_payloads = {}
        self.time_dict = {}

##########################################################################################
identifier_dict = {}  # a container for all the identifier objects
priority_ids_list = []
seconds = 3   # Time resolution in seconds(integer) needed for matching Id timestamps to the Action Timelogs.

# create and specify the necessary paths/folders
if not os.path.exists("Output_CSVs"):
    os.mkdir("Output_CSVs")
    print("Output_CSVs directory has been created")
else:
    print("Output_CSVs directory already exists")

if not os.path.exists("Makeshift_DB"):
    os.mkdir("Makeshift_DB")
    print("Makeshift_DB, the directory for persisted storage has been created.")
else:
    print("Makeshift_DB directory already exists")

input_path = os.path.join(".", "Input_CSVs", "*.csv")
output_path = os.path.join(".", "Output_CSVs")
db_path = os.path.join(".", "Makeshift_DB")

# get the files and create a dataframe df
ixxat_csv_filenames = glob.glob(input_path)
ixxat_csv_filenames.sort()
frame_list = []

for filename in ixxat_csv_filenames:
    single_frame = pd.read_csv(filename, header=0, index_col=None)
    frame_list.append(single_frame)

df = pd.concat(frame_list, ignore_index=True)
print(df.iloc[84780:84790])  # making sure that the concatenation worked ok

# group the ids by the frequency of occurrences
# create an extra helper column and populate it with '1's to achieve this
df['Count'] = 1
grouped_df = df.groupby(df['ID (hex)']).count()['Count'].reset_index()

# add the ids and occurrences to instances of the data class and add the instances to a dictionary
for index, row in grouped_df.iterrows():
    identifier = Identifier(row["ID (hex)"], row["Count"])
    identifier_dict[row["ID (hex)"]] = identifier

# convert hex string to int and save to id_int # useful later when plotting graphs and histograms
for id in identifier_dict:
    identifier_dict[id].id_int = int(id, 16)

# iterate over the dataframe rows and:
# (a)check how often the datafield changes
# (b)see how many different messages an id is used for (it could be in (a) that only the same few messages keep getting changed over and over)
# Time this as the core operation
start_time = datetime.now()
for row in df.itertuples():
    identifier = identifier_dict[row[1]]
    if row[3] in identifier.unique_payloads.keys():
        if row[3] != identifier.last_payload:
            identifier.last_payload = row[3]
            identifier.payload_changes += 1
            identifier.time_dict.update({row[5]: row[3]})
        identifier.unique_payloads[row[3]] += 1

    else:
        identifier.unique_payloads[row[3]] = 1
        identifier.last_payload = row[3]
        identifier.payload_changes += 1
        identifier.time_dict.update({row[5]: row[3]})
stop_time = datetime.now()

print("ID 0x108 changes at the following times:")
print(identifier_dict["108"].time_dict)

# Save the identifier objects in a makeshift file database
with open(os.path.join(db_path, "Identifier_DB_pickle"), "wb") as db:
    pickle.dump(identifier_dict, db)

# Some printing for debugging purposes
print("0x108 unique payloads are:")
print(identifier_dict["108"].unique_payloads)

print("0x303 unique payloads are:")
print(identifier_dict["303"].unique_payloads)

# go through the dictionary, check the payload changes and the size of the unique_payloads dict
# write out this data to a csv file
with open(os.path.join(output_path, "Filtered_Sorted(by_ID).csv"), 'w') as csvfile:
    writer = csv.writer(csvfile)
    writer.writerow(["ID_(dec)", "ID_(hex)", "Occurrences", "Payload_Changes", "Total_Unique_Payloads"])
    for key in identifier_dict:
        writer.writerow(
            [identifier_dict[key].id_int, key, identifier_dict[key].occurrences, identifier_dict[key].payload_changes,
             len(identifier_dict[key].unique_payloads)])

# generate a csv of the output sorted by size of the payload changes
# read in the previously output csv file to do this
filtered_df = pd.read_csv(os.path.join(output_path, "Filtered_Sorted(by_ID).csv"), header=0, index_col=None)
filtered_df = filtered_df.sort_values(["Payload_Changes", "Occurrences"], ascending=[1, 0])
filtered_df.to_csv(os.path.join(output_path, "Filtered_Sorted(by_PayloadChanges).csv"))

# generate additional different csv files based on the number of changes
# create 3 different csv files for this...an extra csv file in the db to save the names of the analysable ID
f1 = open(os.path.join(output_path, "Filtered(No_Change).csv"), "w")
f2 = open(os.path.join(output_path, "Filtered(Analysable_Changes).csv"), "w")
f3 = open(os.path.join(output_path, "Filtered(Many_Changes).csv"), "w")

writer1 = csv.writer(f1)
writer1.writerow(["ID_(dec)", "ID_(hex)", "Occurrences", "Payload_Changes"])
writer2 = csv.writer(f2)
writer2.writerow(["ID_(dec)", "ID_(hex)", "Occurrences", "Payload_Changes", "Total_Unique_Payloads"])
writer3 = csv.writer(f3)
writer3.writerow(["ID_(dec)", "ID_(hex)", "Occurrences", "Payload_Changes", "Total_Unique_Payloads"])

for identifier in identifier_dict.values():
    if identifier.payload_changes == 0 and identifier.occurrences > 25:
        writer1.writerow([identifier.id_int, identifier.id_hex, identifier.occurrences, identifier.payload_changes])

    elif (identifier.payload_changes == 0 and identifier.occurrences <= 25) or (1 <= identifier.payload_changes <= 25):
        writer2.writerow([identifier.id_int, identifier.id_hex, identifier.occurrences, identifier.payload_changes,
                          len(identifier.unique_payloads)])
        priority_ids_list.append(identifier.id_hex)

    else:
        writer3.writerow([identifier.id_int, identifier.id_hex, identifier.occurrences, identifier.payload_changes,
                          len(identifier.unique_payloads)])

f1.close()
f2.close()
f3.close()

########################################################################################################################
# MATCHING THE ANALYSABLE IDS TO THE TIMELOGGED ACTIONS
# This should probably be done in a separate script

# Read in the timelogs and save to a timelog dictionary
tlog_input_path = os.path.join(".", "TimeLog_CSVs", "*.csv")
tlog_filenames = glob.glob(tlog_input_path)
frame_list = []
for filename in tlog_filenames:
    single_frame = pd.read_csv(filename, header=0, index_col=None)
    frame_list.append(single_frame)
timelog_df = pd.concat(frame_list, ignore_index=True)

# Save the timelogs in a dictionary with a string version of the time without milliseconds as the key and the action as the value.
timelog_dict = {}
for index, row in timelog_df.iterrows():
    time_key = datetime.strftime((datetime.strptime(row["Time"], "%H:%M:%S.%f").replace(microsecond=0)), "%H:%M:%S.%f")
    timelog_dict[time_key] = row["Action"]
print(timelog_dict)

# Create a new csv file to save the action to id match.
with open(os.path.join(output_path, "ID_Action_Match.csv"), "w") as f_match:
    writer_m = csv.writer(f_match)
    writer_m.writerow(["ID_Occurrences", "Payload_Changes", "ID_(hex)", "ID_Timestamp", "Action_Timestamp(w/out milliseconds)", "Action"])

    # Match the IDs to the timelogs #
    # for each timestamp in each Identifier's time_dict, create a new list that has the timestamp range +/-3 seconds
    # remove the milliseconds
    # check if any of these new timestamps match any of the timelog timestamps (also without milliseconds)

    for id in priority_ids_list:
        for t_stamp in identifier_dict[id].time_dict.keys():
            t_keys = []
            t1 = datetime.strptime(t_stamp, "%H:%M:%S.%f").replace(microsecond=0)
            for i in range(-3, 4):
                t_key = datetime.strftime((t1 + timedelta(seconds=i)), "%H:%M:%S.%f")
                t_keys.append(t_key)
            for tkey in t_keys:
               if tkey in timelog_dict.keys():
                    writer_m.writerow([identifier_dict[id].occurrences, identifier_dict[id].payload_changes,
                                       id, t_stamp, tkey, timelog_dict[tkey]])
                    print("id = {}  t_stamp = {}     t_key = {}".format(id, t_stamp, tkey))

print(f"The core operation runs for {stop_time - start_time} seconds")


import numpy as np
import h5py
import torch
from datasets import ExtractDataset

def extract_collate(batch):
    assert len(batch) == 1
    x = batch[0][0]
    spk_id = batch[0][1]
    utt_id = batch[0][2]
    return [x[None, ...], spk_id, utt_id]

def sequential_extract(model, mfcc_file, save_xvec_to):
    print(f"Reading mfcc from {mfcc_file}")
    with h5py.File(mfcc_file, "r") as fr:
        dset = ExtractDataset(fr)
        loader = torch.utils.data.DataLoader(
            dset, batch_size=1, shuffle=False, collate_fn=extract_collate
        )
        print(f"Saving xvectors to {save_xvec_to}")
        with h5py.File(save_xvec_to, "w") as fw:
            fw["X"], fw["spk_ids"], fw["spk_path"] = _extraction(model, loader)

def _extraction(model, loader):
    unicode = h5py.special_dtype(vlen=str)
    model.eval()
    X, spk_ids, utt_ids = [], [], []
    with torch.no_grad():
        for batch_idx, (mfcc, spk_id, utt_id) in enumerate(loader):
            mfcc = mfcc.to(torch.device(torch.cuda.current_device()))
            x = model.extract(mfcc)
            spk_ids.append(spk_id)
            utt_ids.append(utt_id)
            X.append(x)

        X = torch.cat(X).to("cpu").numpy()
        spk_ids = np.stack(spk_ids).astype(unicode)
        utt_ids = np.stack(utt_ids).astype(unicode)
    return X, spk_ids, utt_ids


import argparse
import logging

from tokenizers import Tokenizer

from faser.data_utils import fstrs_to_tokeniser_training_data
from faser.eval import FaserGeneralFuncSearchEval, FaserVulnSearchEval
from faser.tokeniser import print_tokeniser_encode_example, train_bpe_tokeniser
from faser.train import FASERTrain

parser = argparse.ArgumentParser(
    prog="cli.py", description="Frontend for training and testing FASER models"
)

sub_parsers = parser.add_subparsers(dest="command")
sub_parsers.required = True

# Creating Tokenisers
parser_tokeniser = sub_parsers.add_parser(
    "tokeniser", help="train a huggingface tokeniser"
)
parser_tokeniser.add_argument(
    "-o", "--output-path", required=True, help="the name of the tokeniser"
)
parser_tokeniser.add_argument(
    "-i",
    "--input-data",
    nargs="+",
    required=True,
    help="the input data corpus to train the tokeniser",
)

parser_tokeniser.add_argument(
    "-m",
    "--max-seq-len",
    required=True,
    help="the max seq length for the tokeniser to make (including specials)",
)

# Testing Tokeniser/Encode
parser_encode = sub_parsers.add_parser(
    "encode", help="encode a string with a given tokeniser"
)
parser_encode.add_argument("input", type=str, help="the input sequence to encode")
parser_encode.add_argument("-t", "--tokeniser-path", help="path to a trained tokeniser")

# Convert function as string data to a text file
parser_fstr2txt = sub_parsers.add_parser(
    "fstr2txt",
    help="convert fstr JSON to newline delimited text file to train tokeniser",
)
parser_fstr2txt.add_argument("data_dir", help="path to data directory")
parser_fstr2txt.add_argument("-o", "--output-name", help="output name", required=True)

# Training a FASER model
parser_train = sub_parsers.add_parser("train", help="Train a FASER model")
parser_train.add_argument("--train_data", help="path to traini data", required=True)
parser_train.add_argument("--test_data", help="path to test data", required=True)
parser_train.add_argument(
    "-t", "--tokeniser_fp", help="path to tokeniser", required=True
)
parser_train.add_argument("-n", "--name", help="name of model", required=True)

## Good Defaults
parser_train.add_argument("-e", "--epochs", help="Num training epochs", default=30)
parser_train.add_argument("-b", "--batch_size", help="Training batch size", default=8)
parser_train.add_argument(
    "-lr", "--learning_rate", help="Training learning rate", default=0.00005
)
parser_train.add_argument(
    "--num_accumlation_steps", help="Steps to do gradient accumlation too", default=512
)
parser_train.add_argument(
    "--gradient_accumulation",
    help="toggle to set gradient accumlation",
    action="store_false",
)
parser_train.add_argument("--num_pos_pairs_in_batch", default=2)
parser_train.add_argument("--filter-str", default=None)

# Evaluating FASER Model - General Function Search
parser_func_search_eval = sub_parsers.add_parser(
    "fseval", help="General function search evaluation for FASER"
)

parser_func_search_eval.add_argument(
    "-n",
    "--num-eval-sp",
    type=int,
    help="Number of Search Pools to eval with",
    default=1400,
)
parser_func_search_eval.add_argument(
    "-i", "--input-dim", type=int, help="Size of input dimension", default=4096
)
parser_func_search_eval.add_argument(
    "-d", "--eval-data", type=str, help="Path to the evaluation data", required=True
)
parser_func_search_eval.add_argument(
    "-t", "--tokeniser", type=str, help="Path to the tokeniser", required=True
)
parser_func_search_eval.add_argument(
    "-m", "--model", type=str, help="Path to the model", required=True
)
parser_func_search_eval.add_argument(
    "-s",
    "--sp-size",
    type=int,
    help="Number of elements in a single search pool",
    default=100,
)
parser_func_search_eval.add_argument(
    "-f",
    "--filter-str",
    type=str,
    help="Filter which training filenames to use",
    required=False,
)

# Evaluating FASER Model - Vulnerability Function Search
parser_func_search_eval.add_argument("-v", "--verbose", action="store_true")

parser_vuln_search_eval = sub_parsers.add_parser(
    "vseval", help="Vulnerability search evaluation for FASER"
)
parser_vuln_search_eval.add_argument(
    "-i", "--input-dim", type=int, help="Size of input dimension", default=4096
)
parser_vuln_search_eval.add_argument(
    "-d", "--eval-data", type=str, help="Path to the evaluation data", required=True
)
parser_vuln_search_eval.add_argument(
    "-t", "--tokeniser", type=str, help="Path to the tokeniser", required=True
)
parser_vuln_search_eval.add_argument(
    "-m", "--model", type=str, help="Path to the model", required=True
)
parser_vuln_search_eval.add_argument(
    "-f",
    "--model-friendly-name",
    type=str,
    help="Model name for output artefacts",
    required=True,
)
args = parser.parse_args()

if args.command == "fstr2txt":
    fstrs_to_tokeniser_training_data(args.data_dir, args.output_name)

elif args.command == "tokeniser":
    tokeniser = train_bpe_tokeniser(
        args.input_data, max_seq_length=int(args.max_seq_len)
    )
    tokeniser.save(f"{args.max_seq_len}-{args.output_path}")

elif args.command == "encode":
    tokeniser = Tokenizer.from_file(args.tokeniser_path)
    print_tokeniser_encode_example(tokeniser, args.input)

elif args.command == "train":
    model = FASERTrain(
        name=args.name,
        train_data_fp=args.train_data,
        test_data_fp=args.test_data,
        tokeniser_fp=args.tokeniser_fp,
        batch_size=args.batch_size,
        num_pos_pairs_in_batch=args.num_pos_pairs_in_batch,
        learning_rate=args.learning_rate,
        num_training_epochs=args.epochs,
        num_accumlation_steps=args.num_accumlation_steps,
        filter_str=args.filter_str,
    )

    model.train()

elif args.command == "fseval":
    if args.verbose:
        logging.basicConfig(level=logging.DEBUG)

    eval_run = FaserGeneralFuncSearchEval(
        torch_model_bin_fp=args.model,
        eval_data_fp=args.eval_data,
        max_seq_len=args.input_dim,
        tokeniser_fp=args.tokeniser,
        filter_str=args.filter_str,
        num_eval_search_pools=args.num_eval_sp,
        search_pool_size=args.sp_size,
    )

    eval_run.eval()

elif args.command == "vseval":
    eval_run = FaserVulnSearchEval(
        torch_model_bin_fp=args.model,
        eval_data_dir=args.eval_data,
        max_seq_len=args.input_dim,
        tokeniser_fp=args.tokeniser,
        model_friendly_name=args.model_friendly_name,
    )

    eval_run.rank()


import argparse
import re
from multiprocessing import Lock, Process

from bot_factory import BotFactory
from data_storage import DataStorage
from notifier import CHECK_FREQUENCY, start_notifier
from sheets_api import SheetsApi
from tabulate import tabulate
from utils import formatted_link, formatted_link_from_id, id_from_link

def parse_args():
    parser = argparse.ArgumentParser()
    # parser.add_argument("config_path", help="path to the configuration file")
    parser.add_argument("token", help="telegram bot token")
    parser.add_argument("api_key", help="google sheets api key")
    parser.add_argument("db_path", help="notifiers database filepath")
    args = parser.parse_args()
    return args

if __name__ == "__main__":

    launch_args = parse_args()

    bot_factory = BotFactory(launch_args.token)
    bot = bot_factory.create_bot()

    db_path = launch_args.db_path
    db_storage = DataStorage(db_path)
    db_lock = Lock()

    sheets_service = SheetsApi(launch_args.api_key)

    @bot.message_handler(commands=["set"])
    def set_command(message):
        text = message.text
        args_patterns = ['/set "[^"]*"', ' [^" ]*', ' "[^"]*"']
        args = []

        for pattern in args_patterns:
            enter = re.match(pattern, text)
            if enter is None:
                bot.reply_to(message, text="Wrong format")
                return

            enter = enter.group(0)
            args.append(enter)
            text = text[len(enter) :]

        name = args[0].split('"')[-2]
        sheet_id = args[1][1:]
        cell_range = args[2][2:-1]

        if len(sheet_id) > 44:  # interpret as an url
            sheet_id = id_from_link(sheet_id)
            if sheet_id is None:
                bot.reply_to(message, "Invalid URL")
                return

        chat_id = message.chat.id

        db_lock.acquire()

        with_same_props = db_storage.find_by_props(chat_id, sheet_id, cell_range)
        if with_same_props is not None:
            if with_same_props != name:
                message_text = (
                    f"The notifier '{with_same_props}' has the same properties"
                )
            else:
                message_text = "Already set"
            bot.reply_to(message, message_text)
            db_lock.release()
            return
        if db_storage.find_by_name(chat_id, name):
            bot.reply_to(message, "A notifier with this name is already set")
            db_lock.release()
            return

        cur_value_hash = sheets_service.get_range_hash(sheet_id, cell_range)
        if cur_value_hash is None:
            message_text = (
                f"An error occurred trying to set the {name} "
                f"notifier. \n"
                f"Check the Sheet ID and range format. "
                f"Probably the {formatted_link_from_id(sheet_id, 'table')} is "
                f"private or doesn't exist. "
            )
            bot.send_message(
                chat_id=chat_id,
                text=message_text,
                parse_mode="HTML",
                disable_web_page_preview=True,
            )
            db_lock.release()
            return
        # add new notifier
        db_storage.add_record(
            chat_id,
            name,
            sheet_id,
            cell_range,
            cur_value_hash,
        )
        bot.reply_to(message, text="Successfully set")
        db_lock.release()

    @bot.message_handler(commands=["list"])
    def list_command(message):
        db_lock.acquire()

        records = db_storage.records_by_user(message.chat.id)

        headers = ["Name", "Cell Range"]

        data = [[rec["name"], rec["range"]] for rec in records]
        tabulated_text = tabulate(data, headers=headers)

        message_text = ""

        for line, sheet_id in zip(
            tabulated_text.split("\n"), ["", ""] + [rec["sheet_id"] for rec in records]
        ):
            line = "
" + line + "
"
            link = ""
            if sheet_id != "":
                link = formatted_link_from_id(sheet_id)
            message_text += line + "    " + link + "\n"

        bot.send_message(
            chat_id=message.chat.id,
            text=message_text,
            parse_mode="HTML",
            disable_web_page_preview=True,
        )
        db_lock.release()

    @bot.message_handler(commands=["del"])
    def del_command(message):
        if len(message.text) < 6:
            bot.reply_to(message, text="Wrong format")
            return

        db_lock.acquire()

        name = message.text[5:]
        if db_storage.delete_with_name(message.chat.id, name):
            message_text = f"The notifier {name} has been deleted"
        else:
            message_text = f"The notifier {name} not found"
        bot.reply_to(message, text=message_text, parse_mode="HTML")
        db_lock.release()

    @bot.message_handler(commands=["help"])
    def help_command(message):
        a1_examples_link = (
            "https://developers.google.com/sheets/api/guides/concepts#expandable-1"
        )
        r1c1_examples_link = (
            "https://developers.google.com/sheets/api/guides/concepts#expandable-2"
        )

        message_text = (
            "To set a notifier enter a message in the following format:\n"
            '
 /set "{name}" {google_sheet_id | table_url} "{cell range}"
\n'
            f"cell range format - {formatted_link(a1_examples_link, 'A1')} "
            f"or {formatted_link(r1c1_examples_link, 'R1C1')} notation\n"
            "for example:\n"
            '
 /set "test results" 17h7GKuJ1gS7faOiyM7dBaM1XPUHPXPgIb7zCFfFGUOE '
            '"My Sheet!A1:A2"
\n'
            '
 /set "test results" '
            "https://docs.google.com/spreadsheets/d/17h7GKuJ1gS7faOiyM7dBaM1XPUHPXPgIb7zCFfFGUOE "
            '"My Sheet!A1:A2"
\n\n'
            ""
            "To delete a notifier use\n"
            "
 /del {name}
\n"
            "for example:\n"
            "
 /del test results
"
        )
        bot.send_message(
            chat_id=message.chat.id,
            text=message_text,
            parse_mode="HTML",
            disable_web_page_preview=True,
        )

    @bot.message_handler(commands=["check_frequency"])
    def check_freq_command(message):
        bot.send_message(
            chat_id=message.chat.id,
            text=f"Current update frequency is {CHECK_FREQUENCY} seconds",
        )

    notifier_process = Process(
        target=start_notifier, args=(bot_factory, db_path, db_lock, launch_args.api_key)
    )
    notifier_process.start()

    print(f"Bot started with token{launch_args.token}")
    bot.infinity_polling()