from django.shortcuts import render_to_response
from core.utils import *
import urllib, urllib2
import libxml2, csv, os
#from sets import Set
from django.db import connection
from datetime import datetime

monthName = ['', 'January', 'February', 'March', 'April',
             'May', 'June', 'July', 'August', 'September',
             'October', 'November', 'December']

def index(request):
    return viewNormal(request)

def viewNormal(request):
    data = getData()
    data['c5report_active'] = 1
    return render_to_response('c5report.html', data)

def viewMobile(request):
    data = getData()
    return render_to_response('c5report.mobile.html', data)

# Not used at the moment
def viewMobileManifest(request):
    response = render_to_response('c5report.mobile.manifest')
    response['Content-Type'] = 'text/cache-manifest'
    return response

def viewText(request):
    data = getData()
    response = render_to_response('c5report.txt', data)
    response['Content-Type'] = 'text/plain'
    response['Content-Description'] = 'C5 Report generated by GStat 2.0'
    return response

def viewXml(request):
    data = getData()
    response = render_to_response('c5report.xml', data)
    response['Content-Type'] = 'application/xml'
    response['Content-Description'] = 'C5 Report generated by GStat 2.0'
    return response

def prettyPrintNumber(number):
    len_number = len(number)
    key = 9; value = 'B';
    if len_number > key:
        return number[0:len_number-key] + '.' + number[len_number-key:len_number-key+1] + value
    key = 6; value = 'M';
    if len_number > key:
        return number[0:len_number-key] + '.' + number[len_number-key:len_number-key+1] + value
    key = 3; value = 'K';
    if len_number > key:
        return number[0:len_number-key] + '.' + number[len_number-key:len_number-key+1] + value
    return number

def getJobsPerMonth(startMonth, startYear, endMonth, endYear):
    urlquery = 'http://www3.egee.cesga.es/gridsite/accounting/CESGA/csv_export_extend.php'
    params = {'startYear'   : startYear,
              'startMonth'  : startMonth,
              'endYear'     : endYear,
              'endMonth'    : endMonth,
              'yrange'      : 'REGION',
              'xrange'      : 'VO',
              'option'      : 'REGION',
              'optval'      : '',
              'type'        : 'Production',
              'dteamVO'     : '0',
              'query'       : 'njobs',
              'voGroup'     : 'lhc',
              'dataGroup'   : 'Production',
              'newCateg'    : '0',
              'isextend'    : '0'}
    response = urllib2.urlopen(urlquery, urllib.urlencode(params)).read()
    rows = csv.reader(response.split(os.linesep), delimiter=',', quoting=False)

    # Skip initial comments
    for i in range(0,5):
        rows.next()

    # Get the total number of jobs
    for row in rows:
        if row[0] == 'Total':
            return row[5]

    return False

def getData():
    # Initialization
    cursor = connection.cursor()
    # SAM DB API
    urlquery = 'http://lcg-sam.cern.ch:8080/reports/lcg_deployment_C5.xsql'
    response = urllib2.urlopen(urlquery).read()
    doc = libxml2.parseDoc(response)
    # Summary OSes
    #urlquery2 = 'http://horat.web.cern.ch/horat/summary-oses.xml'
    #response2 = urllib2.urlopen(urlquery2).read()
    #doc2 = libxml2.parseDoc(response2)
    # Sites in GStat 2.0
    site_list = get_sites('GRID', 'ALL')
    # Month names for easy access. E.g.: monthName[1] == 'January'
    monthName = ['', 'January', 'February', 'March', 'April',
                 'May', 'June', 'July', 'August',
                 'September', 'October', 'November', 'December']

    # 1.a Total number of Sites
    # Extracted from GStat 2.0 or SAM API
    # GStat 2.0 way: total_number_sites = len(site_list)
    total_number_sites = doc.xpathEval("/LCG_DEPLOYMENT/TOTAL_NUM_EGEE_SITES_CERT_PROD/VALUE/TOTAL_SITES")[0].content

    # 1.b Status (ok, degraded, down, maintenance, not available)
    # Extracted from SAM API
    # http://lcg-sam.cern.ch:8080/reports/lcg_deployment_C5.xsql
    sites_ok = doc.xpathEval("/LCG_DEPLOYMENT/TOTAL_EGEE_SITES_STATUS_CERT_PROD/VALUE[NAME='ok']/TOTAL")[0].content
    sites_degraded = doc.xpathEval("/LCG_DEPLOYMENT/TOTAL_EGEE_SITES_STATUS_CERT_PROD/VALUE[NAME='degraded']/TOTAL")[0].content
    sites_degraded += doc.xpathEval("/LCG_DEPLOYMENT/TOTAL_EGEE_SITES_STATUS_CERT_PROD/VALUE[NAME='degrated80']/TOTAL")[0].content
    sites_down = doc.xpathEval("/LCG_DEPLOYMENT/TOTAL_EGEE_SITES_STATUS_CERT_PROD/VALUE[NAME='down']/TOTAL")[0].content
    sites_maint = doc.xpathEval("/LCG_DEPLOYMENT/TOTAL_EGEE_SITES_STATUS_CERT_PROD/VALUE[NAME='maint']/TOTAL")[0].content
    sites_na = doc.xpathEval("/LCG_DEPLOYMENT/TOTAL_EGEE_SITES_STATUS_CERT_PROD/VALUE[NAME='na']/TOTAL")[0].content

    # 2. Numer of sites per software (glite 3.2 and glite 3.1)
    # Extracted from SAM API
    # http://lcg-sam.cern.ch:8080/reports/lcg_deployment_C5.xsql
    sites_glite32 = doc.xpathEval("/LCG_DEPLOYMENT/GLITE_VERSION_IN_SITES/VALUES[VERSION='3.2.0']/NUMBER_OF_SITES")[0].content
    sites_glite31 = doc.xpathEval("/LCG_DEPLOYMENT/GLITE_VERSION_IN_SITES/VALUES[VERSION='3.1.0']/NUMBER_OF_SITES")[0].content

    # 3.a Number of jobs executed last month
    # 3.b Number of jobs executed since the beginning of this month
    # Extracted from accounting portal: http://www3.egee.cesga.es/
    now = datetime.now()
    thisYear = now.year
    thisMonth = now.month
    jobs_this_month = getJobsPerMonth(thisMonth, thisYear, thisMonth, thisYear)
    jobs_this_month = prettyPrintNumber(jobs_this_month)
    jobs_this_month_monthName = monthName[thisMonth]
    jobs_this_month_monthNumber = thisMonth
    jobs_this_month_yearNumber = thisYear

    if thisMonth == 1:
        lastYear = thisYear - 1
        lastMonth = 12
    else:
        lastYear = thisYear
        lastMonth = thisMonth - 1
    jobs_last_month = getJobsPerMonth(lastMonth, lastYear, lastMonth, lastYear)
    jobs_last_month = prettyPrintNumber(jobs_last_month)
    jobs_last_month_monthName = monthName[lastMonth]
    jobs_last_month_monthNumber = lastMonth
    jobs_last_month_yearNumber = lastYear

    # 4. Number of CREAM CEs unique hosts deployed
    cursor.execute("SELECT count(DISTINCT ce.hostname) FROM glue_gluece ce WHERE ce.implementationname = 'CREAM';")
    creamces_unique_hosts = int(cursor.fetchall()[0][0])

    # 5. Number of LCG-CEs unique hosts deployed
    cursor.execute("SELECT count(DISTINCT ce.hostname) FROM glue_gluece ce WHERE ce.implementationname = 'LCG-CE';")
    lcgces_unique_hosts = int(cursor.fetchall()[0][0])

    # 6. - Number of sites supporting CREAM CEs
    cursor.execute("SELECT COUNT(DISTINCT cluster.gluesite_fk) FROM glue_gluece ce, glue_gluecluster cluster, glue_gluesite site WHERE ce.implementationname='CREAM' and ce.gluecluster_fk = cluster.uniqueid and cluster.gluesite_fk = site.uniqueid;")
    sites_creamces = int(cursor.fetchall()[0][0])

    # 7. - Number of sites supporting LCG-CEs
    cursor = connection.cursor()
    cursor.execute("SELECT COUNT(DISTINCT cluster.gluesite_fk) FROM glue_gluece ce, glue_gluecluster cluster, glue_gluesite site WHERE ce.implementationname='LCG-CE' and ce.gluecluster_fk = cluster.uniqueid and cluster.gluesite_fk = site.uniqueid;")
    sites_lcgces = int(cursor.fetchall()[0][0])

    # 8 Number of sites supporting MPI
    # Before, extracted from SAM API:
    # http://lcg-sam.cern.ch:8080/reports/lcg_deployment_C5.xsql
    #ces_mpi = doc.xpathEval("/LCG_DEPLOYMENT/LCG_CEs_and_CREAM_CEs_SUPPORTING_MPI/VALUE/TOTAL_CES")[0].content
    # Extracted from GStat 2.0
    cursor.execute("SELECT COUNT(*) FROM glue_gluemultivalued mv, glue_gluecluster cluster, glue_gluesite site WHERE mv.attribute = 'GlueHostApplicationSoftwareRunTimeEnvironment' and mv.value = 'MPI-START' and mv.uniqueid = cluster.uniqueid and cluster.gluesite_fk = site.uniqueid;")
    sites_mpi = int(cursor.fetchall()[0][0])

    # 9 Number of logical CPUs supporting MPI
    cursor.execute("SELECT SUM(subcluster.logicalcpus) FROM glue_gluemultivalued mv, glue_gluesubcluster subcluster WHERE mv.attribute = 'GlueHostApplicationSoftwareRunTimeEnvironment' and mv.value = 'MPI-START' and mv.uniqueid = subcluster.uniqueid;")
    logical_cpus_mpi = int(cursor.fetchall()[0][0])

    # 10. Installed Capacity per OS
    # Check with: ldapsearch -LLL -x -h lcg-bdii -p 2170 -b o=grid '(&(objectClass=GlueSubCluster)(GlueHostOperatingSystemName=*AIX*))'
    temp = {}
    cursor.execute("SELECT operatingsystemname, operatingsystemrelease, logicalcpus, benchmarksi00 FROM glue_gluesubcluster;")
    normalOS = ['AIX', 'Debian', 'Ubuntu', 'SUSE LINUX']
    rhelOS = ['Scientific Linux', 'Scientific Linux CERN', 'ScientificCERNSLC',
              'CentOS', 'ScientificSL', 'ScientificFermiLTS',
              'RedHatEnterpriseAS', 'RedHatEnterpriseWS', 'ScientificSLF']

    for row in cursor.fetchall():
        os = row[0]
        ver = row[1]
        if ver.find('.') != -1:
            ver = ver[0:ver.find('.')]
        logicalcpus = int(row[2])
        si2000 = logicalcpus * int(row[3])

        if os in normalOS:
            key = os + ' ' + ver
        elif os in rhelOS and ver != '':
            key = 'RHEL ' + ver + ' Compat'
        else:
            key = 'Unknown'

        if temp.has_key(key):
            tempkey = temp[key]
            tempkey[0] += 1
            tempkey[1] += logicalcpus
            tempkey[2] += si2000
        else:
            temp[key] = [1, logicalcpus, si2000]

    ic_per_os = []
    for key, value in temp.items():
        temp = [key]
        temp.extend(value)
        ic_per_os.append(temp)
    ic_per_os.sort(lambda x, y: cmp(x[0],y[0]))

    mySubClusters = 0
    myLogicalCPUs = 0
    mySI2000 = 0
    for temp in ic_per_os:
        mySubClusters += temp[1]
        myLogicalCPUs += temp[2]
        mySI2000 += temp[3]
    ic_per_os.append(['Total', mySubClusters, myLogicalCPUs, mySI2000])

    # 11. Installed Capacity per CE Type
    cursor.execute("SELECT SUM(a.logicalcpus) total_cpus FROM (SELECT subcluster.logicalcpus FROM glue_gluece ce, glue_gluecluster cluster, glue_gluesubcluster subcluster WHERE ce.implementationname = 'CREAM' and ce.gluecluster_fk = cluster.uniqueid and subcluster.gluecluster_fk = cluster.uniqueid GROUP BY subcluster.uniqueid) a;")
    temp = cursor.fetchall()[0]
    ic_per_ce = [['CREAM', int(temp[0])]]
    cursor.execute("SELECT SUM(a.logicalcpus) total_cpus FROM (SELECT subcluster.logicalcpus FROM glue_gluece ce, glue_gluecluster cluster, glue_gluesubcluster subcluster WHERE ce.implementationname = 'LCG-CE' and ce.gluecluster_fk = cluster.uniqueid and subcluster.gluecluster_fk = cluster.uniqueid GROUP BY subcluster.uniqueid) a;")
    temp = cursor.fetchall()[0]
    ic_per_ce.append(['LCG-CE', int(temp[0])])

    return {'date': datetime.utcnow().strftime("%Y-%m-%d %H:%MZ"),
            'total_number_sites': total_number_sites,
            'sites_ok': sites_ok,
            'sites_degraded': sites_degraded,
            'sites_down': sites_down,
            'sites_maint': sites_maint,
            'sites_na': sites_na,
            'sites_glite32': sites_glite32,
            'sites_glite31': sites_glite31,
            'jobs_this_month': jobs_this_month,
            'jobs_this_month_monthName': jobs_this_month_monthName,
            'jobs_this_month_monthNumber': jobs_this_month_monthNumber,
            'jobs_this_month_yearNumber': jobs_this_month_yearNumber,
            'jobs_last_month': jobs_last_month,
            'jobs_last_month_monthName': jobs_last_month_monthName,
            'jobs_last_month_monthNumber': jobs_last_month_monthNumber,
            'jobs_last_month_yearNumber': jobs_last_month_yearNumber,
            'creamces_unique_hosts': creamces_unique_hosts,
            'lcgces_unique_hosts': lcgces_unique_hosts,
            'sites_creamces': sites_creamces,
            'sites_lcgces': sites_lcgces,
            'sites_mpi': sites_mpi,
            'logical_cpus_mpi': logical_cpus_mpi,
            'ic_per_os': ic_per_os,
            'ic_per_ce': ic_per_ce}


import os
from typing import Any, Optional, List, Dict

import gspread
import pandas as pd
from oauth2client.service_account import ServiceAccountCredentials
import socket
from datetime import datetime
from time import sleep
import traceback

_current_experiment = None  # type: Optional[ExperimentParams]

def _experiment_running():
    return _current_experiment is not None

def log_experiment_id(experiment_id: str):
    """
    Sets experiment id
    :param experiment_id:
    :return:
    """
    if _experiment_running():
        _current_experiment.log_experiment_id(experiment_id)

def log_status(status: str):
    """
    Sets status of the current experiment
    :param status:
    """
    if _experiment_running():
        _current_experiment.log_status(status)

def log_comment(text: str):
    """
    Sets comment of the current experiment
    """
    if _experiment_running():
        _current_experiment.log_comment(text)

def log_progress(current: int, max_value: int):
    """
    Sets progress of the current experiment
    """
    if _experiment_running():
        _current_experiment.log_progress(current, max_value)

def log_metric(metric: str, value: Any):
    """
    Logs metric of the current experiment
    :param metric: metric name
    :param value: metric value
    """
    if _experiment_running():
        _current_experiment.log_metric(metric, value)

class ParamsException(Exception):
    pass

class ParamsSheet(object):
    def __init__(self, parser, client_credentials: str, params_sheet_id = None,
                 writable_column_types: Dict[str, Any]=None, experiment_id_column: str='experiment_id',
                 server_name: str=None, fill_empty_with_defaults=True):
        """
        ParamsSheet object represents state of google spreadsheet with parameters and all methods to access it.
        :param parser: Argparse parser object with no positional arguments
        :param writable_column_types: dict of names of model metric columns and their types.
        If it is None, writing unknown metrics won't generate warnings.
        :param experiment_id_column: str name of experiment id column.
        """
        self.client_credentials = client_credentials
        self.params_sheet_id = params_sheet_id
        self.experiment_id_column = experiment_id_column
        self.fill_empty_with_defaults = fill_empty_with_defaults
        self.parser = parser
        self.defaults = {
            _action.dest: _action.default
            for _action in self.parser._actions[1:]
        }

        # Setting up dictionaries holding column types
        self.metric_column_types = writable_column_types
        if self.metric_column_types is None:
            self.column_types = {}
        else:
            assert isinstance(self.metric_column_types, dict), "writable_column_types has to be a dict"
            self.column_types = writable_column_types.copy()
        self.column_types.update({
            _action.dest: _action.type
            for _action in self.parser._actions[1:]
        })

        # Getting server name
        if server_name is None:
            self.server = os.environ.get('SERVERNAME', None)
            if self.server is None:
                self.server = socket.gethostname()
        else:
            self.server = server_name + "_" + socket.gethostname()

        # Getting credentials
        self._generate_credentials()

    def update_cell(self, row, column_name, value):
        col = self.columns.index(column_name) + 1
        try:
            self.sheet.update_cell(row, col, value)
        except gspread.exceptions.APIError:
            self._generate_credentials()
            self.sheet.update_cell(row, col, value)

    def exec_loop(self, train_loop):
        """
        :param train_loop: train function that takes args namespace
        :return: None
        """
        global _current_experiment
        print("Starting worker...")
        timeout_power = 1
        while True:
            exp_params = None
            try:
                exp_params = self._get_params_for_training()
                _current_experiment = exp_params
                timeout_power = 4
                exp_params.log_status('running')
                exp_params.log_server()
                exp_params.log_time_to_column('time_started')
                train_loop(exp_params.args)
                exp_params.log_status('finished')
            except ParamsException:
                print("No params to process, waiting for {} seconds and checking again".format(2 ** timeout_power))
                sleep(2 ** timeout_power)
                if timeout_power <= 10:
                    timeout_power += 1
            except KeyboardInterrupt:
                if exp_params is not None:
                    exp_params.log_status('stopped')
                _current_experiment = None
            except Exception as e:
                tb = traceback.format_exc()
                print(tb)
                if exp_params is not None:
                    exp_params.log_comment(tb)
                    exp_params.log_status('error')
                _current_experiment = None

    def _generate_credentials(self):
        if isinstance(self.client_credentials, dict):
            credentials = ServiceAccountCredentials._from_parsed_json_keyfile(
                self.client_credentials, ['https://spreadsheets.google.com/feeds']
            )
        else:
            credentials = ServiceAccountCredentials.from_json_keyfile_name(
                self.client_credentials,['https://spreadsheets.google.com/feeds']
            )
        gc = gspread.authorize(credentials)
        self.sheet = gc.open_by_key(self.params_sheet_id).sheet1
        first_cell = self.sheet.cell(1, 1).value
        try:
            self.column_row_id = int(first_cell)
        except ValueError:
            self.column_row_id = 1
        self.columns = self.sheet.row_values(self.column_row_id)

    def _get_params_for_training(self) -> "ExperimentParams":
        """
        This function gets next experiment parameter settings
        """
        try:
            all_spreadsheet = pd.DataFrame(self.sheet.get_all_records(head=self.column_row_id, default_blank=""))
        except gspread.exceptions.APIError:
            self._generate_credentials()
            all_spreadsheet = pd.DataFrame(self.sheet.get_all_records(head=self.column_row_id, default_blank=""))
        params_to_process = all_spreadsheet[
            (all_spreadsheet.status == "") & (all_spreadsheet.server == "")]
        params_for_this_server = all_spreadsheet[
            (all_spreadsheet.status == "") & (all_spreadsheet.server == self.server)]
        if len(params_for_this_server):
            params_to_process = params_for_this_server
        if len(params_to_process) == 0:
            raise ParamsException("No params to process!")
        return ExperimentParams(
            params_to_process.index[0] + 1 + self.column_row_id,
            dict(params_to_process.iloc[0]),
            self, self.fill_empty_with_defaults)

class ExperimentParams(object):
    _system_columns = ('time_started', 'last_update', 'progress_bar', 'server', 'status', 'comment')

    def __init__(self, row_id, params, _params_sheet: ParamsSheet, fill_empty_with_defaults: bool):
        self._params_sheet = _params_sheet
        self._params_row_id = row_id
        self.args = _params_sheet.parser.parse_args()
        parameter_columns = set(_params_sheet.columns) - set(self._system_columns) - {_params_sheet.experiment_id_column}
        for _column_name in parameter_columns:
            # Skipping metric columns
            if _params_sheet.metric_column_types is not None and _column_name in _params_sheet.metric_column_types:
                continue

            parameter_value = params[_column_name]
            if isinstance(params[_column_name], str) and (params[_column_name] == ""):
                # This parameter field is empty
                if fill_empty_with_defaults:
                    parameter_value = getattr(self.args, _column_name, '')
                    self._params_sheet.update_cell(
                        self._params_row_id, _column_name,
                        parameter_value
                    )

            if _column_name not in _params_sheet.column_types:
                print("Unexpected column %s (known column: %s)" % (_column_name, ",".join(_params_sheet.column_types)))
                continue

            dtype = _params_sheet.column_types[_column_name]
            if dtype is None:  # type for bool is None
                dtype = lambda x: x.lower() == 'true'
            setattr(self.args, _column_name, dtype(parameter_value))

    def log_server(self):
        self._write_to_field('server', self._params_sheet.server)

    def log_time_to_column(self, column_name):
        now = datetime.now()
        self._write_to_field(column_name, now.strftime('%Y-%m-%d %H:%M:%S'))

    def log_status(self, status):
        self._write_to_field('status', status)

    def log_comment(self, text):
        self._write_to_field('comment', text)

    def log_progress(self, i, max_i):
        assert isinstance(i, int) and isinstance(max_i, int)
        if i == max_i:
            color = "green"
        else:
            color = "blue"
        formula = '=SPARKLINE({},{{"charttype","bar";"color1", "{}";"max",{}}})'.format(i, color, max_i)
        self._write_to_field('progress_bar', formula)
        self.log_time_to_column('last_update')

    def log_experiment_id(self, value):
        self._write_to_field(self._params_sheet.experiment_id_column, value)

    def log_metric(self, metric, value):
        if (self._params_sheet.metric_column_types is not None) and \
                (metric not in self._params_sheet.metric_column_types):
            print("Logging unknown metric %s" % metric)
        self._write_to_field(metric, value)
        self.log_time_to_column('last_update')

    def _write_to_field(self, column_name, value):
        self._params_sheet.update_cell(
            self._params_row_id,
            column_name,
            value)


# -*- coding: UTF-8 -*-
from boto.sdb.db.property import StringProperty, IntegerProperty
from sdb_orm.model import Model
from sdb_orm.property import AutoIDProperty, BlobProperty

# 宣告游戏地图资料模型
class GameMap(Model):
    # 资料表名称
    _db_name = 'game-maps'
    # 预设使用一致性读取，不然有可能会读到不是最新的资料，因为sdb是分散式数据库
    __consistent__ = True

    # 自动流水号，此字段不可设定，只会自动捞资料表中最大值+1
    # bound是流水号可能值的上限，只可以小于它，这边设为10的8次方
    mid = AutoIDProperty(bound=10**8)
    name = StringProperty(required=True) # 地图名称，字串字段，必要值
    width = IntegerProperty(default=None, required=True, min=1) # 宽，整数字段
    height = IntegerProperty(default=None, required=True, min=1)
    # 因sdb有字段值长度不可超过1024的限制，这边把资料存到Amazon s3变成一个档案
    # 然后在sdb的字段里存路径，这个属性经过包装，可以当字串直接读写
    # 传入auto_id作为自动产生档名的依据
    grid = BlobProperty(required=True, auto_id=mid)


import requests
from bs4 import BeautifulSoup

class pesos_automation():

    def start_script(self):
        print("start pesos script")
        try:
            URL = 'https://wise.com/es/currency-converter/usd-to-cop-rate'
            page = requests.get(URL)
            soup = BeautifulSoup(page.content, 'html.parser')
            dollarContainer = soup.find("span", {"class": "text-success"})
            print("pesos value",dollarContainer.text.split(',')[0])
            return dollarContainer.text.split(',')[0]
        except Exception as e:
            print("Could not load site: "+str(e))
            return None


# Note: you need to be using OpenAI Python v0.27.0 for the code below to work

import os
import openai
openai.organization = "org-DZn6MOgACWIbxxYLzuk2MdHz" #填你自己的OpenAI-Organization，这里获得：https://platform.openai.com/account/org-settings
openai.api_key = os.getenv("OPENAI_API_KEY") #填你自己的api_key，这里获得：https://platform.openai.com/account/api-keys
openai.Model.list()

map=[]
user = {"role": "user", "content": ""}
system = {"role": "system", "content": ""}
user["content"] = input('请问您想了解什么？\n')

while True:
    map.append(user)
    response = openai.ChatCompletion.create(
        model="gpt-3.5-turbo",
        messages = map
    )

    re = response['choices'][0]['message']['content']

    re.encode('unicode_escape').decode('utf-8')

    print(re + "\n")

    system["content"] = re

    map.append(system)

    user = {"role": "user", "content": ""}
    system = {"role": "system", "content": ""}

    user["content"] = input()


import requests as rq
import pandas as pd
import numpy as np
import time

def get_apy(apr):
    """APY assuming compounding daily"""
    return ((1+(apr/365))**(365))-1

def get_cost(price, set):
    """Get cost from current price and amount needed to get in the set"""
    return price*set

def get_monthly_income(p, apy):
    """calc month income from current price and the apy"""
    return (p*apy)/12

def fetch():
    juno_price = rq.get(f"https://api.coingecko.com/api/v3/simple/price?ids=juno-network&vs_currencies=usd").json()['juno-network']['usd']
    osmo_price = rq.get(f"https://api.coingecko.com/api/v3/simple/price?ids=osmosis&vs_currencies=usd").json()['osmosis']['usd']
    cosmos_price = rq.get(f"https://api.coingecko.com/api/v3/simple/price?ids=cosmos&vs_currencies=usd").json()['cosmos']['usd']
    evmos_price = rq.get(f"https://api.coingecko.com/api/v3/simple/price?ids=evmos&vs_currencies=usd").json()['evmos']['usd']
    akash_price = rq.get(f"https://api.coingecko.com/api/v3/simple/price?ids=akash-network&vs_currencies=usd").json()['akash-network']['usd']
    stargaze_price = rq.get(f"https://api.coingecko.com/api/v3/simple/price?ids=stargaze&vs_currencies=usd").json()['stargaze']['usd']
    umee_price = rq.get(f"https://api.coingecko.com/api/v3/simple/price?ids=umee&vs_currencies=usd").json()['umee']['usd']
    secret_price = rq.get(f"https://api.coingecko.com/api/v3/simple/price?ids=secret&vs_currencies=usd").json()['secret']['usd']
    terra_price = rq.get(f"https://api.coingecko.com/api/v3/simple/price?ids=terra-luna-2&vs_currencies=usd").json()['terra-luna-2']['usd']
    injective_price = rq.get(f"https://api.coingecko.com/api/v3/simple/price?ids=injective-protocol&vs_currencies=usd").json()['injective-protocol']['usd']
    sifchain_price = rq.get(f"https://api.coingecko.com/api/v3/simple/price?ids=sifchain&vs_currencies=usd").json()['sifchain']['usd']
    kava_price = rq.get(f"https://api.coingecko.com/api/v3/simple/price?ids=kava&vs_currencies=usd").json()['kava']['usd']
    comdex_price = rq.get(f"https://api.coingecko.com/api/v3/simple/price?ids=comdex&vs_currencies=usd").json()['comdex']['usd']
    cryptoorgchain_price = rq.get(f"https://api.coingecko.com/api/v3/simple/price?ids=crypto-com-chain&vs_currencies=usd").json()['crypto-com-chain']['usd']
    persistence_price = rq.get(f"https://api.coingecko.com/api/v3/simple/price?ids=persistence&vs_currencies=usd").json()['persistence']['usd']

    #minimum amount needed to get in active set https://www.mintscan.io/
    # assets = ['juno', 'osmosis', 'cosmoshub', 'evmos','akash', 'stargaze','umee', 'secretnetwork', 'terra']
    juno_val_set = rq.get("https://validators.cosmos.directory/chains/juno").json()
    juno_val_set_df = pd.DataFrame(juno_val_set['validators'])
    juno_actv_val_set_df = juno_val_set_df[juno_val_set_df['status']=='BOND_STATUS_BONDED'].sort_values(by=['rank'], ascending=True)
    juno_set = float(juno_actv_val_set_df.tail(1)['delegator_shares'])/1000000

    osmosis_val_set = rq.get("https://validators.cosmos.directory/chains/osmosis").json()
    osmosis_val_set_df = pd.DataFrame(osmosis_val_set['validators'])
    osmosis_actv_val_set_df = osmosis_val_set_df[osmosis_val_set_df['status']=='BOND_STATUS_BONDED'].sort_values(by=['rank'], ascending=True)
    osmosis_set = float(osmosis_actv_val_set_df.tail(1)['delegator_shares'])/1000000

    cosmoshub_val_set = rq.get("https://validators.cosmos.directory/chains/cosmoshub").json()
    cosmoshub_val_set_df = pd.DataFrame(cosmoshub_val_set['validators'])
    cosmoshub_actv_val_set_df = cosmoshub_val_set_df[cosmoshub_val_set_df['status']=='BOND_STATUS_BONDED'].sort_values(by=['rank'], ascending=True)
    cosmoshub_set = float(cosmoshub_actv_val_set_df.tail(1)['delegator_shares'])/1000000

    evmos_val_set = rq.get("https://validators.cosmos.directory/chains/evmos").json()
    evmos_val_set_df = pd.DataFrame(evmos_val_set['validators'])
    evmos_actv_val_set_df = evmos_val_set_df[evmos_val_set_df['status']=='BOND_STATUS_BONDED'].sort_values(by=['rank'], ascending=True)
    evmos_set = float(evmos_actv_val_set_df.tail(1)['delegator_shares'])/1000000000000000000

    akash_val_set = rq.get("https://validators.cosmos.directory/chains/akash").json()
    akash_val_set_df = pd.DataFrame(akash_val_set['validators'])
    akash_actv_val_set_df = akash_val_set_df[akash_val_set_df['status']=='BOND_STATUS_BONDED'].sort_values(by=['rank'], ascending=True)
    akash_set = float(akash_actv_val_set_df.tail(1)['delegator_shares'])/1000000

    stargaze_val_set = rq.get("https://validators.cosmos.directory/chains/stargaze").json()
    stargaze_val_set_df = pd.DataFrame(stargaze_val_set['validators'])
    stargaze_actv_val_set_df = stargaze_val_set_df[stargaze_val_set_df['status']=='BOND_STATUS_BONDED'].sort_values(by=['rank'], ascending=True)
    stargaze_set = float(stargaze_actv_val_set_df.tail(1)['delegator_shares'])/1000000

    umee_val_set = rq.get("https://validators.cosmos.directory/chains/umee").json()
    umee_val_set_df = pd.DataFrame(umee_val_set['validators'])
    umee_actv_val_set_df = umee_val_set_df[umee_val_set_df['status']=='BOND_STATUS_BONDED'].sort_values(by=['rank'], ascending=True)
    umee_set = float(umee_actv_val_set_df.tail(1)['delegator_shares'])/1000000

    secretnetwork_val_set = rq.get("https://validators.cosmos.directory/chains/secretnetwork").json()
    secretnetwork_val_set_df = pd.DataFrame(secretnetwork_val_set['validators'])
    secretnetwork_actv_val_set_df = secretnetwork_val_set_df[secretnetwork_val_set_df['status']=='BOND_STATUS_BONDED'].sort_values(by=['rank'], ascending=True)
    secretnetwork_set = float(secretnetwork_actv_val_set_df.tail(1)['delegator_shares'])/1000000

    sifchain_val_set = rq.get("https://validators.cosmos.directory/chains/sifchain").json()
    sifchain_val_set_df = pd.DataFrame(sifchain_val_set['validators'])
    sifchain_actv_val_set_df = sifchain_val_set_df[sifchain_val_set_df['status']=='BOND_STATUS_BONDED'].sort_values(by=['rank'], ascending=True)
    sifchain_set = float(sifchain_actv_val_set_df.tail(1)['delegator_shares'])/1000000000000000000

    kava_val_set = rq.get("https://validators.cosmos.directory/chains/kava").json()
    kava_val_set_df = pd.DataFrame(kava_val_set['validators'])
    kava_actv_val_set_df = kava_val_set_df[kava_val_set_df['status']=='BOND_STATUS_BONDED'].sort_values(by=['rank'], ascending=True)
    kava_set = float(kava_actv_val_set_df.tail(1)['delegator_shares'])/1000000

    comdex_val_set = rq.get("https://validators.cosmos.directory/chains/comdex").json()
    comdex_val_set_df = pd.DataFrame(comdex_val_set['validators'])
    comdex_actv_val_set_df = comdex_val_set_df[comdex_val_set_df['status']=='BOND_STATUS_BONDED'].sort_values(by=['rank'], ascending=True)
    comdex_set = float(comdex_actv_val_set_df.tail(1)['delegator_shares'])/1000000

    cryptoorgchain_val_set = rq.get("https://validators.cosmos.directory/chains/cryptoorgchain").json()
    cryptoorgchain_val_set_df = pd.DataFrame(cryptoorgchain_val_set['validators'])
    cryptoorgchain_actv_val_set_df = cryptoorgchain_val_set_df[cryptoorgchain_val_set_df['status']=='BOND_STATUS_BONDED'].sort_values(by=['rank'], ascending=True)
    cryptoorgchain_set = float(cryptoorgchain_actv_val_set_df.iloc[-1]['delegator_shares'])/100000000

    persistence_val_set = rq.get("https://validators.cosmos.directory/chains/persistence").json()
    persistence_val_set_df = pd.DataFrame(persistence_val_set['validators'])
    persistence_actv_val_set_df = persistence_val_set_df[persistence_val_set_df['status']=='BOND_STATUS_BONDED'].sort_values(by=['rank'], ascending=True)
    persistence_set = float(persistence_actv_val_set_df.iloc[-1]['delegator_shares'])/1000000

    # terra_val_set = rq.get("https://validators.cosmos.directory/chains/terra").json()
    # terra_val_set_df = pd.DataFrame(terra_val_set['validators'])
    # terra_actv_val_set_df = terra_val_set_df[terra_val_set_df['status']=='BOND_STATUS_BONDED'].sort_values(by=['rank'], ascending=True)
    # terra_set = float(terra_actv_val_set_df.tail(1)['delegator_shares'])/1000000
    terra_set = 292900
    injective_set = 10009

    #inflation/bonded rate
    #query infaltion rate
    #bonded rate is total supply / sum balance of all validators
    #staking aprs https://www.mintscan.io/
    juno_apr = .8009
    osmo_apr = .3401
    evmos_apr = 6.2943
    cosmos_apr = .1884
    akash_apr = .2157
    stargaze_apr = .6038
    umee_apr = .5157
    secret_apr = .2337
    terra_apr = .1431
    injective_apr = .1297
    sifchain_apr = 1.2937
    kava_apr = .2634
    comdex_apr = .4037
    cryptoorgchain_apr = .1453
    persistence_apr = .3393

    data = {
        "juno":{"price":juno_price, "set":juno_set, "apr":juno_apr},
        "osmo":{"price":osmo_price, "set":osmosis_set, "apr":osmo_apr},
        "evmos":{"price":evmos_price, "set":evmos_set, "apr":evmos_apr},
        "cosmos":{"price":cosmos_price, "set":cosmoshub_set, "apr":cosmos_apr},
        "akash":{"price":akash_price, "set":akash_set, "apr":akash_apr},
        "stargaze":{"price":stargaze_price, "set":stargaze_set, "apr":stargaze_apr},
        "umee":{"price":umee_price, "set":umee_set, "apr":umee_apr},
        "secret":{"price":secret_price, "set":secretnetwork_set, "apr":secret_apr},
        "terra":{"price":terra_price, "set":terra_set, "apr":terra_apr},
        "injective":{"price":injective_price, "set":injective_set, "apr":injective_apr},
        "sifchain":{"price":sifchain_price, "set":sifchain_set, "apr":sifchain_apr},
        "kava":{"price":kava_price, "set":kava_set, "apr":kava_apr},
        "comdex":{"price":comdex_price, "set":comdex_set, "apr":comdex_apr},
        "cryptoorgchain":{"price":cryptoorgchain_price, "set":cryptoorgchain_set, "apr":cryptoorgchain_apr},
        "persistence":{"price":persistence_price, "set":persistence_set, "apr":persistence_apr},
    }
    for i in data.keys():
        apy = get_apy(data[i]['apr'])
        p = get_cost(data[i]['price'], data[i]['set'])
        monthly_income = get_monthly_income(p,apy)

        data[i]['apy'] = apy
        data[i]['cost_to_enter_set'] = p
        data[i]['est_monthly_revenue'] = monthly_income
        data[i]['est_monthly_income_to_validator'] = monthly_income *.1
    return data

# v_data = pd.DataFrame(data)
# v_data.to_csv("data/validator_data")
    # print(f"""{i}
    # price: ${round(data[i]['price'],2)}
    # apy: {round(apy*100,2)}%
    # cost to get in the set: ${round(p,2)}
    # monthly income: ${round(monthly_income,2)}""")


import numpy as np
import matplotlib.pyplot as plt
# 导入线性回归模型
from sklearn.linear_model import LinearRegression

# 输入3个点的横坐标
X = [[1], [4], [3]]
# 输入3个点的从坐标
y = [3, 5, 3]
# 用线性模型拟合这三个点
lr = LinearRegression().fit(X, y)
z = np.linspace(0, 5, 20)
plt.scatter(X, y, s=80)
plt.plot(z, lr.predict(z.reshape(-1, 1)), c='k')
plt.title('Straight Line')
plt.show()

线性模型/线性模型的图形表示2.py

线性模型的图形表示2.py


import os
os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"]="0"

import torch
import pickle
from tqdm import tqdm
from datasets import load_dataset, load_metric
from transformers import BertTokenizer, BertForMaskedLM, DistilBertTokenizerFast, DistilBertForMaskedLM
from transformers import DataCollatorForLanguageModeling, TrainingArguments, Trainer

tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')
model = DistilBertForMaskedLM.from_pretrained('distilbert-base-uncased')

dataset = load_dataset("./Datasets/Arith")

def encode_with_truncation(examples):
    ab = [example.split('=')[0] + "= [MASK] [MASK] [MASK]" for example in examples["text"]]
    return tokenizer(ab, truncation=True, padding="max_length", max_length=512, return_special_tokens_mask=True)

train_dataset = dataset['train'].map(encode_with_truncation, batched=True)
train_dataset.set_format(type="torch", columns=['input_ids', 'attention_mask', 'special_tokens_mask'])

label_col = []
for idx, val in tqdm(enumerate(dataset['train'])):
    label_ids = tokenizer.encode(val['text'].split('=')[1][1:])[1:-1]
    mask_idx = (train_dataset[idx]['input_ids'] == 103).nonzero(as_tuple=True)[0]
    labels = [-100] * 512
    for i, v in enumerate(label_ids):
        labels[mask_idx[i]] = v
    label_col.append(labels)

train_dataset = train_dataset.add_column("labels", label_col)

training_args = TrainingArguments(
    output_dir='./Models/Arith1',
    overwrite_output_dir=True,
    num_train_epochs=50,
    per_device_train_batch_size=16,
    logging_steps=500,
    save_steps=1000,
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
    tokenizer=tokenizer,
)

trainer.train()


"""
Optimized State
^^^^^^^^^^^^^^^

..contents:: Table of Contents
    :backlinks: none
    :local:

Introduction
------------

This module contains functions can be monkey patched into
`ethereum.frontier.state` to make it use the optimized trie.
"""

from dataclasses import dataclass, field
from typing import Dict, cast

import ethereum.frontier.state
from ethereum.base_types import U256, Bytes
from ethereum.frontier.eth_types import EMPTY_ACCOUNT, Account, Address
from ethereum.frontier.state import state_root

from .trie import Trie, get_internal_key, trie_get, trie_set

@dataclass
class State:
    """
    Contains all information that is preserved between transactions.
    """

    _main_trie: Trie[Account] = field(
        default_factory=lambda: Trie(secured=True, default=EMPTY_ACCOUNT)
    )
    _storage_tries: Dict[Address, Trie[U256]] = field(default_factory=dict)

    def __eq__(self, other: object) -> bool:
        """
        Test for equality by comparing state roots.
        """
        if type(self) != type(other):
            return NotImplemented
        return state_root(
            cast(ethereum.frontier.state.State, self)
        ) == state_root(cast(ethereum.frontier.state.State, other))

def set_storage(
    state: State, address: Address, key: Bytes, value: U256
) -> None:
    """
    Set a value at a storage key on an account. Setting to `U256(0)` deletes
    the key.

    Parameters
    ----------
    state: `State`
        The state
    address : `Address`
        Address of the account.
    key : `Bytes`
        Key to set.
    value : `U256`
        Value to set at the key.
    """
    internal_address = get_internal_key(state._main_trie, address)
    trie = state._storage_tries.get(internal_address)
    if trie is None:
        trie = Trie(secured=True, default=U256(0))
        state._storage_tries[internal_address] = trie
    trie_set(trie, key, value)
    state._main_trie._dirty_set.add(internal_address)
    if trie._data == {}:
        del state._storage_tries[internal_address]

def get_storage(state: State, address: Address, key: Bytes) -> U256:
    """
    Get a value at a storage key on an account. Returns `U256(0)` if the
    storage key has not been set previously.

    Parameters
    ----------
    state: `State`
        The state
    address : `Address`
        Address of the account.
    key : `Bytes`
        Key to lookup.

    Returns
    -------
    value : `U256`
        Value at the key.
    """
    internal_address = get_internal_key(state._main_trie, address)
    trie = state._storage_tries.get(internal_address)
    if trie is None:
        return U256(0)

    value = trie_get(trie, key)

    assert isinstance(value, U256)
    return value


from pymongo import MongoClient
from pymongo import cursor
from Stock import Stock
from flask_login import UserMixin
import re
import os

class User(UserMixin):
    def __init__(self, name):
        self.name = name
    def get_id(self):
        return self.name
    @staticmethod
    def users_collection():
        myclient = MongoClient(f"mongodb+srv://{os.getenv('DB_NAME')}:{os.getenv('PASS')}@cluster0-twyu7.mongodb.net/test?retryWrites=true&w=majority")
        db = myclient["users_stats"]
        users_collection = db["users_collection"]
        return users_collection
    def stats_database(self):
        myclient = MongoClient(f"mongodb+srv://{os.getenv('DB_NAME')}:{os.getenv('PASS')}@cluster0-twyu7.mongodb.net/test?retryWrites=true&w=majority")
        db = myclient["users_stats"]
        user_stats = db[self.name]
        return user_stats

    def buy(self, company_name, quantity):
        if (quantity):
            quantity = int(quantity)
        else:
            quantity = 1
        stock_list = Stock.stock_list()
        user_stats = self.stats_database()
        users_collection = self.users_collection()
        cursor=users_collection.find_one({"user_name": self.name})
        current_balance = float(cursor.get("balance"))
        cost = quantity * float(stock_list.get(company_name))
        if current_balance < cost:
            return False
        current_balance = current_balance - cost
        users_collection.update_one({"user_name": self.name}, {"$set": {"balance": current_balance}})
        current_quantity = 0
        for cursor in user_stats.find({}):
            if company_name in cursor:
                current_quantity = cursor[company_name]
                break
        user_stats.update_one({company_name: current_quantity},
                              {"$set": {company_name: current_quantity + quantity}}, upsert=True)

        return True

    def get_balance(self):
        users_collection = self.users_collection()
        cursor=users_collection.find_one({"user_name": self.name})
        return cursor.get("balance")

    def sell(self, company_name, quantity):
        if quantity:
            quantity = int(quantity)
        else:
            quantity = 1

        stock_list = Stock.stock_list()
        user_stats = self.stats_database()
        for cursor in user_stats.find({}):
            if company_name in cursor:
                current_quantity = cursor[company_name]
                break
        if current_quantity < quantity:
            return False
        if current_quantity - quantity == 0:
            user_stats.delete_one({company_name: current_quantity})
        else:
            user_stats.update({company_name: current_quantity}, {company_name: current_quantity - quantity})

        users_collection = self.users_collection()
        cursor = users_collection.find_one({"user_name": self.name})
        current_balance = cursor.get("balance")

        prise = quantity * float(stock_list.get(company_name))
        current_balance = current_balance + prise
        users_collection.update_one({"user_name": self.name}, {"$set": {"balance": current_balance}})
        return True

    def get_stats(self):
        user_stats = self.stats_database()
        stats = {}
        for stat in user_stats.find({}):
            x = re.split(':|\,', (str(stat)))
            company_name = (x[2][2:-1])
            quantity = (x[3][1:-1])
            stats[company_name] = quantity
        return stats

    def get_share_value(self):
        stock_list = Stock.stock_list()
        stats = self.get_stats()
        return sum((float(stock_list.get(k, 0)) * int(v)) for k, v in zip(stats, stats.values()))


# basic python and ML Libraries
import os
import random
import numpy as np
import pandas as pd
import math
import time

# for ignoring warnings
import warnings
warnings.filterwarnings('ignore')

# We will be reading images using OpenCV
import cv2

# matplotlib for visualization
import matplotlib.pyplot as plt
import matplotlib.patches as patches

# torchvision libraries
import torch
import torchvision
from torchvision import transforms as torchtrans
from torchvision.models.detection.faster_rcnn import FastRCNNPredictor

# helper libraries
import utils
from coco_utils import get_coco_api_from_dataset
from coco_eval import CocoEvaluator

# for image augmentations
import albumentations as A
from albumentations.pytorch.transforms import ToTensorV2

##############
# Utils      #
##############
def plot_img_bbox(img, target):
  fig, a = plt.subplots(1,1)
  fig.set_size_inches(5,5)
  a.imshow(img)
  for box in (target['boxes']):
    box = box.cpu()

    x, y, width, height  = box[0], box[1], box[2]-box[0], box[3]-box[1]

    rect = patches.Rectangle(
      (x, y),
      width, height,
      linewidth = 2,
      edgecolor = 'r',
      facecolor = 'none'
    )

    a.add_patch(rect)
  plt.show()

##############
# Dataset    #
##############
class ConeImagesDataset(torch.utils.data.Dataset):
  def __init__(self, files_dir, width, height, transforms=None):
    self.transforms = transforms
    self.files_dir = files_dir
    self.height = height
    self.width = width

    self.imgs = [image for image in sorted(os.listdir(files_dir)) if image[-4:]=='.jpg']

    self.classes = ['background', 'cone']

  def __getitem__(self, idx):
    img_name = self.imgs[idx]
    image_path = os.path.join(self.files_dir, img_name)

    img = cv2.imread(image_path)
    img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB).astype(np.float32)
    img_res = cv2.resize(img_rgb, (self.width, self.height), cv2.INTER_AREA)
    img_res /= 255.0

    annot_filename = img_name[:-4] + '.txt'
    annot_file_path = os.path.join(self.files_dir, annot_filename)

    boxes = []
    labels = []

    wt = img.shape[1]
    ht = img.shape[0]

    # box coordinates for xml files are extracted and corrected for image size given
    with open(annot_file_path) as f:
      for line in f:
        labels.append(1)

        parsed = [float(x) for x in line.split(' ')]
        x_center = parsed[1]
        y_center = parsed[2]
        box_wt = parsed[3]
        box_ht = parsed[4]

        xmin = x_center - box_wt/2
        xmax = x_center + box_wt/2
        ymin = y_center - box_ht/2
        ymax = y_center + box_ht/2

        xmin_corr = int(xmin*self.width)
        xmax_corr = int(xmax*self.width)
        ymin_corr = int(ymin*self.height)
        ymax_corr = int(ymax*self.height)

        boxes.append([xmin_corr, ymin_corr, xmax_corr, ymax_corr])

    boxes = torch.as_tensor(boxes, dtype=torch.float32)
    area = (boxes[:, 3] - boxes[:, 1]) * (boxes[:, 2] - boxes[:, 0])

    iscrowd = torch.zeros((boxes.shape[0],), dtype=torch.int64)
    labels = torch.as_tensor(labels, dtype=torch.int64)

    target = {}
    target["boxes"] = boxes
    target["labels"] = labels
    target["area"] = area
    target["iscrowd"] = iscrowd
    image_id = torch.tensor([idx])
    target["image_id"] = image_id

    if self.transforms:
      sample = self.transforms(image = img_res,
                                bboxes = target['boxes'],
                                labels = labels)
      img_res = sample['image']
      target['boxes'] = torch.Tensor(sample['bboxes'])

    return img_res, target

  def __len__(self):
    return len(self.imgs)

##############
# Analysis   #
##############
files_dir = './content/img/'
test_dir = './content/img1/'

dataset = ConeImagesDataset(files_dir, 224, 224)
img, target = dataset[78]

print('Length of dataset:', len(dataset), '\n')
print('Image shape:', img.shape)
print('Label example:', target)

img, target = dataset[25]
plot_img_bbox(img, target)

##############
# Transform  #
##############
train_transform = A.Compose(
      [
        A.HorizontalFlip(0.5),
        ToTensorV2(p=1.0)
      ],
      bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']}
    )

test_transform = A.Compose(
      [
        ToTensorV2(p=1.0)
      ],
      bbox_params={'format': 'pascal_voc', 'label_fields': ['labels']}
    )

##############
# Data       #
##############

dataset = ConeImagesDataset(files_dir, 480, 480, transforms=train_transform)
dataset_test = ConeImagesDataset(files_dir, 480, 480, transforms=test_transform)

# split the dataset in train and test set
torch.manual_seed(1)
indices = torch.randperm(len(dataset)).tolist()

# train test split
test_split = 0.2
tsize = int(len(dataset)*test_split)
dataset = torch.utils.data.Subset(dataset, indices[:-tsize])
dataset_test = torch.utils.data.Subset(dataset_test, indices[-tsize:])

# define training and validation data loaders
data_loader = torch.utils.data.DataLoader(
  dataset,
  batch_size=1,
  shuffle=True,
  num_workers=1,
  collate_fn=utils.collate_fn,
)

data_loader_test = torch.utils.data.DataLoader(
  dataset_test,
  batch_size=1,
  shuffle=False,
  num_workers=1,
  collate_fn=utils.collate_fn,
)

def get_object_detection_model(num_classes):
  model = torchvision.models.detection.fasterrcnn_resnet50_fpn(pretrained=True)

  in_features = model.roi_heads.box_predictor.cls_score.in_features
  model.roi_heads.box_predictor = FastRCNNPredictor(in_features, num_classes)
  return model

device =  torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
num_classes = 2 # IMPORTANT: One class is the background, the other the CONE
model = get_object_detection_model(num_classes)
model.to(device)

params = [p for p in model.parameters() if p.requires_grad]
optimizer = torch.optim.SGD(params, lr=0.005, momentum=0.9, weight_decay=0.0005)
lr_scheduler = torch.optim.lr_scheduler.StepLR(
  optimizer,
  step_size=3,
  gamma=0.1
)

##############
# Training   #
##############
def train_epoch(model, optimizer, data_loader, device, epoch, print_freq):
    model.train()
    metric_logger = utils.MetricLogger(delimiter="  ")
    metric_logger.add_meter('lr', utils.SmoothedValue(window_size=1, fmt='{value:.6f}'))
    header = 'Epoch: [{}]'.format(epoch)

    lr_scheduler = None
    if epoch == 0:
        warmup_factor = 1. / 1000
        warmup_iters = min(1000, len(data_loader) - 1)

        lr_scheduler = utils.warmup_lr_scheduler(optimizer, warmup_iters, warmup_factor)

    for images, targets in metric_logger.log_every(data_loader, print_freq, header):
        images = list(image.to(device) for image in images)
        targets = [{k: v.to(device) for k, v in t.items()} for t in targets]

        loss_dict = model(images, targets)

        losses = sum(loss for loss in loss_dict.values())

        # reduce losses over all GPUs for logging purposes
        loss_dict_reduced = utils.reduce_dict(loss_dict)
        losses_reduced = sum(loss for loss in loss_dict_reduced.values())

        loss_value = losses_reduced.item()

        if not math.isfinite(loss_value):
            print("Loss is {}, stopping training".format(loss_value))
            print(loss_dict_reduced)
            return

        optimizer.zero_grad()
        losses.backward()
        optimizer.step()

        if lr_scheduler is not None:
            lr_scheduler.step()

        metric_logger.update(loss=losses_reduced, **loss_dict_reduced)
        metric_logger.update(lr=optimizer.param_groups[0]["lr"])

@torch.no_grad()
def evaluate_model(model, data_loader, device):
    n_threads = torch.get_num_threads()
    torch.set_num_threads(1)
    cpu_device = torch.device("cpu")
    model.eval()
    metric_logger = utils.MetricLogger(delimiter="  ")
    header = 'Test:'

    coco = get_coco_api_from_dataset(data_loader.dataset)

    # IOU Type
    model_without_ddp = model
    if isinstance(model, torch.nn.parallel.DistributedDataParallel):
        model_without_ddp = model.module

    iou_types = ["bbox"]
    if isinstance(model_without_ddp, torchvision.models.detection.MaskRCNN):
        iou_types.append("segm")
    if isinstance(model_without_ddp, torchvision.models.detection.KeypointRCNN):
        iou_types.append("keypoints")
    #

    #coco_evaluator = CocoEvaluator(coco, iou_types)

    for image, targets in metric_logger.log_every(data_loader, 100, header):
        image = list(img.to(device) for img in image)
        targets = [{k: v.to(device) for k, v in t.items()} for t in targets]

        torch.cuda.synchronize()
        model_time = time.time()
        outputs = model(image)
        outputs = [{k: v.to(cpu_device) for k, v in t.items()} for t in outputs]
        model_time = time.time() - model_time

        res = {target["image_id"].item(): output for target, output in zip(targets, outputs)}
        evaluator_time = time.time()
        #coco_evaluator.update(res)
        evaluator_time = time.time() - evaluator_time
        metric_logger.update(model_time=model_time, evaluator_time=evaluator_time)

    metric_logger.synchronize_between_processes()
    print("Averaged stats:", metric_logger)
    #coco_evaluator.synchronize_between_processes()

    #coco_evaluator.accumulate()
    #coco_evaluator.summarize()
    torch.set_num_threads(n_threads)
    #return coco_evaluator

num_epochs = 1
for epoch in range(num_epochs):
    train_epoch(model, optimizer, data_loader, device, epoch, print_freq=10)

    lr_scheduler.step()
    #evaluate_model(model, data_loader_test, device=device)

##############
# Evaluation #
##############
test_dataset = ConeImagesDataset(test_dir, 480, 480, transforms=train_transform)
img, target = test_dataset[10]

model.eval()
with torch.no_grad():
  prediction = model([img.to(device)])[0]

# Image
def torch_to_pil(img):
  return torchtrans.ToPILImage()(img).convert('RGB')
img = torch_to_pil(img.to(torch.device('cpu')))

# Predictions
def apply_nms(orig_prediction, iou_thresh=0.3):
  keep = torchvision.ops.nms(orig_prediction['boxes'], orig_prediction['scores'], iou_thresh)

  final_prediction = orig_prediction
  final_prediction['boxes'] = final_prediction['boxes'][keep]
  final_prediction['scores'] = final_prediction['scores'][keep]
  final_prediction['labels'] = final_prediction['labels'][keep]

  return final_prediction
nms_prediction = apply_nms(prediction, iou_thresh=0.01)

plot_img_bbox(img, nms_prediction)


from datetime import date
import mysql.connector

db = mysql.connector.connect(host="localhost",user="root",password="",database="samochody")

mycursor = db.cursor()

class Rent:
    def __init__(self,excercse):
        if (excercse == 1):
            self.add_rent()
        elif (excercse == 2):
            self.display_rent_by_id()
        elif (excercse == 3):
            self.edit_rent_by_id()

    def add_rent(self):
        try:
            self.data_wyp = date.today()
            print("Podaj id klienta")
            self.id_kli = int(input())
            print("Podaj id samochodu")
            self.id_sam = int(input())
            print("Podaj rok końca wypożyczenia")
            rok = int(input())
            print("Podaj miesiąc końca wypożyczenia")
            miesiac = int(input())
            print("Podaj dzień końca wypożyczenia")
            dzien = int(input())
            self.data_kon = date(rok,miesiac,dzien)
            print("Podaj należność")
            self.kwota = int(input())
            self.add_query()
        except:
            print("Podano nie poprawne wartości.\n Wciśnij 1 aby ponownie wpisać dane, dowolny inny aby wyjść")
            x = int(input())
            if (x == 1):
                self.add_rent()
            else:
                print("Powrót do menu")

    def display_rent_by_id(self):
        self.display_query()

    def edit_rent_by_id(self):
        print("Podaj id wypożyczenia")
        try:
            self.id = int(input())
        except:
            print("Podano nie poprawne wartości.\n Wciśnij 1 aby ponownie wpisać dane, dowolny inny aby wyjść")
            x = int(input())
            if (x == 1):
                self.edit_rent_by_id()
            else:
                print("Powrót do menu")

    def add_query(self):
        try:
            mycursor.execute(
                "INSERT INTO wypozyczenia (idklienta,idauta,datawyp,datazwrotu,naleznosc) VALUES (%s,%s,%s,%s,%s)",
                (self.id_kli, self.id_sam, self.data_wyp, self.data_kon, self.kwota))
            db.commit()
            print("Dodawanie powiodło się")
        except Exception as e:
            print(e)

    def display_query(self):
        try:
            mycursor.execute("SELECT w.idwyp, k.imie, k.nazwisko, k.idklienta, a.model, a.idauta, w.datawyp, w.datazwrotu, w.naleznosc FROM klienci AS k,auta AS a,wypozyczenia AS w WHERE a.idauta=w.idauta AND k.idklienta = w.idklienta ORDER BY w.datawyp DESC")
            for x in mycursor:
                print(x)
        except Exception as e:
            print(e)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2021/3/9 14:03
# @Author : TETE
# @File : urls.py
from django.urls import path, re_path

from post import views

urlpatterns = [
    path('', views.queryAll),
    re_path('page/(\d+)$',views.queryAll),
    re_path('post/(\d+)$',views.postDetail),
    re_path('category/(\d+)$',views.queryPostByCid),
    re_path('archive/(\d+)/(\d+)$',views.queryPostByCreateTime)

]


import os
import sys
import numpy as np
import configparser
import netCDF4 as nC
import matplotlib.pyplot as plt
from main import is_valid_data_file
from tqdm import tqdm

CONFIG_FILENAME = './config.ini'
CONFIG_PATHS = 'MRR_PATHS'
CONFIG_INPUT_PATH = 'INPUT_PATH'
CONFIG_OUTPUT_PATH = 'OUTPUT_PATH'

PLOT_BASE_TITLE = 'MRR-Pro config from netCDF Archive: '

DATA_UNIT_CHECK = 0
DATA_VAR_LIST_CHECK = 1
DATA_DIM_SIZE = 2
DATA_DELTA_VAL = 3

DATA_FORMAT = {
    'Z_UNITS': {'var': 'Z', 'type': DATA_UNIT_CHECK, 'label': 'Z-dBZ', 'row': 0, 'units': 'dBZ'},
    'SPEC_R': {'var': 'spectrum_raw', 'type': DATA_VAR_LIST_CHECK, 'label': 'R-SPEC', 'row': 1, 'units': 'dBZ'},
    'SPEC_Z': {'var': 'spectrum_reflectivity', 'type': DATA_VAR_LIST_CHECK, 'label': 'Z-SPEC', 'row': 2, 'units': 'dBZ'},
    'SPEC_N': {'var': 'spectrum_n_samples', 'type': DATA_DIM_SIZE, 'label': 'N-SPEC', 'row': 3, 'units': 'dBZ'},
    'RANG_N': {'var': 'range', 'type': DATA_DIM_SIZE, 'label': 'N-Rng', 'row': 4, 'units': 'dBZ'},
    'TIME_DT': {'var': 'time', 'type': DATA_DELTA_VAL, 'label': 'dt', 'row': 5, 'units': 'dBZ'},
    'RANG_DR': {'var': 'range', 'type': DATA_DELTA_VAL, 'label': 'dR', 'row': 6, 'units': 'dBZ'}
}

N_DATA_ROWS = len(DATA_FORMAT)

class NcFileProperties:
    def __init__(self, name, nc_id):
        self.file_name = name
        self.time = int(nc_id.variables['time'][0])
        self.var_list = [var for var in nc_id.variables]
        self.data = np.empty(N_DATA_ROWS, dtype=np.int16)

        for item in DATA_FORMAT:
            var = DATA_FORMAT[item]['var']
            units = DATA_FORMAT[item]['units']
            row = DATA_FORMAT[item]['row']
            method = DATA_FORMAT[item]['type']
            self.data[row] = get_nc_diagnostic_vals(nc_id, var, units, self.var_list, method)

def get_nc_diagnostic_vals(file_id, var_name, unit_str, var_list, data_type):
    if data_type == DATA_UNIT_CHECK:
        if file_id.variables[var_name].getncattr('units') == unit_str:
            return 1
        else:
            return 0
    elif data_type == DATA_VAR_LIST_CHECK:
        if var_name in var_list:
            return 1
        else:
            return 0
    elif data_type == DATA_DIM_SIZE:
        return file_id.dimensions[var_name].size
    elif data_type == DATA_DELTA_VAL:
        return round(file_id.variables[var_name][1] - file_id.variables[var_name][0])

def read_nc_diagnostics(path, filename):
    nc_id = nC.Dataset(path+'/'+filename, "r")
    file_info = NcFileProperties(filename, nc_id)
    nc_id.close()
    return file_info

def dir_check(path, folder_name, length):

    if os.path.isdir(path+'/'+folder_name) is False:
        return False
    if len(folder_name) != length:
        return False
    try:
        year_value = int(folder_name[0:4])
        month_value = int(folder_name[4:6])
        if year_value < 2020 or year_value > 2050:
            return False
        if month_value < 1 or month_value > 12:
            return False
        if length == 8:
            day_value = int(folder_name[6:8])
            if day_value < 1 or day_value > 31:
                return False
        return True
    except ValueError:
        return False

def plot_data(time_data, diagnostic_data, figure_path, sub_path):

    plt.ion()
    fig, axs = plt.subplots(N_DATA_ROWS, sharex=True)
    fig.suptitle(PLOT_BASE_TITLE+sub_path)

    for item in DATA_FORMAT:
        row = DATA_FORMAT[item]['row']
        label = DATA_FORMAT[item]['label']
        axs[row].plot(time_data, diagnostic_data[row], 'o')
        axs[row].set_ylabel(label, fontsize=8, labelpad=10)
        if row < (N_DATA_ROWS-1):
            axs[row].tick_params('x', labelbottom=False)
        else:
            axs[row].tick_params('x', labelsize=8)
    plt.show(block=False)
    plt.savefig(figure_path + '/' + sub_path + '.png')
    plt.close()

def daily_data(path, out_path):
    mrr_folders = []
    # data_array = np.empty((N_DATA_VALS, 0), dtype=np.uint16)
    # time_array = np.empty(0, dtype='datetime64[s]')

    folders = sorted([folder for folder in os.listdir(path) if dir_check(path, folder, 6)])
    for folder in folders:
        try:
            os.mkdir(out_path+'/'+folder)
        except FileExistsError:
            pass
        current_path = os.path.join(path, folder)
        sub_folders = [sub_folder for sub_folder in os.listdir(current_path) if dir_check(current_path, sub_folder, 8)]
        sub_folders.sort()
        mrr_folders.extend(['/' + folder + '/' + sub_folder for sub_folder in sub_folders])

    num_folders = len(mrr_folders)
    for j in tqdm(range(num_folders)):
        sub_path = mrr_folders[j]
        current_path = path + sub_path
        file_list = sorted([file for file in os.listdir(current_path) if is_valid_data_file(current_path + '/' + file)])
        data_block = np.empty((N_DATA_ROWS, len(file_list)), dtype=np.uint16)
        time_block = np.empty(len(file_list), dtype='datetime64[s]')
        for i, file in enumerate(file_list):
            file_data = read_nc_diagnostics(current_path, file)
            data_block[:, i] = file_data.data[:]
            time_block[i] = np.datetime64('1970-01-01') + np.timedelta64(file_data.time, 's')
        plot_data(time_block, data_block, out_path, sub_path)
        # data_array = np.concatenate((data_array, data_block), axis=1)
        # time_array = np.concatenate((time_array, time_block), axis=0)
    # return data_array, time_array

def config_okay(config_data):
    try:
        if os.path.isdir(config_data[CONFIG_PATHS][CONFIG_INPUT_PATH]) is False:
            return False
        if os.path.isdir(config_data[CONFIG_PATHS][CONFIG_INPUT_PATH]) is False:
            os.mkdir(config_data[CONFIG_PATHS][CONFIG_OUTPUT_PATH])
    except KeyError:
        return False
    except FileNotFoundError:
        return False
    return True

if __name__ == "__main__":

    config = configparser.ConfigParser()
    config.read(CONFIG_FILENAME)

    if config_okay(config) is False:
        sys.exit()

    data_path = config[CONFIG_PATHS][CONFIG_INPUT_PATH]
    save_path = config[CONFIG_PATHS][CONFIG_OUTPUT_PATH]

    daily_data(data_path, save_path)


#!/usr/bin/env python
# coding: utf-8

import re
import argparse
import pandas as pd
import numpy as np

from scipy.sparse import hstack
from sklearn.preprocessing import OneHotEncoder
from keras.models import Sequential
from keras.layers import LSTM, Dense

# Local modules
import auxiliary

# Constantes
RNA_BASES = [["A"], ["U"], ["C"], ["G"]]
encoder = OneHotEncoder(handle_unknown='ignore', sparse_output=False).fit(RNA_BASES)

# Function to keep rows with maximum signal_to_noise within identical sequences
def filter_identical_sequences(df):
    # Group by 'sequence' and keep the row with max 'signal_to_noise'
    filtered_df = df.groupby('sequence').apply(lambda x: x.loc[x['signal_to_noise'].idxmax()])
    return filtered_df

def dict_from_data(data, keys_name=["2A3_MaP", "DMS_MaP"]):
    n_duplicate = sum(data.duplicated(subset=["sequence", "experiment_type"]))
    if n_duplicate > 0:
        return None

    seq_reactivity = dict()
    for seq, group in data.groupby("sequence"):
        seq_reactivity[seq] = dict()
        seq_reactivity[seq]["id"] = group["sequence_id"].values[0]
        for key in keys_name:
            mask = group["experiment_type"] == key
            seq_reactivity[seq][key] = group[mask].drop(
                labels=["sequence_id", "sequence", "experiment_type"], axis=1
            ).values.reshape(-1)
            seq_reactivity[seq][key] = np.expand_dims(seq_reactivity[seq][key], axis=1)

    return seq_reactivity

def XY_from_dict(dict_data, encoder, maxlen=457):
    id_list = []
    x_list = []
    y_list = []
    for sequence, reactivities in dict_data.items():
        y = np.hstack([reactivities["2A3_MaP"], reactivities["DMS_MaP"]])
        x_list.append(onehot_from_sequence(sequence, encoder, maxlen=maxlen))
        y_list.append(padded_matrix(y, maxlen=maxlen))
        id_list.append(reactivities["id"])

    return np.array(x_list), np.array(y_list), np.array(id_list)

def onehot_from_sequence(sequence, encoder, to_add="0", maxlen=457):
    """sequence: str"""
    if maxlen is None:
        maxlen = 0
    proccessed_sequence = sequence.upper()
    proccessed_sequence += to_add * (maxlen - len(sequence))
    proccessed_sequence = [[nbase] for nbase in proccessed_sequence]
    onehot_sequence = encoder.transform(proccessed_sequence)
    return onehot_sequence

def padded_matrix(matrix_2d, maxlen=457):
    """"""
    if not isinstance(matrix_2d, np.ndarray):
        matrix_2d = np.array(matrix_2d)

    n_toadd = maxlen - matrix_2d.shape[0]
    padding = ((0, n_toadd), (0, 0))  # padding on axis
    matrix_2d_padded = np.pad(matrix_2d, pad_width=padding, mode="constant")
    return matrix_2d_padded

def reactivity_normalization(df_2A3_MaP, df_DMS_MaP, y_columns):
    """
    Robust Z-score Normalization of reactivities

    Parameters:
    - df_2A3_MaP(DataFrame): dataframe with the 2A3 experiment results
    - df_DMS_MaP(DataFrame): dataframe with the DMS experiment results

    Returns:
    - df_2A3_MaP(DataFrame): Normalized dataframe with the 2A3 experiment results
    - df_DMS_MaP(DataFrame) : Normalized dataframe with the DMS experiment results
    """
    # Calculate median and Median Absolute Deviation (MAD) for robust normalization
    a3_median = np.nanmedian(df_2A3_MaP[y_columns].values.reshape(-1))
    a3_mad = np.nanmedian(np.abs(df_2A3_MaP[y_columns].values.reshape(-1) - a3_median), axis=0)
    dms_median = np.nanmedian(df_DMS_MaP[y_columns].values.reshape(-1))
    dms_mad = np.nanmedian(np.abs(df_DMS_MaP[y_columns].values.reshape(-1) - dms_median), axis=0)

    # Apply robust z-score normalization to all DataFrames from column 5 onwards
    cst = 1.482602218505602
    df_2A3_MaP[y_columns] = (df_2A3_MaP[y_columns] - a3_median) / (cst * a3_mad)
    df_DMS_MaP[y_columns] = (df_DMS_MaP[y_columns] - dms_median) / (cst * dms_mad)

    return df_2A3_MaP, df_DMS_MaP

def getReactivities(filepath,
                    key_2A3="2A3_MaP",
                    key_DMS="DMS_MaP",
                    experiment_column="experiment_type",
                    pattern="^reactivity_[0-9]{4}$",
                    **kwargs
):
    """Returns reactivities values for each experiment.
    filepath should redirect to a cleaned dataframe.
    """
    dt = auxiliary.read_dataframe(filepath, **kwargs)
    y_columns = [colname for colname in dt.columns if re.match(pattern, colname)]
    values_2A3 = dt[dt[experiment_column] == key_2A3]
    values_DMS = dt[dt[experiment_column] == key_DMS]
    # Reactivities
    reactivities_2A3 = values_2A3[y_columns].to_numpy().flatten()
    reactivities_DMS = values_DMS[y_columns].to_numpy().flatten()

    return {"2A3": reactivities_2A3, "DMS": reactivities_DMS}

if __name__ == "__main__":
    data = pd.read_csv("./data/SN_filtered_train.csv", header=0)

    # Get columns name for X, and Y
    x_columns = ["sequence_id", "sequence"]
    conditional_columns = ["experiment_type", "signal_to_noise"]
    y_columns = [colname for colname in data.columns if re.match("^reactivity_[0-9]{4}$", colname)]

    # Keep the necessary columns from the DataFrame
    cleaned_data = data[x_columns + conditional_columns + y_columns]

    # Create two separate DataFrames based on "experiment_type"
    df_2A3_MaP = cleaned_data[cleaned_data['experiment_type'] == '2A3_MaP']
    df_DMS_MaP = cleaned_data[cleaned_data['experiment_type'] == 'DMS_MaP']

    # Delete cleaned_train_data to free space memory
    del cleaned_data

    df_2A3_MaP_filtered = filter_identical_sequences(df_2A3_MaP)  # Filter df_2A3_MaP
    df_DMS_MaP_filtered = filter_identical_sequences(df_DMS_MaP)  # Filter df_DMS_MaP

    df_2A3_MaP_normalized, df_DMS_MaP_normalized = reactivity_normalization(df_2A3_MaP_filtered, df_DMS_MaP_filtered, y_columns)

    # Concatenate the two data frames
    mask_2A3 = df_2A3_MaP_normalized["sequence"].isin(df_DMS_MaP_normalized["sequence"])
    mask_DMS = df_DMS_MaP_normalized["sequence"].isin(df_2A3_MaP_normalized["sequence"])

    cleared_data = pd.concat([df_2A3_MaP_normalized[mask_2A3], df_DMS_MaP_normalized[mask_DMS]], ignore_index=True)
    cleared_data.drop(columns=['signal_to_noise'], inplace=True)

    # columns type
    cleared_data[y_columns] = cleared_data[y_columns].astype(np.float32)

    # Save cleared_train_data as a CSV file
    csv_path = './data/cleared_data.csv'
    cleared_data.to_csv(csv_path, index=False)
    dict_data = dict_from_data(cleared_data)
    x, y, id = XY_from_dict(dict_data, encoder)

    # Split the data into training and validation sets
    x_train, x_val, x_test, \
    y_train, y_val, y_test, \
    id_train, id_val, id_test = \
        auxiliary.train_val_test_split(x, y, id)

    auxiliary.save_npy(
        x, "x.npy",
        y, "y.npy",
        id, "id.npy",
        x_train, "x_train.npy",
        y_train, "y_train.npy",
        id_train, "id_train.npy",
        x_val, "x_val.npy",
        y_val, "y_val.npy",
        id_val, "id_val.npy",
        x_test, "x_test.npy",
        y_test, "y_test.npy",
        id_test, "id_test.npy"
    )


"""SnapBuddy is the Buddy you want to take a timelapse of your plant.  The SnapBuddy device has
the ESP32-CAM Tasmota with the AI Thinker Template installed.
"""
import sys

sys.path.insert(0, "../")

from growbuddies.settings_code import Settings
import requests
import datetime
import time
import os
from growbuddies.logginghandler import LoggingHandler

class SnapBuddy:
    def __init__(self, settings_string=None):
        self.logger = LoggingHandler()
        if settings_string is None:
            self.logger.error(
                "An empty string was provided as an input for a valid entry in the growbuddies_settings for a snapbuddy. "
                " Please provide a valid string representing an entry in the growbuddies_settings for the snapbuddy."
            )
            sys.exit()
        settings = Settings()
        settings.load()
        self.snapbuddy_settings_dict = settings.get(settings_string)
        self.settings_string = settings_string  # For __repr__ documentation.

        if self.snapbuddy_settings_dict is None:
            self.logger.error(
                f"The settings_string '{settings_string}' was not found in the growbuddies_settings. "
                "Please provide a valid string representing an entry in the growbuddies_settings for the snapbuddy."
            )
            sys.exit()
        try:
            self.hostname = self.snapbuddy_settings_dict["hostname"]
            self._interval_secs = self.snapbuddy_settings_dict["interval_secs"]
            self.timelapse_dir = self.snapbuddy_settings_dict["timelapse_dir"]
        except KeyError as e:
            self.logger.error(
                f"The settings_string '{settings_string}' was found in the growbuddies_settings but the key '{e}' was not found. "
                "Please provide a valid {e} : value pair in the growbuddies_settings for the snapbuddy"
            )
            sys.exit()

    def __repr__(self):
        return (
            f"SnapBuddy(settings_string='{self.settings_string}', "
            f"hostname='{self.hostname}', "
            f"interval_secs={self.interval_secs}, "
            f"timelapse_dir='{self.timelapse_dir}')"
        )

    def snap(self):
        # This is the URL to get a snapshot from a Tasmota running
        # an ESP32-CAM with the AI Thinker template installed.
        url = "http://" + self.hostname + ":80/snapshot.jpg"
        # Send the request to the Tasmota device to capture a snapshot
        response = requests.get(url)

        if response.status_code == 200:
            filename = self._get_filename()

            # Save the snapshot image to a file
            with open(filename, "wb") as f:
                f.write(response.content)
            self.logger.debug(f"Snapshot saved to {filename}")
        else:
            self.logger.error(
                f"Could not get the snapshot from {self.hostname}. The response code was {response.status_code} and the reason was {response.reason}."
            )

        # Check if the request was successful
        self.logger.debug(f"response code: {response.status_code}")

    @property
    def interval_secs(self):
        return self._interval_secs

    def _get_filename(self):
        """Get the filename for the snapshot image.
        The filename is a combination of the timelapse directory and the current date and time.
        """
        # Make sure the timelapse directory exists
        # Split the file path into individual directories
        dirs = self.timelapse_dir.split("/")
        # remove the root directory
        dirs.pop(0)

        # Iterate through the directories, creating each one if it does not exist
        subdir = ""
        for i in range(len(dirs)):
            subdir += "/" + dirs[i]
            if not os.path.exists(subdir):
                os.mkdir(subdir)
        filename = f"{self.timelapse_dir}/{datetime.datetime.now().strftime('%m%d%Y_%H%M%S')}.jpg"
        return filename

def main():
    logger = LoggingHandler()

    snapbuddy = SnapBuddy("snapbuddy_settings_1")
    logger.debug(f"-> SnapBuddy class initialized. {snapbuddy}")
    while True:
        try:
            snapbuddy.snap()
        except KeyboardInterrupt:
            sys.exit()

        time.sleep(snapbuddy.interval_secs)

if __name__ == "__main__":
    main()


from django.test import TestCase
from django.http import HttpRequest
from django.template.loader import render_to_string
from MyWeb.views import mainpage
from MyWeb.views import Page
from django.urls import resolve
from MyWeb.models import Item, User
from django.urls import reverse

class MainpageTest(TestCase):

	def html_mainpage_root_mainpage(self):
		found = resolve('/')
		self.assertEqual(found.func, mainpage)
		self.assertEqual(found.func, Add)

	def test_mainpage_returns_correct_view(self):
		request = HttpRequest()
		response = mainpage(request)
		html = response.content.decode('UTF-8')
		response = self.client.get('/')
		self.assertTemplateUsed(response, 'mainpage.html')
		self.assertTrue(html.strip().startswith(''))
		self.assertTemplateUsed(response, 'mainpage.html')
		self.assertIn('', html)
		self.assertIn('', html)
		self.assertIn('<meta name="viewport" content="width=device-width, initial-scale=1.0">', html)
		self.assertIn('<title>Restaurant Dine-In Booking</title>', html)
		self.assertIn('', html)
		self.assertIn('', html)
		self.assertIn('
', html)
		self.assertIn('<form class="form" action="" method="post">' ,html)
		self.assertIn('
', html)
		self.assertIn('
Restaurant Dine-In Booking
', html)
		self.assertIn('
', html)
		self.assertIn('
Name
', html)
		self.assertIn('<input size="40" type="text" id=Name name="Name" placeholder="Fisrtname, Lastname">', html)
		self.assertIn('
Restaurant
', html)
		self.assertIn('<input size="40" type="text" id=Name1 name="Name1" placeholder="Name of Restaurant">', html)
		self.assertIn('
Address
', html)
		self.assertIn('<input size="40" type="text" id=Address name="Address" placeholder="Restaurant Address">', html)
		self.assertIn('
Date
', html)
		self.assertIn('<input  style="height: 35px; width: 200px; " type="date" id="Date" placeholder="Date" name="Date">', html)
		self.assertIn('
Time
', html)
		self.assertIn('<input size="40" type="text" id=Time name="Time" placeholder="HH:MM Am/Pm">', html)
		self.assertIn('
', html)
		self.assertIn('<input type="submit" id="Submit" name="Submit" value="Submit">', html)
		self.assertIn('</form>', html)
		self.assertIn('', html)
		self.assertIn('', html)
		self.assertTrue(html.strip().endswith(''))

class orm_testing_save(TestCase):
	def test_saving(self):
		Item1 = Item()
		Item1.Name = 'Eden Mendoza'
		Item1.save()
		Item2 = Item()
		Item2.Name1 = 'Unli Wings'
		Item2.save()
		Item3 = Item()
		Item3.Address = 'Palipran 2, Dasmarinas City, Cavite'
		Item3.save()
		Item4 = Item()
		Item4.Date = '2021-04-26'
		Item4.save()
		Item5 = Item()
		Item5.Time = '12:00 Pm'
		Item5.save()
		saveall = Item.objects.all()
		self.assertEqual(saveall.count(), 5)
		save1 = saveall[0]
		save2 = saveall[1]
		save3 = saveall[2]
		save4 = saveall[3]
		save5 = saveall[4]
		self.assertEqual(Item1.Name, 'Eden Mendoza')
		self.assertEqual(Item2.Name1, 'Unli Wings')
		self.assertEqual(Item3.Address, 'Palipran 2, Dasmarinas City, Cavite')
		self.assertEqual(Item4.Date, '2021-04-26')
		self.assertEqual(Item5.Time, '12:00 Pm')

class orm_testing_save1(TestCase):
	def test_saving1(self):
		Item1 = Item()
		Item1.Name = 'Charles DelaCruz'
		Item1.save()
		Item2 = Item()
		Item2.Name1 = 'Samgyupsal'
		Item2.save()
		Item3 = Item()
		Item3.Address = 'Salawag, Dasmarinas City, Cavite'
		Item3.save()
		Item4 = Item()
		Item4.Date = '2021-04-26'
		Item4.save()
		Item5 = Item()
		Item5.Time = '12:00 Pm'
		Item5.save()
		saveall = Item.objects.all()
		self.assertEqual(saveall.count(), 5)
		save1 = saveall[0]
		save2 = saveall[1]
		save3 = saveall[2]
		save4 = saveall[3]
		save5 = saveall[4]
		self.assertEqual(Item1.Name, 'Charles DelaCruz')
		self.assertEqual(Item2.Name1, 'Samgyupsal')
		self.assertEqual(Item3.Address, 'Salawag, Dasmarinas City, Cavite')
		self.assertEqual(Item4.Date, '2021-04-26')
		self.assertEqual(Item5.Time, '12:00 Pm')

class Views_Test(TestCase):
	def test_views(self):
		Item.objects.create(
			Name='Name',
			Name1='Name1',
			Address='Address',
			Date='2021-04-26',
			Time='12:00 Pm',
			)
		response = self.client.get('/app/views.mainpage/')

class PageViewTest(TestCase):

	def test_uses_page_template(self):
		viewer = User.objects.create()
		response = self.client.get('/')
		self.assertTemplateUsed(response, 'mainpage.html')
	def test_uses_home_template(self):
		response = self.client.get('/next/')
	def test_displays_all_list_items(self):
		viewer = User.objects.create()
		Name = Item.objects.create(Name='Name')
		Name1 = Item.objects.create(Name1='Name1')
		Address = Item.objects.create(Address='Address')
		Date = Item.objects.create(Date='2021-04-26')
		Time = Item.objects.create(Time='12:00 Pm')
		response = self.client.get('/')
		self.assertIn('Name', response.content.decode())
		self.assertIn('Name1', response.content.decode())
		self.assertIn('Address', response.content.decode())
		self.assertIn('Date', response.content.decode())
		self.assertIn('Time', response.content.decode())
		Name = Item.objects.get(Name='Name')
		Name1 = Item.objects.get(Name1='Name1')
		Address = Item.objects.get(Address='Address')
		Date = Item.objects.get(Date='2021-04-26')
		Time = Item.objects.get(Time='12:00 Pm')

		self.assertEqual(Item.objects.count(), 5)

class Views(TestCase):
	def setUp(self):
		Name = Name.objects.create()
		Name1 = Item.objects.create()
		Address = Item.objects.create()
		Date = Item.objects.create()
		Time = Item.objects.create()

		Item.objects.create(
			Name = 'Eden Mendoza',
			Name1 = 'Unli Wings',
			Address = 'Palipran 2, Dasmarinas City, Cavite',
			Date = '2021-04-26',
			Time='12:00 Pm',
			)
		self.client.post('/Add/', Name='Juan DelaCruz')
		response = self.client.post('/next/')
		self.assertRedirects(response, '/next/')

class Test_models(TestCase):

	def mainpage(self,
		Name='Name',
		Name1='Name1',
		Address='Address',
		Date='2021-04-26',
		Time='12:00 Pm'):

		return User.objects.create()
		return Item.objects.create(
		Name='Name',
		Name1='Name1',
		Address='Address',
		Date='2021-04-26',
		Time='12:00 Pm',)

	def test_whatever_creation(self):
		w = self.mainpage()
		self.assertTrue(isinstance(w, User))
		self.assertFalse(isinstance(w, Item))


# websockets client (Async)

import asyncio
import logging
import json
import websockets

from datetime import datetime

print("Sample code for powermon client")

async def data_send(ws, interval):
    while True:
      await ws.send(json.dumps({
        'msg_name': 'data',
        'sensor_id': 'gresystem000001',
        'transaction_id': '123456789',
        'timestamp': str(datetime.now()),
        'data': {
          'probe_id': 1,
          'probe_type': 'CT',
          'power': [20, 30, 20, 30, 40, 50, 30, 30, 20, 20],
          'pf': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
          'voltage': 220
        }
      }))
      conf = await ws.recv()
      print('Message received:', conf)
      await asyncio.sleep(interval)

async def main():
    interval = 5
    async with websockets.connect(
        'ws://127.0.0.1:5000/powermon/data/gresystem000001'
    ) as ws:

        await asyncio.gather(
          # interval = sensor_boot(ws),
          data_send(ws, interval),
        )

if __name__ == '__main__':
    try:
        # asyncio.run() is used when running this example with Python 3.7 and
        # higher.
        asyncio.run(main())
    except AttributeError:
        # For Python 3.6 a bit more code is required to run the main() task on
        # an event loop.
        loop = asyncio.get_event_loop()
        loop.run_until_complete(main())
        loop.close()


# -*- coding:utf-8 -*-

import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import seaborn as sns

import matplotlib.ticker as ticker

def save_to_exel(df, file_name):
	writer = pd.ExcelWriter("%s.xlsx"%file_name, engine='xlsxwriter')
	df.to_excel(writer, sheet_name='Sheet1')
	writer.save()

#df_data = pd.read_excel("sort_betSA_PV_H2_up_2.xlsx")

def plot_pv(df_data):

    value_pv = df_data["Value_PV(cm3/g)"]
    h2_up = df_data["H2_UP (w/W %)"]

    fig = plt.figure()

    ax = fig.add_axes([0.1, 0.1, 1.0, 1.0])
    #ax.plot([value_pv, value_pv*value_pv])

    ax.scatter(value_pv, h2_up, alpha=0.5)
    ax.set_ylabel("$H_2$ Uptake (w/W %)")
    ax.set_xlabel("Pore Volume ($cm_3$/g)")
    ax.set_title("Distributions of H2_UP $H_2$ Uptake (w/W %) by Pore Volume ($cm_3$/g)")
    ax.scatter(h2_up, value_pv, alpha=0.2)

    limited_h2_up = df_data.loc[df_data["H2_UP (w/W %)"] > 8]

    limited_h2_up_index = limited_h2_up.index

    #limited_value_pv = df_data.loc[df_data["Value_PV(cm3/g)"] > 5]

    mof_names = limited_h2_up["MOF Name"]
    for i, txt in zip(limited_h2_up_index, mof_names):
        ax.annotate(txt, (value_pv[i], h2_up[i]))

#plot_pv(df_data)

def plot_sa(df_data):

    value_sa = df_data["Value_SA(m2/g)"]
    h2_up = df_data["H2_UP (w/W %)"]

    fig1 = plt.figure()
    ax1 = fig1.add_axes([0.1, 0.1, 1.0, 1.0])

    ax1.scatter(value_sa, h2_up, alpha=0.7)
    ax1.set_ylabel("$H_2$ Uptake (w/W %)")
    ax1.set_xlabel("Surface Area ($cm_2$/g)")
    ax1.set_title("Distributions of H2_UP $H_2$ Uptake (w/W %) bySurface Area ($cm_2$/g)")
    ax1.set_xlim([0, 12000])
    ax1.set_ylim([0, 50])

    plt.xticks(np.arange(10))

    #ax1.set_yscale("log")
    #ax1.set_xscale('log')

    ax2 = fig1.add_axes([0.2, 0.5, 0.4, 0.4])
    ax2.scatter(value_sa, h2_up, alpha=0.7)
    ax2.set_xlim([5000, 10000])
    ax2.set_ylim([10, 20])
    ax2.set_title("Zoom")
    ax2.set_xscale('log')

    limited_h2_up = df_data.loc[df_data["H2_UP (w/W %)"] > 8]
    limited_h2_up_index = limited_h2_up.index
    mof_names = limited_h2_up["MOF Name"]
    for i, txt in zip(limited_h2_up_index, mof_names):
        ax2.annotate(txt, (value_sa[i], h2_up[i]))

    plt.show()

#plot_sa(df_data)

#fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(4, 2))

#axes[0].plot(x, y, color="blue", linestyle="-")
#axes[1].plot(x, z, color="red", linestyle="-.")

def get_all_mofs_from_web_ofsci_scopus():
    i = 1
    df_mofs_web_ofsci = []
    while i <= 26:
        fl = "/home/omert/Desktop/mof_text_minig/get_dois/web_of_sci_all_mof1/web_of_sci_all_mof_%s" %i

        df_mof = pd.read_csv("%s.txt" %fl, sep='\t', lineterminator='\r')
        df_mofs_web_ofsci.append(df_mof)
        i += 1

    df_mofs_web_ofsci = pd.concat(df_mofs_web_ofsci)
    df_mofs_scopus = pd.read_csv("/home/omert/Desktop/mof_text_minig/get_dois/scopus_all_mof_all_year_1.csv", low_memory=False)

    df_mofs_web_ofsci_doi_year = df_mofs_web_ofsci[["AR", "PD"]]
    df_mofs_web_ofsci_doi_year = df_mofs_web_ofsci_doi_year.rename(columns={"AR": "DOI"})
    df_mofs_web_ofsci_doi_year = df_mofs_web_ofsci_doi_year.rename(columns={"PD": "Year"})

    df_mofs_scopus_doi_year = df_mofs_scopus[["DOI", "Year"]]

    df_all_mofs_doi_year = pd.concat([df_mofs_web_ofsci_doi_year, df_mofs_scopus_doi_year])

    df_all_mofs_doi_year = df_all_mofs_doi_year.drop_duplicates()

    #save_to_exel(df_all_mofs_doi_year, "all_mofs_doi_year_from_web_ofsci")
    return df_all_mofs_doi_year

#get_all_mofs_from_web_ofsci_scopus()

def graph_mofs_byyear(df_all_mofs_doi_year):
    sns.set()
    all_mofs_byyears = df_all_mofs_doi_year["Year"]

    start_year = 2017
    finish_year = 1969

    year_Number = {}
    for i in range(start_year-finish_year):
        year = start_year - i

        n_mofs = len(all_mofs_byyears.loc[all_mofs_byyears==year])
        #print (year, "--->", n_mofs)
        year_Number[year] = n_mofs

    years = np.array(list(year_Number.keys()))
    number_ofmofs_byyear = np.array(list(year_Number.values()))

    #get graph

    fig = plt.figure()
    ax = fig.add_axes([0.1, 0.1, 1.0, 1.0])

    ax.bar(years, number_ofmofs_byyear, alpha=0.5, align='center')
    ax.set_ylabel("Years")
    ax.set_xlabel("Number of Publication related MOF and Coordination Polymers")
    ax.set_title("Distribution of Number of Publication related MOF and Coordination Polymers by years")
    ax.set_xlim(1995, 2020)
    ax.set_ylim(0, 30000)

    # polt regression line
    from numpy.polynomial.polynomial import polyfit
    b, m = polyfit(years, number_ofmofs_byyear, 1)
    plt.plot(years, b + m * years)
    #plt.xticks(year_Number.keys(), year_Number.values())

    plt.show()

def graph_mofs_byyear_sns(df_all_mofs_doi_year):

    all_mofs_byyears = df_all_mofs_doi_year["Year"]

    start_year = 2017
    finish_year = 1969

    df_number_ofmofs_byyear = pd.DataFrame(columns=["Year", "Number of MOF"])

    year_Number = {}
    for i in range(start_year-finish_year):
        year = start_year - i

        n_mofs = len(all_mofs_byyears.loc[all_mofs_byyears==year])
        #print (year, "--->", n_mofs)
        year_Number[year] = n_mofs

    df_number_ofmofs_byyear["Year"] = list(year_Number.keys())
    df_number_ofmofs_byyear["Number of MOF"] = list(year_Number.values())

    #get graph
    sns.set(style="ticks", color_codes=True, font_scale=1)
    #sns.lmplot("Year", "Number of MOF", data=df_number_ofmofs_byyear, fit_reg=False)

    #fig = plt.figure()

    ax = sns.barplot(x="Year", y="Number of MOF", data=df_number_ofmofs_byyear)
    ax.set_xlabel("Years")
    ax.set_ylabel("Number of Publication")
    ax.set_title("Distribution of Number of Publication Related MOF and Coordination Polymers by Years")

    ax.set_ylim(0, 30000)
    ax.autoscale_view(scalex=True)

    #ax.set_xticklabels(labels=sorted(list(year_Number.keys())[:28]), rotation=45, ha='right', )

    ax.xaxis.set_major_locator(plt.AutoLocator())
    ax.xaxis.set_minor_locator(plt.MultipleLocator(1))

    #ax.yaxis.set_major_locator(plt.LogLocator(base=10.0, numticks=15))

    #ax1 = sns.regplot(x="Year", y="Number of MOF", data=df_number_ofmofs_byyear, truncate=True)
    #ax1.set_yscale("log")

    plt.show()

    #manager = plt.get_current_fig_manager()

    #plt.savefig("number_of_mofs_per_year.jpeg",)

#df_all_mofs_doi_year = pd.read_excel("all_mofs_doi_year_from_web_ofsci.xlsx")
#graph_mofs_byyear_sns(df_all_mofs_doi_year)

def compare_indexed():
    value = "| POAV_cm^3/g"
    label_value = "Surface Area"
    tm = pd.read_csv("/home/omert/Desktop/mof_text_minig/in_output/mof_mining/result_analysis/clean_indexed/clean_pv_alphanum_indexed_name.csv")\
            [["MOF_Name_alphnum_indexed", "Value"]]
    dm = pd.read_csv("/home/omert/Desktop/mof_text_minig/in_output/mof_mining/result_analysis/dm_mofname_indexed/ours_dm_pv_alphanum_indexed_name.csv")
    df_merge = pd.merge(tm, dm, how="left", left_on="MOF_Name_alphnum_indexed", right_on="MOF_Name_alphnum_indexed")
    df_merge = df_merge[df_merge["MOF_Name_alphnum_indexed"].apply(lambda x: len(x) > 1)]
    df_merge = df_merge.dropna(subset=[value])
    df_merge = df_merge[df_merge["Value"].apply(lambda x: float(x) > 0)]
    df_merge = df_merge[df_merge[value].apply(lambda x: float(x) > 0)]
    df_merge["Value"] = df_merge["Value"].astype(float)

    #df_merge.to_csv("merge_test.csv")

    sns.relplot(x=value, y="Value", data=df_merge)
    plt.ylabel("TM %s ($cm_2$/g)" %label_value)
    plt.xlabel("DM %s ($cm_2$/g)" %label_value)

    #values_tm = df_merge["Value"].astype(float)
    #values_dm = df_merge[value].astype(float)

    #fig1 = plt.figure()
    #ax1 = fig1.add_subplot(111)
    #ax1.scatter(values_tm, values_dm)
    ##ax1.set_title("Distributions of H2_UP $H_2$ Uptake (w/W %) bySurface Area ($cm_2$/g)")
    #ax1.set_xlim([0, 10000])
    #ax1.set_ylim([0, 10000])

    #plt.xticks(np.arange(10))

    #ax1.set_yscale("log")
    #ax1.set_xscale('log')

    #ax2 = fig1.add_axes([0.2, 0.5, 0.4, 0.4])
    #ax2.scatter(value_sa, h2_up, alpha=0.7)
    #ax2.set_xlim([5000, 10000])
    #ax2.set_ylim([10, 20])
    #ax2.set_title("Zoom")
    #ax2.set_xscale('log')

    plt.show()
compare_indexed()


import motor.motor_asyncio

class MongoDBConnector:
    """Handles connection with a MongoDB database."""

    def __init__(self, srv_url, db_name, loop):
        self.client = motor.motor_asyncio.AsyncIOMotorClient(srv_url, io_loop=loop)
        self.db = self.client[db_name]

    async def put_member(self, member_id):
        """Store a Discord guild member ID."""
        await self.db.members.insert_one({'id': member_id})

    async def get_all_members(self):
        """Retrieve a list of all member IDs from the database."""
        cursor = self.db.members.find()
        members = await cursor.to_list(length=None)
        return [member['id'] for member in members]

    async def remove_member(self,member_id):
        """Removes ID of member leaving the server"""
        await self.db.members.delete_one({'id':member_id})