import numpy as np
import pandas as pd
from .individuo import Individuo
from collections import Counter, defaultdict
import datetime as dt

class SimuladorBase:
    def __init__(self, info_poblacion, precision_tests, probabilidad, duracion_infeccion, R0, dia_inicial):
        self.info_poblacion = info_poblacion
        self.tamano_poblacion = info_poblacion.shape[0]
        self.precision_tests = precision_tests
        self.probabilidad = probabilidad
        self.S = 'susceptible'
        self.E = 'expuesto'
        self.I = 'infeccioso'
        self.R = 'recuperado'
        self.duracion_infeccion = duracion_infeccion
        self.R0_empresa = R0['empresa']
        self.R0_poblacion = R0['poblacion']
        self.dia_inicial = dia_inicial

        self.alpha = 1 / 3  #periodo de incubacion
        self.gamma = 1 / 18 #periodo medio de infeccion
        self.beta_poblacion = self.gamma * self.R0_poblacion
        self.beta_empresa = self.gamma * self.R0_empresa
        self.p_contagio_diaria = {'trabajo': self.probabilidad[self.I] * self.beta_empresa,
                                  'cuarentena': self.probabilidad[self.I] * self.beta_poblacion}

        self.tiempo = 0
        self.dia = dia_inicial

        self.poblacion = self.crear_poblacion()

        self.historia_numero_tests = [0]
        self.historia_infectados_totales = {ind.id for ind in self.poblacion.values() if ind.estado == self.I}
        self.numero_infectados_totales = [len(self.historia_infectados_totales)]

        self.historia_observados_totales = {ind.id for ind in self.poblacion.values() if ind.estado == self.I and ind.estado_observado == self.I}
        self.numero_observados_totales = [len(self.historia_observados_totales)]

    def crear_poblacion(self):
        poblacion = {}
        lista_estados = [self.S, self.E, self.I, self.R]
        probs = [self.probabilidad[est] for est in lista_estados]

        for ind, row in self.info_poblacion.iterrows():
            estado_init = row['estado_inicial']
            if estado_init == 'random':
                estado_init = np.random.choice(lista_estados, 1, p=probs)[0]

            inicio_sintomas = min(np.random.lognormal(np.log(4.1) - 8 / 9, 4 / 3), 20) + 1
            final_sintomas = inicio_sintomas + np.random.randint(7, 10)
            duracion_infeccion = final_sintomas + np.random.randint(5, 10)
            dia_muerte = final_sintomas - 1
            dias_sintomas = [int(inicio_sintomas), int(final_sintomas)]

            dia_aparicion_ac = 0
            sintomatico = False
            muere = False

            if np.random.rand() < self.probabilidad['sintomatico']:
                sintomatico = True
                if np.random.rand() < self.probabilidad['muerte']:
                    muere = True

            tipo_turno = row['tipo_turno']

            poblacion[ind] = Individuo(estado_init, ind, duracion_infeccion, dias_sintomas, dia_muerte, dia_aparicion_ac, sintomatico, muere, self.dia_inicial, tipo_turno)
            if estado_init == 'random':
                poblacion[ind].tiempo_infeccioso = random.randint(10, 17)

        return poblacion

    def actualizar_estados(self):
        self.estados = dict((ind.id, ind.estado) for ind in self.poblacion.values())

    def actualizar_estados_observados(self):
        self.estados_observados = dict((ind.id, ind.estado_observado) for ind in self.poblacion.values())

    def actualizar_actividades(self):
        self.actividades = dict((ind.id, ind.actividad) for ind in self.poblacion.values())

    def actualizar_estados_actividades(self):
        self.estados_actividades = dict((ind.id, (ind.estado, ind.actividad)) for ind in self.poblacion.values())

    def historia_estado_poblacion(self):
        return [Counter(estados.values()) for estados in self.historia_estados]

    def historia_estado_observado_poblacion(self):
        return [Counter(estados.values()) for estados in self.historia_estados_observados]

    def historia_actividad_poblacion(self):
        return [Counter(actividades.values()) for actividades in self.historia_actividades]

    def historia_estado_actividad_poblacion(self):
        return [Counter(estados_actividades.values()) for estados_actividades in self.historia_estados_actividades]

    def actualizar_probabilidad(self):
        trabajando = [ind for ind in self.poblacion.values() if (ind.actividad == 'trabajo') & (ind.determinar_turno() != 0)]
        infecciosos_trabajando = [ind for ind in trabajando if ind.estado == self.I]

        self.p_contagio_diaria['trabajo'] = self.beta_empresa * len(infecciosos_trabajando) / max(1, len(trabajando))
        # / 2 + 3/1000 * self.beta_poblacion / 2
        # print(self.beta_empresa * len(infecciosos_trabajando) / max(1, len(trabajando)))
        # print(len(infecciosos_trabajando))

    def tick(self):
        for ind in self.poblacion.values():
            ind.tick(self.p_contagio_diaria)

        self.poblacion = dict((ind.id, ind) for ind in self.poblacion.values() if ind.vive)

        self.actualizar_probabilidad()

        self.historia_numero_tests.append(self.numero_tests)

        self.historia_infectados_totales.update({ind.id for ind in self.poblacion.values() if ind.estado == 'infeccioso'})
        self.numero_infectados_totales.append(len(self.historia_infectados_totales))

        self.historia_observados_totales.update({ind.id for ind in self.poblacion.values() if ind.estado == 'infeccioso' and ind.estado_observado == self.I})
        self.numero_observados_totales.append(len(self.historia_observados_totales))

        self.tiempo += 1
        self.dia += dt.timedelta(days=1)

    def terminar(self):
        pass

    def simular(self, algoritmo, tiempo_simulacion):
        while self.tiempo < tiempo_simulacion:
            self.poblacion = algoritmo.decidir(self.poblacion, self.tiempo, self.dia, self.precision_tests)
            self.numero_tests = algoritmo.numero_tests
            self.tick()
            algoritmo.tick()

        self.terminar()

class SimuladorEficiente(SimuladorBase):
    def __init__(self, tamano_poblacion, precision_tests, probabilidad, duracion_infeccion, R0_poblacion, dia_inicial):
        super().__init__(tamano_poblacion, precision_tests, probabilidad, duracion_infeccion, R0_poblacion, dia_inicial)

        self.estados_actividades_obs = dict((ind.id, (ind.estado, ind.estado_observado, ind.actividad, ind.sintomatico)) for ind in self.poblacion.values())
        self.df_estados_actividades_obs = pd.DataFrame.from_dict(Counter(self.estados_actividades_obs.values()), orient='index')
        self.df_estados_actividades_obs['tiempo'] = self.tiempo

    def contar_estados_actividades_obs(self):
        estado_actividad_actual = pd.DataFrame.from_dict(Counter(self.estados_actividades_obs.values()), orient='index')
        estado_actividad_actual['tiempo'] = self.tiempo
        self.df_estados_actividades_obs = pd.concat([self.df_estados_actividades_obs, estado_actividad_actual], axis=0)

    def actualizar_estados_actividades_obs(self):
        self.estados_actividades_obs =  dict((ind.id, (ind.estado, ind.estado_observado, ind.actividad, ind.sintomatico)) for ind in self.poblacion.values())

    def tick(self):
        super().tick()

        self.actualizar_estados_actividades_obs()
        self.contar_estados_actividades_obs()


# when email comes in,
# parse email using email parser
# save to firestore
import base64
import email
import json
import os
import time
from email import encoders
from email.mime.base import MIMEBase
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText

import pandas as pd
from dotenv import load_dotenv
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build
from firebase import firestore

load_dotenv()

# authenticate with gmail api
SCOPES = ['https://www.googleapis.com/auth/gmail.readonly', 'https://www.googleapis.com/auth/gmail.modify']
creds = None
# The file token.json stores the user's access and refresh tokens, and is
# created automatically when the authorization flow completes for the first
# time.
if os.path.exists('token.json'):
    creds = Credentials.from_authorized_user_file('token.json', SCOPES)
# If there are no (valid) credentials available, let the user log in.
if not creds or not creds.valid:
    if creds and creds.expired and creds.refresh_token:
        creds.refresh(Request())
    else:
        flow = InstalledAppFlow.from_client_secrets_file(
            os.getenv('GMAIL_CREDENTIAL_JSON_PATH'), SCOPES)
        creds = flow.run_local_server(port=0)
    # Save the credentials for the next run
    with open('token.json', 'w') as token:
        token.write(creds.to_json())

# Build the Gmail API service
service = build('gmail', 'v1', credentials=creds)

def extract_email():
    unread_emails = get_unread_emails()

def is_csv(filename):
    return filename.lower().endswith('.xlsx')

def get_unread_emails():
    results = service.users().messages().list(userId='me', q='is:unread').execute()
    return results.get('messages', [])

def reply_email(message, reply_content, attachment_path=None):
    reply_message = MIMEMultipart()

    reply_text = MIMEText(reply_content + f'\n\n This is an automated reply.')

    reply_message['To'] = message['From']
    reply_message['Subject'] = 'Re: ' + message['Subject']
    reply_message['In-Reply-To'] = message['Message-ID']
    reply_message['References'] = message['Message-ID']

    reply_message.attach(reply_text)

    try:

        if attachment_path:
            with open(attachment_path, 'rb') as f:
                attachment = MIMEBase('application', 'octet-stream')
                attachment.set_payload(f.read())
                encoders.encode_base64(attachment)
                attachment.add_header('Content-Disposition',
                                      f'attachment; filename="{os.path.basename(attachment_path)}"')
                reply_message.attach(attachment)
    except Exception as e:
        print('Error: Unable to attach file')
        print(e)

    raw = base64.urlsafe_b64encode(reply_message.as_string().encode('utf-8')).decode('utf-8')
    body = {'raw': raw}

    service.users().messages().send(userId='me', body=body).execute()
    print(f'Replied to {message["From"]} about {message["Subject"]}')

def extract_info(message):
    info = {}
    email_id = message['id']
    mark_email_as_read(email_id)
    gmail = service.users().messages().get(userId='me', id=email_id, format='raw').execute()
    msg_bytes = base64.urlsafe_b64decode(gmail['raw'])
    msg = email.message_from_bytes(msg_bytes)

    # Extract the title, and sender email
    info['subject'] = msg['subject']
    info['sender'] = msg['from']

    # Extract the email body
    for part in msg.walk():
        if part.get_content_type() == 'text/plain':
            info['body'] = part.get_payload(decode=True).decode('utf-8')
            break

    # Extract attachments if any
    for part in msg.walk():
        if part.get_filename():
            attachment_data = part.get_payload(decode=True)
            if attachment_data:
                path = part.get_filename()

                if not path.endswith('.xlsx'):
                    reply_email(msg, f'Please ensure that your attachment is an excel file.')
                    continue

                with open(path, 'wb') as f:
                    f.write(attachment_data)

                df = pd.read_excel(path)

                expected_columns = ['items', 'quantity']
                if not set(expected_columns).issubset(df.columns):
                    # reply_email(msg, f'Please ensure that your excel file has the following columns: {expected_columns}')
                    continue

                info['attachment'] = df.set_index('items')['quantity'].to_dict()
    if not info.get('attachment'):
        reply_email(msg,
                    f'Please ensure that your email has an excel attachment following the format. Attached is the template file. Do download and edit the attached file.',
                    r'template.xlsx')
    else:
        reply_email(msg, f'Thank you for your request. We will get back to you shortly.')

    return info

def mark_email_as_read(email_id):
    try:
        # Modify the email by removing the 'UNREAD' label
        service.users().messages().modify(userId='me', id=email_id, body={'removeLabelIds': ['UNREAD']}).execute()
        print(f"Email with ID {email_id} marked as read.")
    except Exception as e:
        print("Error marking email as read:", e)

while True:
    print(f'Checking for new emails...')
    mails = get_unread_emails()
    print(f'Found {len(mails)} new emails.')

    for message in mails:
        info = extract_info(message)

        if not info.get('attachment'):
            continue

        firestore.collection('Requests').add(
            {
                'Subject': info.get('subject'),
                'Sender': info.get('sender'),
                'Body': info.get('body'),
                'Attachments': info.get('attachment') or {},
                'Status': 'pending',
                'Conversation': [],
                'timestamp': time.time_ns() // 1000000
            }
        )

    time.sleep(10)


#!/usr/bin/env/python
# _*_coding:utf-8_*_
# Data:2017-08-13
# Auther:苏莫
# Link:http://blog.csdn.net/lingluofengzang
# PythonVersion:python2.7
# filename:xlsx.py

import sys
# import os
import xlsxwriter  # pip install xlsxwriter

import importlib
import sys
importlib.reload(sys)

# sys.setdefaultencoding("utf-8")
# path = os.path.dirname(os.path.abspath(__file__))

# 建立文件
workbook = xlsxwriter.Workbook("text.xlsx")
# 可以制定表的名字
# worksheet = workbook.add_worksheet('text')
worksheet = workbook.add_worksheet()

# 设置列宽
# worksheet.set_column('A:A',10)
# 设置祖体
bold = workbook.add_format({'bold': True})
# 定义数字格式
# money = workbook.add_format({'num_format':'$#,##0'})

# 写入带粗体的数据
worksheet.write('A1', 'data', bold)
worksheet.write('B1', 'work')
'''
worksheet.write(0, 0, 'Hello')     # write_string()
worksheet.write(1, 0, 'World')     # write_string()
worksheet.write(2, 0, 2)        # write_number()
worksheet.write(3, 0, 3.00001)     # write_number()
worksheet.write(4, 0, '=SIN(PI()/4)')  # write_formula()
worksheet.write(5, 0, '')        # write_blank()
worksheet.write(6, 0, None)       # write_blank()
'''

worksheet.write('A3', 15)
worksheet.write('B3', 20)
worksheet.write('C3', 44)
worksheet.write('D3', 36)
# xlsx计算数据
worksheet.write('E3', '=SUM(A3:D3)')

'''
建立Chart对象： chart = workbook.add_chart({type, 'column'})
Chart: Area, Bar, Column, Doughnut, Line, Pie, Scatter, Stock, Radar
将图插入到sheet中： worksheet.insert_chart('A7', chart)
'''

# 定义插入的图标样式
chart = workbook.add_chart({"type": 'column'})

headings = ['a', 'b', 'c']
data = [
    [1, 2, 3, 4, 5],
    [2, 4, 6, 8, 10],
    [3, 6, 9, 12, 15],
]
# 按行插入数据
worksheet.write_row('A4', headings)
# 按列插入数据
worksheet.write_column('A5', data[0])
worksheet.write_column('B5', data[1])
worksheet.write_column('C5', data[2])
# 图行的数据区
# name：代表图例名称；
# categories：是x轴项，也就是类别；
# values:是y轴项，也就是值；
chart.add_series({
    'name': '=Sheet1!$B$4',
    'categories': '=Sheet1!$A$5:$A$9',
    'values': '=Sheet1!$B$5:$B$9',
})
chart.add_series({
    'name': ['Sheet1', 3, 2],
    'categories': ['Sheet1', 4, 0, 8, 0],
    'values': ['Sheet1', 4, 2, 8, 2],
})
# 图形的标题
chart.set_title({'name': 'Percent Stacked Chart'})
# 图形X轴的说明
chart.set_x_axis({'name': 'Test number'})
# 图形Y轴的说明
chart.set_y_axis({'name': 'Sample length (mm)'})
# 设置图表风格
chart.set_style(11)
# 插入图形,带偏移
worksheet.insert_chart('D12', chart, {'x_offset': 25, 'y_offset': 10})

workbook.close()


class Solution:
    def middleNode(self, head: ListNode) -> ListNode:
        midPoint = 0.5
        curNode = head
        while curNode.next:
            curNode = curNode.next
            midPoint += 0.5
        midPoint = round(midPoint - 0.4)
        curNode = head
        for x in range(midPoint):
            curNode = curNode.next
        return curNode

Coding_Challenges/30 Day Coding Challenge April/Week 2/Day 8) Middle of Linked List.py


from __future__ import print_function
from __future__ import division
from __future__ import absolute_import

import datetime
import json
import mock

from dashboard.pinpoint.handlers import results2
from dashboard.pinpoint.models.results2 import Results2Error
from dashboard.pinpoint import test

class _Results2Test(test.TestCase):

  def setUp(self):
    super().setUp()

    self._job_from_id = mock.MagicMock()
    patcher = mock.patch.object(results2.job_module, 'JobFromId',
                                self._job_from_id)
    self.addCleanup(patcher.stop)
    patcher.start()

  def _SetJob(self, job):
    self._job_from_id.return_value = job

class Results2GetTest(_Results2Test):

  def testGet_InvalidJob_Error(self):
    self._SetJob(None)

    response = self.testapp.get('/api/results2/456', status=400)
    self.assertIn(b'Error', response.body)

  @mock.patch.object(results2.results2, 'GetCachedResults2',
                     mock.MagicMock(return_value=None))
  @mock.patch.object(results2.results2, 'ScheduleResults2Generation',
                     mock.MagicMock(return_value=False))
  def testGet_NotStarted_Incomplete(self):
    self._SetJob(_JobStub('456', started=False))

    result = json.loads(self.testapp.get('/api/results2/456').body)
    self.assertEqual('job-incomplete', result['status'])

  @mock.patch.object(results2.results2, 'GetCachedResults2',
                     mock.MagicMock(return_value='foo'))
  def testGet_AlreadyRunning_UsesCached(self):
    self._SetJob(_JobStub('456'))

    result = json.loads(self.testapp.get('/api/results2/456').body)
    self.assertEqual('complete', result['status'])
    self.assertEqual('foo', result['url'])

  @mock.patch.object(results2.results2, 'GetCachedResults2',
                     mock.MagicMock(return_value=None))
  @mock.patch.object(results2.results2, 'ScheduleResults2Generation',
                     mock.MagicMock(return_value=True))
  def testGet_Schedule_Succeeds(self):
    self._SetJob(_JobStub('456'))

    result = json.loads(self.testapp.get('/api/results2/456').body)
    self.assertEqual('pending', result['status'])

  @mock.patch.object(results2.results2, 'GetCachedResults2',
                     mock.MagicMock(return_value=None))
  @mock.patch.object(results2.results2, 'ScheduleResults2Generation',
                     mock.MagicMock(return_value=False))
  def testGet_Schedule_Fails(self):
    self._SetJob(_JobStub('456'))

    result = json.loads(self.testapp.get('/api/results2/456').body)
    self.assertEqual('failed', result['status'])

@mock.patch.object(results2.results2, 'GenerateResults2')
@mock.patch('dashboard.common.cloud_metric._PublishTSCloudMetric',
            mock.MagicMock())
class Results2GeneratorPostTest(_Results2Test):

  def testGet_CallsGenerate(self, mock_generate):
    self._SetJob(_JobStub('789'))
    self.testapp.post('/api/generate-results2/789')
    self.assertTrue(mock_generate.called)

  def testGet_ReturnsError(self, mock_generate):
    mock_generate.side_effect = Results2Error('foo')
    self._SetJob(_JobStub('101112'))

    response = self.testapp.post('/api/generate-results2/101112', status=400)
    self.assertIn(b'foo', response.body)

class _TaskStub:
  pass

class _JobStub:

  def __init__(self, job_id, started=True, task=None):
    self.job_id = job_id
    self.task = task
    self.started = started
    self.updated = datetime.datetime(2019, 1, 1, 11, 22, 33)

  @property
  def completed(self):
    return self.started and not self.task


import urllib.request, urllib.parse, urllib.error
import json
import datetime
import time
import boto3
import oauth
import twitter as xt

def compute_sleep_time(requests_per_window, max_frac=0.90, window_min=15):
    """
    Return a minimum time interval between requests to
    avoid violating twitter API rules.

    Input
    -----

    requests_per_window : int
        Maximum number of requests allowed per time window.
        Check https://developer.twitter.com/en/docs/twitter-api/v1
        for values.

    max_frac : float
        Factor from 0.0 to 1.0 to multiply `requests_per_window`
        to derive a safer maximum number of requests per window
        (with a buffer from the true limit).

    window_min : int or float
        Number of minutes correponding to one time window interval
        (tipically 15 minutes).

    Return
    ------

    sleep_time : float
        Number of seconds to wait between two consecutive
        API calls.
    """

    window_sec = window_min * 60
    max_req_per_window = int(max_frac * requests_per_window)
    sleep_time = window_sec / max_req_per_window

    return sleep_time

def get_s3_json(bucket, key):
    """
    Load JSON file stored in AWS S3 as a dict, and return it.
    """

    # Load credentials from AWS S3:
    s3 = boto3.client('s3')
    response = s3.get_object(Bucket=bucket, Key=key)
    # Translate S3 object content to a dict:
    string_content = response['Body'].read().decode('utf-8')
    dict_content = json.loads(string_content)

    return dict_content

def augment(url, parameters, credentials, http_method='GET'):
    """
    Create a URL for HTTP request with authentication
    information. These are added as extra parameters.
    """

    # Create an object with the api key and secret key as attributes `key` and `secret`:
    consumer = oauth.OAuthConsumer(credentials['api_key'], credentials['api_secret_key'])
    # Create an object with the token and token secret as attributes; this object has to/from string methods:
    token    = oauth.OAuthToken(credentials['access_token'], credentials['access_token_secret'])

    # Create an object with all the provided information plus OAuth version, timestamp and random number:
    oauth_request = oauth.OAuthRequest.from_consumer_and_token(consumer, token=token, http_method=http_method, http_url=url, parameters=parameters)
    # Create the attribute 'oauth_signature' in the object; this attribute is a authentication signature built from my secret key and the full message to be sent:
    oauth_request.sign_request(oauth.OAuthSignatureMethod_HMAC_SHA1(), consumer, token)

    return oauth_request.to_url()

def captura_de_tweets(id, credentials, max_id=None):
    """
    Given an twitter user `id`, returns the last 200 tweets before the tweet `max_id`.
    If `max_id` is None (default), return the most recent 200 tweets.
    """

    if max_id == None:
        pars = {'user_id': id, 'count':200, 'tweet_mode':'extended'}
    else:
        pars = {'user_id': id, 'count':200, 'max_id': max_id, 'tweet_mode':'extended'}

    url = augment('https://api.twitter.com/1.1/statuses/user_timeline.json', pars, credentials)

    connection = urllib.request.urlopen(url)
    data = connection.read().decode()
    js = json.loads(data)

    return js

def string_to_date(date):
    """
    Convert a twitter created_at string to a datetime object.
    """

    date_time_str = date

    date_time_obj = datetime.datetime.strptime(date_time_str, '%a %b %d %H:%M:%S +0000 %Y')

    return str(date_time_obj.date())

def compare_dates(tweet, str_date, end_date):
        """
        Return True if `tweet` (dict) happened is inside the
        inclusive date interval defined by `str_date` and
        `end_date` (given in format '%Y-%m-%d').
        """

        date = string_to_date(tweet['created_at'])

        if (date <= end_date) and (date >= str_date):
            return True
        else:
            return False

def get_tweets(id, str_date, end_date):

    ''' returns twittes from a user since start_date to end_date limited to 3.200 twittes'''

    # Load twitter OAuth token from AWS S3:
    credentials = get_s3_json('config-lambda', 'gabitwitter.json')

    data = captura_de_tweets(id, credentials)
    date = string_to_date(data[-1]['created_at']) # formata a data do post mais antigo em string
    while date >= str_date:

        max_id = data[-1]['id'] # pega o id do post mais antigo
        data += captura_de_tweets(id, credentials, max_id)
        date = string_to_date(data[-1]['created_at'])

    # Adiciona a lista de twites apenas os twittes dentro da data especificada
    dados = list(filter(lambda x: compare_dates(x, str_date, end_date), data))

    return dados

def get_list_tweets(list_id, str_date, end_date, requests_per_window=900, verbose=False):
    """
    Returns tweets posted in Twitter list `list_id` (int)
    since `str_date` up to `end_date` (note that there is a
    tweet age limit to the Twiter API).

    To avoid violaring Twitter API request limits, set
    `requests_per_window` to the API limit.
    """

    # Load twitter OAuth token from AWS S3:
    if verbose:
        print('Loading credentials...')
    credentials = get_s3_json('config-lambda', 'twitter_compartwi.json')

    # Config for API requests:
    endpoint = 'https://api.twitter.com/1.1/lists/statuses.json'
    pars = {'list_id': list_id, 'count': 200, 'include_entities': True, 'include_rts': True, 'tweet_mode': 'extended'}

    # Get tweets in list:
    if verbose:
        print('Making the first API request...')
    data = xt.request_twitter_api(endpoint, pars, credentials=credentials)
    if verbose:
        print('# tweets: {}'.format(len(data)))
    # Set max_id to skip already captured tweets:
    pars['max_id'] = data[-1]['id'] - 1

    if verbose:
        print('Prior do loop:')
    # Get date of oldest captured tweet:
    date = string_to_date(data[-1]['created_at'])
    while date >= str_date:

        if verbose:
            print('LOOP: current capture <date> <tweet_id>: {} {}'.format(date, pars['max_id']))

        # Sleep to avoid violating Twitter maximum request rates:
        if verbose:
            print('LOOP: sleeping...')
        sleep_time = compute_sleep_time(requests_per_window, window_min=15)
        time.sleep(sleep_time)

        # Configure the request to start from the oldest tweet:
        if verbose:
            print('LOOP: making API request...')
        new_data = xt.request_twitter_api(endpoint, pars, credentials=credentials)
        if verbose:
            print('# tweets: {}'.format(len(new_data)))

        # Exit loop if requests do not return new data (probably due to limit in API historical response):
        if len(new_data) == 0 or new_data[-1]['id'] - 1 == pars['max_id']:
            break

        # Update data to continue loop:
        data += new_data
        pars['max_id'] = data[-1]['id'] - 1
        date = string_to_date(data[-1]['created_at'])

    # Adiciona a lista de twites apenas os twittes dentro da data especificada
    if verbose:
        print('Filtering results by date...')
    dados = list(filter(lambda x: compare_dates(x, str_date, end_date), data))

    return dados


# from github import Github
import numpy as np
import pandas as pd
import pickle as pickle
# import pprint

from fake_useragent import UserAgent
import os
import requests
import time as time
import re
import random
import json
# import urllib
import math

ua = UserAgent()

list_of_content_pages = pd.read_pickle('lst_of_content_urls.pkl')

def get_readme_links(contents_links):
    contents_list = []

    z = requests.get('https://api.github.com/rate_limit').json()
    reset_time = z['resources']['core']['reset']

    for idx, repo in enumerate(lst_of_content_urls):
        if reset_time >= math.floor(time.time()):
            time.sleep(60)
            z = requests.get('https://api.github.com/rate_limit').json()
            reset_time = z['resources']['core']['reset']

        headers = {'User-Agent': ua.random}

        response = requests.get(repo[1], headers)
        response_json = json.loads(response.text)

        lst_of_filenames = [ doc["name"].lower() for idx, doc in enumerate(response_json) ]
        idx_of_readme = -1

        for i, x in enumerate(lst_of_filenames):
                if 'readme' in x:
                    idx_of_readme = i
                else:
                    continue
        try:
            contents_list.append([ repo[0], response_json[idx_of_readme]['download_url'] ])
        except:
            contents_list.append([ repo[0], None])

        z = requests.get('https://api.github.com/rate_limit').json()
        reset_time = z['resources']['core']['reset']

        print(str(idx / len(contents_links)) + '%' + '\n')

        print(contents_list[idx])
        with open('contents_list.pkl', 'wb') as f:
            pickle.dump(contents_list, f)

    return contents_list

    x = get_readme_links(list_of_content_pages)

    with open('contents_list_final.pkl', 'wb') as f:
            pickle.dump(x, f)


import os
import sys

testRoot = os.path.join(os.environ.get('SUMO_HOME', '.'), 'tests')
neteditTestRoot = os.path.join(
    os.environ.get('TEXTTEST_HOME', testRoot), 'netedit')
sys.path.append(neteditTestRoot)
import neteditTestFunctions as netedit

# Open netedit
neteditProcess, match = netedit.setupAndStart(neteditTestRoot, False)

# go to select mode
netedit.selectMode()

# select all busStops with lanes that contains "E1_0" and remove it
netedit.selectItems("Additional", "busStop", "lane", "E0_0")
netedit.deleteSelectedItems()

# select all busStops with id "busStop_IDToRemove1" and remove it
netedit.selectItems("Additional", "busStop", "id", "=busStop_IDToRemove1")
netedit.deleteSelectedItems()

# select all busStops with id "busStop_IDToRemove2" and remove it
netedit.selectItems("Additional", "busStop", "id", "=busStop_IDToRemove2")
netedit.deleteSelectedItems()

# select all busStops with startPos greather than 15
netedit.selectItems("Additional", "busStop", "startPos", ">14")

# change to remove mode
netedit.modificationModeRemove()

# select all busStops with startPos greather than 25
netedit.selectItems("Additional", "busStop", "startPos", ">24")

# change to add mode and remove selected busStops
netedit.modificationModeAdd()
netedit.deleteSelectedItems()
netedit.abortSelection()

# select all busStops with endPos minor that 30 and delete it
netedit.selectItems("Additional", "busStop", "endPos", "<36")
netedit.deleteSelectedItems()
netedit.abortSelection()

# select all busStops with lines that contains "lineToRemove" and remove it
netedit.selectItems("Additional", "busStop", "lines", "lineToRemove")
netedit.deleteSelectedItems()
netedit.abortSelection()

# select all busStops with lines that contains "linetoRemove" and remove it
netedit.selectItems("Additional", "busStop", "lines", "linetoRemove")
netedit.deleteSelectedItems()
netedit.abortSelection()

# save additionals
netedit.saveAdditionals()

# save newtork
netedit.saveNetwork()

# quit netedit
netedit.quit(neteditProcess)


#Author: Miguel Martinez Cano
#Homework Number & Name: Homework #9: Serendipity 3 - New System
#Due Date: Monday, November 14, 2016 at 5:00 P.M.
#Ice Cream
#Class Name: Ice Cream
#List of Attributes: Size, Price
#List of Methods:__init__(name, kind), get_size(), update_size(new_size),
#get_price(), update_price(new_price), calculate_cost(), __str__()

#Define global constants
SMALL_PRICE = 1.49
MEDIUM_PRICE = 1.99
LARGE_PRICE = 2.49

#Import superclass
import dessert

#Define the Ice Cream class - inherits from Dessert
class IceCream(dessert.Dessert):
    #Initializer method
    def __init__(self, name, kind):

        #Call the initializor of parent and pass parent attribute values
        super().__init__(name, kind)

        #Set default values for Ice Cream attributes
        self.__size = "S"
        self.__price = 1

    #Retreive the ice cream's size
    def get_size(self):
        return self.__size

    #Retreive the ice cream's price
    def get_price(self):
        return self.__price

    #Mutators to allow code outside of IceCream to update the values
    #of the hidden attributes

    #Update the ice cream's size
    def update_size(self, new_size):
        #Ensure size is valid
        if new_size == "S" or new_size == "M" or new_size == "L":
            #Change the size of the ice cream
            self.__size = new_size

            #Call the function to change the price of the ice cream
            self.update_price(self.__size)

            return True
        else:
            #Print a message that the size was not changed.
            print ("Invalid size. Size was not changed.")
            return False

    #Update the ice cream's price
    def update_price(self, size):
        #Change price for a small
        if size =="S":
            self.__price = SMALL_PRICE
        #Change prie for a medium
        elif size == "M":
            self.__price = MEDIUM_PRICE
        #Change price for a large
        elif size == "L":
            self.__price = LARGE_PRICE
        else:
            #Print a message that the price was not changed
            print ("Invalid price. Price was not changed.")

    #Calculate the cost of the ice cream
    def calculate_cost(self):
        return self.__price

    #Create a string with the values of the object's attributes
    def __str__(self):

        cost = self.calculate_cost()

        ice_cream_print = super().__str__() + " $" + str(cost)

        return ice_cream_print


import cv2
import numpy as np
import matplotlib as plt

ASCENDING = 1
LEVELED = 0
DESCENDING =-1

def rotate(img, angle):
    M  = cv2.getRotationMatrix2D((img.shape[1]//2, img.shape[0]//2), angle, 1)
    return cv2.warpAffine(img, M, (img.shape[1],img.shape[0]),borderValue = 0)

def BaselineFeature(img):

    img[img == 0] = 1
    img[img == 255] = 0

    center = img.shape[0]//2
    top = center - img.shape[0]//5
    bottom = center + img.shape[0]//5

    best = 0
    angle = 0
    for i in range(-30,30):
        imgx = rotate(img, i)
        s = np.sum(imgx[top:bottom, 0:img.shape[1]])
        if s > best:
            best = s
            angle = i
            plt.pyplot.imshow(imgx, cmap="binary")

    if angle < -5:
        return ASCENDING, angle
    elif angle > 5:
        return DESCENDING, angle
    return LEVELED, angle


# Membuat fungsi sederhana
def suaraayam():
    print("Kukuruyuk")

def hargaayam():
    suaraayam()
    print("Harga ayam per 1 kg adalah Rp 20.000,-")

# Membuat fungsi dengan input argument
def hargatotalayam(kg):
    suaraayam()
    harga = 20000
    hargatotal = kg * harga
    print("Harga",kg,"kg ayam adalah",hargatotal)

hargaayam()
hargatotalayam(2)
hargatotalayam(5)
hargatotalayam(0.5)
hargatotalayam(1)

print("\n")
# Fungsi dengan argumen sederhana
def siswa(nama):
    print("Nama siswa adalah",nama)

siswa("Mario")

print("\n")
# Fungsi dengan menggunakan keywords arguments
def guru(nama,pelajaran):
    print("Guru ini bernama",nama)
    print("Guru ini mengajar",pelajaran)

guru(nama="Popong",pelajaran="seni rupa")
guru(pelajaran="olahraga",nama="Wawan")
guru("matematika","Raihan") # Ini adalah contoh yang salah

print("\n")
# Fungsi dengan menggunakan default arguments
def penjagasekolah(nama,shift="Siang",galak="Galak"):
    print("Penjaga sekolah ini bernama",nama)
    print("Shiftnya adalah",shift)
    print("Galak atau tidak:",galak)

penjagasekolah("Mathias")
penjagasekolah("Maman",shift="Malam")
penjagasekolah("Asep",galak="sangat")
#penjagasekolah(shift="Pagi",galak="sangat") # Contoh yang menghasilkan error


from collections import Counter
with open('input.txt') as f:
	polymer,rules = f.read().split('\n\n')
	polymer = list(polymer.rstrip())
	rule_dict = dict()
	for rule in rules.split('\n'):
		key,value= rule.split(' -> ')
		rule_dict[key] = value
	steps = 10
	for i in range(steps):
		pairs=[]
		to_add= ""
		for p_id in range(len(polymer)):
			if p_id + 2 <= len(polymer):
				pairs.append((polymer[p_id:p_id+2]))
			else:
				to_add = polymer[p_id:p_id+2][0]
		# print(pairs)
		for p_id in range(len(pairs)):
			insert_letter = rule_dict[''.join(pairs[p_id])]
			pairs[p_id].insert(1,insert_letter)
		flat_list = [item for sublist in pairs for item in sublist[:2]]
		polymer = flat_list
		polymer.append(to_add)
		print(len(polymer))
	counted = Counter(polymer).values()
	print(max(counted)-min(counted))

	# print(len(polymer))


'''
The default exp_name is tmp. Change it before formal training! isic2018 PH2 DMF SKD
nohup python -u multi_train_adapt.py --exp_name test --config_yml Configs/multi_train_local.yml --model MedFormer --batch_size 16 --adapt_method False --num_domains 1 --dataset PH2  --k_fold 4 > 4MedFormer_PH2.out 2>&1 &
'''
import argparse
from sqlite3 import adapt
import yaml
import os, time
from datetime import datetime

import pandas as pd
import torch.nn as nn
import torch.utils.data
import torch.optim as optim
import medpy.metric.binary as metrics
from torch.utils.tensorboard import SummaryWriter

from Datasets.create_dataset import Dataset_wrap_csv
from Utils.losses import dice_loss
from Utils.pieces import DotDict

torch.cuda.empty_cache()

def main(config):
    # set gpu
    device_ids = range(torch.cuda.device_count())

    # prepare train, val, test datas
    train_loaders = {}  # initialize data loaders
    val_loaders = {}
    test_loaders = {}
    for dataset_name in config.data.name:
        datas = Dataset_wrap_csv(k_fold=config.data.k_fold, use_old_split=True, img_size=config.data.img_size,
            dataset_name = dataset_name, split_ratio=config.data.split_ratio,
            train_aug=config.data.train_aug, data_folder=config.data.data_folder)
        train_data, val_data, test_data = datas['train'], datas['test'], datas['test']

        train_loader = torch.utils.data.DataLoader(train_data,
                                                batch_size=config.train.batch_size,
                                                shuffle=True,
                                                num_workers=config.train.num_workers,
                                                pin_memory=True,
                                                drop_last=True)
        val_loader = torch.utils.data.DataLoader(val_data,
                                                batch_size=config.test.batch_size,
                                                shuffle=False,
                                                num_workers=config.test.num_workers,
                                                pin_memory=True,
                                                drop_last=False)
        test_loader = torch.utils.data.DataLoader(test_data,
                                                batch_size=config.test.batch_size,
                                                shuffle=False,
                                                num_workers=config.test.num_workers,
                                                pin_memory=True,
                                                drop_last=False)
        train_loaders[dataset_name] = train_loader
        val_loaders[dataset_name] = val_loader
        test_loaders[dataset_name] = test_loader
        print('{} has {} training samples'.format(dataset_name, len(train_loader.dataset)))
    print('{} k_folder, {} val'.format(config.data.k_fold, config.data.use_val))

    # prepare model
    if config.model == 'SwinSeg':
        from Models.Transformer.Swin_adapters import SwinSimpleSeg_adapt
        model = SwinSimpleSeg_adapt(img_size=config.data.img_size,pretrained=True,pretrained_swin_name=config.swin.name,
                                pretrained_folder=config.pretrained_folder,
                              embed_dim=config.swin.EMBED_DIM,drop_path_rate=config.swin.DROP_PATH_RATE,
                              depths=config.swin.DEPTHS,num_heads=config.swin.NUM_HEADS,window_size=config.swin.WINDOW_SIZE,
                               debug=config.debug, adapt_method=False, num_domains=K)
        # freeze some parameters
        # for name, param in model.encoder.named_parameters():
        #     param.requires_grad = False
    elif config.model == 'SwinSeg_adapt':
        from Models.Transformer.Swin_adapters import SwinSimpleSeg_adapt
        model = SwinSimpleSeg_adapt(img_size=config.data.img_size,pretrained=True,pretrained_swin_name=config.swin.name,
                                pretrained_folder=config.pretrained_folder,
                              embed_dim=config.swin.EMBED_DIM,drop_path_rate=config.swin.DROP_PATH_RATE,
                              depths=config.swin.DEPTHS,num_heads=config.swin.NUM_HEADS,window_size=config.swin.WINDOW_SIZE,
                               debug=config.debug, adapt_method=config.model_adapt.adapt_method, num_domains=K)
        # freeze some parameters
        for name, param in model.encoder.named_parameters():
            if 'adapter' not in name and 'norm' not in name:
                param.requires_grad = False
    elif config.model == 'SwinSeg_CNNprompt_adapt':
        from Models.Transformer.Swin_adapters import SwinSimpleSeg_CNNprompt_adapt
        model = SwinSimpleSeg_CNNprompt_adapt(img_size=config.data.img_size,pretrained=True,pretrained_swin_name=config.swin.name,
                                pretrained_folder=config.pretrained_folder,
                              embed_dim=config.swin.EMBED_DIM,drop_path_rate=config.swin.DROP_PATH_RATE,
                              depths=config.swin.DEPTHS,num_heads=config.swin.NUM_HEADS,window_size=config.swin.WINDOW_SIZE,
                               debug=config.debug, adapt_method=config.model_adapt.adapt_method, num_domains=K)
        # freeze some parameters
        for name, param in model.encoder.named_parameters():
            if 'adapter' not in name and 'norm' not in name:
                param.requires_grad = False
    elif config.model == 'ViTSeg':
        from Models.Transformer.ViT_adapters import ViTSeg_adapt
        model = ViTSeg_adapt(pretrained=True, pretrained_vit_name=config.vit.name,
        pretrained_folder=config.pretrained_folder,img_size=config.data.img_size, patch_size=config.vit.patch_size,
        embed_dim=config.vit.embed_dim, depth=config.vit.depth, num_heads=config.vit.num_heads,
        mlp_ratio=config.vit.mlp_ratio, drop_rate=config.vit.dropout_rate,
        attn_drop_rate=config.vit.attention_dropout_rate, drop_path_rate=0.2,
        debug=config.debug, adapt_method=False, num_domains=K)
        # for name, param in model.encoder.named_parameters():
        #     param.requires_grad = False
    elif config.model == 'ViTSeg_adapt':
        from Models.Transformer.ViT_adapters import ViTSeg_adapt
        model = ViTSeg_adapt(pretrained=True, pretrained_vit_name=config.vit.name,
        pretrained_folder=config.pretrained_folder,img_size=config.data.img_size, patch_size=config.vit.patch_size,
        embed_dim=config.vit.embed_dim, depth=config.vit.depth, num_heads=config.vit.num_heads,
        mlp_ratio=config.vit.mlp_ratio, drop_rate=config.vit.dropout_rate,
        attn_drop_rate=config.vit.attention_dropout_rate, drop_path_rate=0.2,
        debug=config.debug, adapt_method=config.model_adapt.adapt_method, num_domains=K)
        for name, param in model.encoder.named_parameters():
            if 'adapter' not in name and 'norm' not in name:
                param.requires_grad = False
    elif config.model == 'ViTSeg_CNNprompt_adapt':
        from Models.Transformer.ViT_adapters import ViTSeg_CNNprompt_adapt
        model = ViTSeg_CNNprompt_adapt(pretrained=True, pretrained_vit_name=config.vit.name,
        pretrained_folder=config.pretrained_folder,img_size=config.data.img_size, patch_size=config.vit.patch_size,
        embed_dim=config.vit.embed_dim, depth=config.vit.depth, num_heads=config.vit.num_heads,
        mlp_ratio=config.vit.mlp_ratio, drop_rate=config.vit.dropout_rate,
        attn_drop_rate=config.vit.attention_dropout_rate, drop_path_rate=0.2,
        debug=config.debug, adapt_method=config.model_adapt.adapt_method, num_domains=K)
        for name, param in model.encoder.named_parameters():
            if 'adapter' not in name and 'norm' not in name:
                param.requires_grad = False
    elif config.model =='AdaptFormer':
        from Models.Transformer.AdapterFormer import AdaptFormer
        model = AdaptFormer(pretrained=True, pretrained_vit_name=config.vit.name,
        pretrained_folder=config.pretrained_folder,img_size=config.data.img_size, patch_size=config.vit.patch_size,
        embed_dim=config.vit.embed_dim, depth=config.vit.depth, num_heads=config.vit.num_heads,
        mlp_ratio=config.vit.mlp_ratio, drop_rate=config.vit.dropout_rate,
        attn_drop_rate=config.vit.attention_dropout_rate, drop_path_rate=0.2,
        debug=config.debug, adapt_method=config.model_adapt.adapt_method, num_domains=K)
        for name, param in model.encoder.named_parameters():
            if 'adapter' not in name:
                param.requires_grad = False
    elif config.model == 'DeiTSeg':
        from Models.Transformer.ViT_adapters import ViTSeg_adapt
        model = ViTSeg_adapt(pretrained=True, pretrained_vit_name=config.deit.name,
        pretrained_folder=config.pretrained_folder,img_size=config.data.img_size, patch_size=config.deit.patch_size,
        embed_dim=config.deit.embed_dim, depth=config.deit.depth, num_heads=config.deit.num_heads,
        mlp_ratio=config.deit.mlp_ratio, drop_rate=config.deit.dropout_rate,
        attn_drop_rate=config.deit.attention_dropout_rate, drop_path_rate=0.2,
        debug=config.debug, adapt_method=False, num_domains=K)
        # for name, param in model.encoder.named_parameters():
        #     param.requires_grad = False
    elif config.model == 'DeiTSeg_CNNprompt_adapt':
        from Models.Transformer.ViT_adapters import ViTSeg_CNNprompt_adapt4
        model = ViTSeg_CNNprompt_adapt4(pretrained=True, pretrained_vit_name=config.deit.name,
        pretrained_folder=config.pretrained_folder,img_size=config.data.img_size, patch_size=config.deit.patch_size,
        embed_dim=config.deit.embed_dim, depth=config.deit.depth, num_heads=config.deit.num_heads,
        mlp_ratio=config.deit.mlp_ratio, drop_rate=config.deit.dropout_rate,
        attn_drop_rate=config.deit.attention_dropout_rate, drop_path_rate=0.2,
        debug=config.debug, adapt_method=config.model_adapt.adapt_method, num_domains=K)
        for name, param in model.encoder.named_parameters():
            if 'adapter' not in name and 'norm' not in name:
                param.requires_grad = False
    elif config.model == 'VPT':
        from Models.Transformer.ViT_prompt import ViTSeg_prompt
        model = ViTSeg_prompt(pretrained=True, pretrained_vit_name=config.vit.name,
        pretrained_folder=config.pretrained_folder,img_size=config.data.img_size, patch_size=config.vit.patch_size,
        embed_dim=config.vit.embed_dim, depth=config.vit.depth, num_heads=config.vit.num_heads,
        mlp_ratio=config.vit.mlp_ratio, drop_rate=config.vit.dropout_rate,
        attn_drop_rate=config.vit.attention_dropout_rate, drop_path_rate=0.2,
        debug=config.debug, adapt_method=False, num_domains=K,prompt_len=100,prompt_drop_rate=0.1)
        for name, param in model.encoder.named_parameters():
            if 'prompt' not in name:
                param.requires_grad = False
    elif config.model == 'TransFuse':
        from Models.Hybrid_models.TransFuseFolder.TransFuse import TransFuse_L
        model = TransFuse_L(pretrained=True, pretrained_folder=config.pretrained_folder)
    elif config.model == 'FAT-Net':
        from Models.Hybrid_models.FAT_Net import FAT_Net
        model = FAT_Net()
    elif config.model == 'SwinUNETR':
        from monai.networks.nets import SwinUNETR
        model = SwinUNETR(img_size=(224,224), in_channels=3, out_channels=1, feature_size=48, use_checkpoint=False, spatial_dims=2)
    elif config.model == 'UNETR':
        from monai.networks.nets import UNETR
        model = UNETR(img_size=224, in_channels=3, out_channels=1, spatial_dims=2)
    elif config.model == 'H2Former':
        from Models.Hybrid_models.H2FormerFolder.H2Former import res34_swin_MS
        model = res34_swin_MS(image_size=224,num_class=1,pretrained=True,pretrained_folder=config.pretrained_folder)
    elif config.model == 'UTNet':
        from Models.Hybrid_models.UTNetFolder.UTNet import UTNet
        model = UTNet(in_chan=3,base_chan=32,num_classes=1,reduce_size=config.data.img_size//32,block_list='1234',num_blocks=[1,1,1,1],
        num_heads=[4,4,4,4], projection='interp', attn_drop=0.1, proj_drop=0.1, rel_pos=True, aux_loss=False, maxpool=True)
    elif config.model == 'MedFormer':
        from Models.Hybrid_models.MedFormerFolder.MedFormer import MedFormer
        model = MedFormer(in_chan=3, num_classes=1, base_chan=32, conv_block='BasicBlock', conv_num=[2,0,0,0, 0,0,2,2], trans_num=[0,2,2,2, 2,2,0,0],
        num_heads=[1,4,8,16, 8,4,1,1], fusion_depth=2, fusion_dim=512, fusion_heads=16, map_size=3,
        proj_type='depthwise', act=nn.GELU, expansion=2, attn_drop=0., proj_drop=0.)
    elif config.model == 'SwinUnet':
        from Models.Transformer.SwinUnet import SwinUnet
        model  = SwinUnet(img_size=config.data.img_size)

    total_trainable_params = sum(
                    p.numel() for p in model.parameters() if p.requires_grad)
    total_params = sum(p.numel() for p in model.parameters())
    print('{}M total parameters'.format(total_params/1e6))
    print('{}M total trainable parameters'.format(total_trainable_params/1e6))

    # from thop import profile
    # input = torch.randn(1,3,224,224)
    # flops, params = profile(model, (input,))
    # print(f"total flops : {flops/1e9} G")

    # test model
    x = torch.randn(5,3,224,224)
    y = model(x)
    print(y.shape)

    # model = model.cuda()

    # ## If multiple GPUs
    # if len(device_ids) > 1:
    #     model = torch.nn.DataParallel(model).cuda()

    # criterion = [nn.BCELoss(), dice_loss]

    # # only test
    # if config.test.only_test == True:
    #     test(config, model, config.test.test_model_dir, test_loaders, criterion)
    # else:
    #     train_val(config, model, train_loaders, val_loaders, criterion)
    #     test(config, model, best_model_dir, test_loaders, criterion)

# =======================================================================================================
def train_val(config, model, train_loaders, val_loaders, criterion):
    # optimizer loss
    if config.train.optimizer.mode == 'adam':
        # optimizer = optim.Adam(model.parameters(), lr=float(config.train.optimizer.adam.lr))
        print('choose wrong optimizer')
    elif config.train.optimizer.mode == 'adamw':
        optimizer = optim.AdamW(filter(lambda p: p.requires_grad, model.parameters()),lr=float(config.train.optimizer.adamw.lr),
                                weight_decay=float(config.train.optimizer.adamw.weight_decay))
    scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=50, gamma=0.5)

    # ---------------------------------------------------------------------------
    # Training and Validating
    #----------------------------------------------------------------------------
    epochs = config.train.num_epochs
    max_iou = 0 # use for record best model
    best_epoch = 0 # use for recording the best epoch
    # create training data loading iteration
    train_iters = {}
    for dataset_name in train_loaders.keys():
        train_iters[dataset_name] = iter(train_loaders[dataset_name])
    if config.train.num_iters:
        iterations = config.train.num_iters
    else:
        iterations = max([len(train_loaders[x]) for x in train_iters.keys()])

    torch.save(model.state_dict(), best_model_dir)
    for epoch in range(epochs):
        start = time.time()
        # ----------------------------------------------------------------------
        # train
        # ---------------------------------------------------------------------
        model.train()
        for train_step in range(epoch*iterations, (epoch+1)*iterations):
            # for each dataset, get one minibatch, get loss, sum all losses together
            # update once
            datas_loss_list = []  #record loss for datasets
            dice_train_list = []
            iou_train_list = []
            for dataset_name in config.data.name:
                try:
                    batch = next(train_iters[dataset_name])
                except StopIteration:
                    train_iters[dataset_name] = iter(train_loaders[dataset_name])
                    batch = next(train_iters[dataset_name])
                img = batch['image'].cuda().float()
                label = batch['label'].cuda().float()
                # domain_label = batch['set_id']
                if K == 1:
                    d = '0'
                else:
                    d = str(data2domain[dataset_name])
                # domain_label = torch.nn.functional.one_hot(domain_label, 4).float().cuda()
                other_models = set(['SwinUNETR','UNETR','SwinUnet','MedFormer','UTNet'])
                if config.model in other_models:
                    output = model(img)
                else:
                    output = model(img,d=d)['seg']

                output = torch.sigmoid(output)

                # calculate loss
                assert (output.shape == label.shape)
                losses = []
                for function in criterion:
                    losses.append(function(output, label))
                loss = sum(losses)
                datas_loss_list.append(loss)

                # calculate metrics
                with torch.no_grad():
                    output = output.cpu().numpy() > 0.5
                    label = label.cpu().numpy()
                    assert (output.shape == label.shape)
                    dice_train = metrics.dc(output, label)
                    iou_train = metrics.jc(output, label)
                    dice_train_list.append(dice_train)
                    iou_train_list.append(iou_train)

                # logging per batch
                # writer.add_scalar('Train/{}/BCEloss'.format(dataset_name), losses[0].item(), train_step)
                # writer.add_scalar('Train/{}/Diceloss'.format(dataset_name), losses[1].item(), train_step)
                writer.add_scalar('Train/{}/loss'.format(dataset_name), loss.item(), train_step)
                # writer.add_scalar('Train/{}/Di_score'.format(dataset_name), dice_train, train_step)
                writer.add_scalar('Train/{}/IOU'.format(dataset_name), iou_train, train_step)

            # backward
            multi_loss = sum(datas_loss_list)
            optimizer.zero_grad()
            multi_loss.backward()
            optimizer.step()

            # logging average per batch
            writer.add_scalar('Train/Average/sum_loss',multi_loss.item(), train_step)
            # writer.add_scalar('Train/Average/Di_score', sum(dice_train_list)/len(dice_train_list), train_step)
            writer.add_scalar('Train/Average/IOU', sum(iou_train_list)/len(iou_train_list), train_step)

            # end one training batch
            if config.debug: break

            # print
        print('Epoch {}, Total train step {} || sum_loss: {}, Avg Dice score: {}, Avg IOU: {}'.
        format(epoch, train_step, round(multi_loss.item(),5), round(sum(dice_train_list)/len(dice_train_list),4),
        round(sum(iou_train_list)/len(iou_train_list),4)))
        print('Datasets: ', config.data.name, ' || loss: ', [round(x.item(), 4) for x in datas_loss_list],
        ' || Dice score: ', [round(x, 4) for x in dice_train_list],
            ' || IOU: ', [round(x, 4) for x in iou_train_list])

        # -----------------------------------------------------------------
        # validate
        # ----------------------------------------------------------------
        model.eval()
        dice_val_list = []  # record results for each dataset
        iou_val_list = []
        loss_val_list = []
        # eval each dataset
        for dataset_name in config.data.name:
            dice_val_sum= 0
            iou_val_sum = 0
            loss_val_sum = 0
            num_val = 0
            for batch_id, batch in enumerate(val_loaders[dataset_name]):
                img = batch['image'].cuda().float()
                label = batch['label'].cuda().float()
                # domain_label = batch['set_id']
                if K == 1:
                    d = '0'
                else:
                    d = str(data2domain[dataset_name])
                # domain_label = torch.nn.functional.one_hot(domain_label, 4).float().cuda()
                batch_len = img.shape[0]

                with torch.no_grad():
                    other_models = set(['SwinUNETR','UNETR'])
                    if config.model in other_models:
                        output = model(img)
                    else:
                        output = model(img,d=d)['seg']
                    output = torch.sigmoid(output)

                    # calculate loss
                    assert (output.shape == label.shape)
                    losses = []
                    for function in criterion:
                        losses.append(function(output, label))
                    loss_val_sum += sum(losses)*batch_len

                    # calculate metrics
                    output = output.cpu().numpy() > 0.5
                    label = label.cpu().numpy()
                    dice_val_sum += metrics.dc(output, label)*batch_len
                    iou_val_sum += metrics.jc(output, label)*batch_len

                    num_val += batch_len
                    # end one val batch
                    if config.debug: break

            # logging per epoch for one dataset
            loss_val_epoch, dice_val_epoch, iou_val_epoch = loss_val_sum/num_val, dice_val_sum/num_val, iou_val_sum/num_val
            dice_val_list.append(dice_val_epoch)
            loss_val_list.append(loss_val_epoch.item())
            iou_val_list.append(iou_val_epoch)
            writer.add_scalar('Val/{}/loss'.format(dataset_name), loss_val_epoch.item(), epoch)
            writer.add_scalar('Val/{}/Di_score'.format(dataset_name), dice_val_epoch, epoch)
            writer.add_scalar('Val/{}/IOU'.format(dataset_name), iou_val_epoch, epoch)

        # logging average per epoch
        writer.add_scalar('Val/Average/sum_loss', sum(loss_val_list), epoch)
        writer.add_scalar('Val/Average/Di_score', sum(dice_val_list)/len(dice_val_list), epoch)
        writer.add_scalar('Val/Average/IOU', sum(iou_val_list)/len(iou_val_list), epoch)
        # print
        print('Epoch {}, Validation || sum_loss: {}, Avg Dice score: {}, Avg IOU: {}'.
                format(epoch, round(sum(loss_val_list),5),
                round(sum(dice_val_list)/len(dice_val_list),4), round(sum(iou_val_list)/len(iou_val_list),4)))
        print('Datasets: ', config.data.name, ' || loss: ', [round(x, 4) for x in loss_val_list],
        ' || Dice score: ', [round(x, 4) for x in dice_val_list],
         ' || IOU: ', [round(x, 4) for x in iou_val_list])

        # scheduler step, record lr
        writer.add_scalar('Lr', scheduler.get_last_lr()[0], epoch)
        scheduler.step()

        # store model using the average iou
        avg_val_iou_epoch = sum(iou_val_list)/len(iou_val_list)
        if avg_val_iou_epoch > max_iou:
            torch.save(model.state_dict(), best_model_dir)
            max_iou = avg_val_iou_epoch
            best_epoch = epoch
            print('New best epoch {}!==============================='.format(epoch))

        end = time.time()
        time_elapsed = end-start
        print('Training and evaluating on epoch{} complete in {:.0f}m {:.0f}s'.
            format(epoch, time_elapsed // 60, time_elapsed % 60))

        # end one epoch
        if config.debug: return

    print('Complete training ---------------------------------------------------- \n The best epoch is {}'.format(best_epoch))

    return

# ========================================================================================================
def test(config, model, model_dir, test_loaders, criterion):
    model.load_state_dict(torch.load(model_dir))
    model.eval()
    dice_test_list = []  # record results for each dataset
    iou_test_list = []
    loss_test_list = []
    # test each dataset
    for dataset_name in config.data.name:
        dice_test_sum= 0
        iou_test_sum = 0
        loss_test_sum = 0
        num_test = 0
        for batch_id, batch in enumerate(test_loaders[dataset_name]):
            img = batch['image'].cuda().float()
            label = batch['label'].cuda().float()
            # domain_label = batch['set_id']
            if K == 1:
                d = '0'
            else:
                d = str(data2domain[dataset_name])
            batch_len = img.shape[0]
            other_models = set(['SwinUNETR','UNETR'])
            with torch.no_grad():
                if config.model in other_models:
                    output = model(img)
                else:
                    output = model(img,d=d)['seg']
                output = torch.sigmoid(output)

                # calculate loss
                assert (output.shape == label.shape)
                losses = []
                for function in criterion:
                    losses.append(function(output, label))
                loss_test_sum += sum(losses)*batch_len

                # calculate metrics
                output = output.cpu().numpy() > 0.5
                label = label.cpu().numpy()
                dice_test_sum += metrics.dc(output, label)*batch_len
                iou_test_sum += metrics.jc(output, label)*batch_len

                num_test += batch_len
                # end one test batch
                if config.debug: break

        # logging results for one dataset
        loss_test_epoch, dice_test_epoch, iou_test_epoch = loss_test_sum/num_test, dice_test_sum/num_test, iou_test_sum/num_test
        dice_test_list.append(dice_test_epoch)
        loss_test_list.append(loss_test_epoch.item())
        iou_test_list.append(iou_test_epoch)

    # logging average and store results
    dataset_name_list = config.data.name+['Total']
    loss_test_list.append(sum(loss_test_list))
    dice_test_list.append(sum(dice_test_list)/len(dice_test_list))
    iou_test_list.append(sum(iou_test_list)/len(iou_test_list))
    df = pd.DataFrame({
        'Name': dataset_name_list,
        'loss': loss_test_list,
        'Di_score': dice_test_list,
        'IOU': iou_test_list
    })
    df.to_csv(test_results_dir, index=False)

    # print
    print('========================================================================================')
    print('Test || Average loss: {}, Dice score: {}, IOU: {}'.
            format(round(sum(loss_test_list),5),
            round(sum(dice_test_list)/len(dice_test_list),4), round(sum(iou_test_list)/len(iou_test_list),4)))
    print('Datasets: ', config.data.name, ' || loss: ', [round(x, 4) for x in loss_test_list],
    ' || Dice score: ', [round(x, 4) for x in dice_test_list], ' || IOU: ', [round(x, 4) for x in iou_test_list])

    return

if __name__=='__main__':
    now = datetime.now()
    torch.cuda.empty_cache()
    parser = argparse.ArgumentParser(description='Train experiment')
    parser.add_argument('--exp_name', type=str, default='tmp')
    parser.add_argument('--config_yml', type=str,default='Configs/multi_train_local.yml')
    parser.add_argument('--model', type=str,default='DeepResUnet')
    parser.add_argument('--batch_size', type=int, default=32)
    parser.add_argument('--adapt_method', type=str, default=False)
    parser.add_argument('--num_domains', type=str, default=False)
    parser.add_argument('--dataset', type=str, nargs='+', default='isic2018')
    parser.add_argument('--k_fold', type=str, default='No')
    args = parser.parse_args()
    config = yaml.load(open(args.config_yml), Loader=yaml.FullLoader)
    config['model'] = args.model
    config['train']['batch_size']=args.batch_size
    config['data']['name'] = args.dataset
    config['model_adapt']['adapt_method']=args.adapt_method
    config['model_adapt']['num_domains']=args.num_domains
    config['data']['k_fold'] = args.k_fold

    # print config and args
    print(yaml.dump(config, default_flow_style=False))
    for arg in vars(args):
        print("{:<20}: {}".format(arg, getattr(args, arg)))

    store_config = config
    config = DotDict(config)

    # logging tensorbord, config, best model
    exp_dir = '{}/results/{}_{}_{}'.format(config.root_dir,args.exp_name,config.model,now.strftime("%Y%m%d_%H%M"))
    os.makedirs(exp_dir, exist_ok=True)
    writer = SummaryWriter(exp_dir)
    best_model_dir = '{}/best.pth'.format(exp_dir)
    test_results_dir = '{}/test_results.csv'.format(exp_dir)

    # store yml file
    if config.debug == False:
        yaml.dump(store_config, open('{}/exp_config.yml'.format(exp_dir), 'w'))

    if config.model_adapt.num_domains != False:
        K = int(config.model_adapt.num_domains)
    else:
        from typing import List
        K = len(config.data.name) if isinstance(config.data.name, List) else 1 # num of domains
    print('K == {}'.format(K))
    data2domain = {config.data.name[i]:i for i in range(len(config.data.name))}

    main(config)


# -*- coding: utf-8 -*-

import pandas as pd
import numpy as np
import shapeslib
from multiprocessing import Pool
import psutil
import json

max_cpu = psutil.cpu_count(logical=False) - 1

income_file = 'source/gapminder/mean_income.csv'
gini_file = 'source/gapminder/gini.csv'
known_shapes_file = 'povcal_country_year.csv'

def get_distances_res(v):
    i = v[0]
    g = v[1]
    cno, neis = shapeslib.get_neighbors(i, g, known_income_gini)
    return (i, g), (cno, neis)

if __name__ == '__main__':

    income = pd.read_csv(income_file)
    gini = pd.read_csv(gini_file)
    known_shapes = pd.read_csv(known_shapes_file)

    income = income.set_index(['geo', 'year'])[['Mean income']]
    income.index.names = ['country', 'year']
    income.columns = ['income']

    gini = gini.set_index(['geo', 'time'])[['gini_2100']]
    gini.index.names = ['country', 'year']
    gini.columns = ['gini']

    income_gini = pd.concat([income, gini], axis=1)
    income_gini = income_gini.dropna(how='any')

    income_gini_noc = income_gini.reset_index(drop=True).drop_duplicates()

    # below countries are not available in income/gini data
    mask = known_shapes['country'].isin(['mhl', 'nru', 'twn', 'kos'])
    try:
        known_shapes_list = known_shapes[~mask].values.tolist()
        known_shapes_list = [tuple(x) for x in known_shapes_list]
        known_income_gini = income_gini.loc[known_shapes_list]
    except KeyError:
        print("could not find some countries in povcalnet data, please exclude them.")
        raise

    with Pool(max_cpu) as p:
        res_distances = p.map(get_distances_res, income_gini_noc.values)

    res_distances_dict = dict(res_distances)

    all_neighbours = []
    for cy in income_gini.index.values:
        i, g = shapeslib.get_income_gini(cy, income_gini)
        all_neighbours.append((cy, res_distances_dict[(i, g)]))

    all_neighbours = dict(all_neighbours)
    all_neighbours_json = dict()

    for k, v in all_neighbours.items():
        c, y = k
        if c not in all_neighbours_json.keys():
            all_neighbours_json[c] = dict()
        all_neighbours_json[c][y] = {
            "countries": v[0],
            "neighbours": [list(x) for x in v[1]]
        }

    k = json.dumps(all_neighbours_json)

    with open('neighbours_list.json', 'w') as f:
        f.write(k)
        f.close()


import asyncio
import logging
import os
import shutil
import stat
import tempfile

logger = logging.getLogger()

ssh_add = shutil.which("ssh-add")

async def install_private_key(private_key):

    """
    Install a private key into the ssh-agent.
    """

    with tempfile.TemporaryDirectory() as local_working_directory:
        # Create a file for write with mode 0600
        key_file_fd = os.open(
            os.path.join(local_working_directory, "key"),
            os.O_WRONLY | os.O_CREAT | os.O_EXCL,
            mode=stat.S_IRUSR | stat.S_IWUSR,
        )
        with os.fdopen(key_file_fd, "w") as f:
            f.write(private_key)

        cmd_args = [os.path.join(local_working_directory, "key")]
        logger.debug(ssh_add)
        logger.debug(cmd_args)

        proc = await asyncio.create_subprocess_exec(
            ssh_add,
            *cmd_args,
            cwd=local_working_directory,
        )

        await proc.wait()


import nonebot
from nonebot.adapters.onebot.v11 import MessageSegment, Event, Bot
import httpx
import os
import time
import random
from .sql_sgin import data_select

def re_(dat):
    re_speak = httpx.get(url=fr'https://fanyi.sogou.com/reventondc/synthesis?text={dat}&speed=1&lang=zh-CHS&from=translateweb&speaker=6')
    with open(file=fr'E:\desktop\云酱bot\nonebotfile\nonebotfile\plugins\speak\{dat}.mp3', mode="wb") as f:
        f.write(re_speak.content)
    os.system(rf'ffmpeg -i E:\desktop\云酱bot\nonebotfile\nonebotfile\plugins\speak\{dat}.mp3 -ar 8000 -ab 7.9k -ac 1 -y E:\desktop\云酱bot\nonebotfile\nonebotfile\plugins\speak\{dat}.amr')

say = nonebot.on_startswith(msg="纯宝说", priority=3, block=True)

@say.handle()
async def _(event: Event, bot: Bot):
    if data_select(event.get_user_id()) != None:
        if int(data_select(event.get_user_id())[2]) >= 40:
            dat = event.get_message()
            ip_dat = str(dat).strip("纯宝说")
            re_(ip_dat)
            await say.send(MessageSegment.text(random.choice(['让我想想怎么说', 'ちょっと待って让我想想', ])))
            time.sleep(2)
            await bot.send(event=event,message=MessageSegment.record(file=rf'file:///E:\desktop\云酱bot\nonebotfile\nonebotfile\plugins\speak\{ip_dat}.amr'))
            time.sleep(2)
            os.remove(rf'E:\desktop\云酱bot\nonebotfile\nonebotfile\plugins\speak\{ip_dat}.mp3')
            os.remove(rf'E:\desktop\云酱bot\nonebotfile\nonebotfile\plugins\speak\{ip_dat}.amr')
        else:
            await say.send(MessageSegment.text(random.choice(['抱歉最近风控严重除主人外暂时不支持', '小宝贝,你的好感度不足哟', '就不和你说话~哼']))+MessageSegment.image(r'file:///E:\desktop\云酱bot\nonebotfile\nonebotfile\plugins\emj\1.jpg'))
    else:
        await bot.send(event=event, message=MessageSegment.text('先签到, 不然不和你说话...'))


import sklearn
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.datasets import make_circles
import numpy as np
import torch
from torch import nn

from sklearn.model_selection import train_test_split

torch.manual_seed(1955)
n_samples=1000
X,y=make_circles(n_samples, noise=0.03,
                 random_state=1955)

circles=pd.DataFrame(
{
    "X1": [x[0] for x in X ],
    "X2": [x[1] for x in X],
    "label":y
}
)

print(circles.head(5))

print(circles.label.value_counts())
circles.plot(x='X1', y="X2",
             kind='scatter',
             c='label' )
#plt.show()

X=torch.from_numpy(X).type(torch.float)
#maybe int?
y=torch.from_numpy(y).type(torch.float)

X_train, X_test, y_train, y_test = train_test_split(X,
                                                    y,
                                                    test_size=0.2, # 20% test, 80% train
                                                    random_state=42)
print(len(X_train), len(X_test),len(y_train), len(y_test))

device="cuda" if torch.cuda.is_available() else "cpu"
print(device)

class model1(nn.Module):
    def __init__(self):
        super().__init__()
        self.layer_1=nn.Linear(in_features=2,out_features=5)
        self.layer_2=nn.Linear(in_features=5, out_features=1)

    def forward(self,x):
        return self.layer_2(self.layer_1(x))

model1_0=model1().to(device)
print(model1_0)

# Make predictions with the model
untrained_preds = model1_0(X_test).to(device)
print(f"Length of predictions: {len(untrained_preds)}, Shape: {untrained_preds.shape}")
print(f"Length of test samples: {len(y_test)}, Shape: {y_test.shape}")
print(f"\nFirst 10 predictions:\n{untrained_preds[:10]}")
print(f"\nFirst 10 test labels:\n{y_test[:10]}")

## this works with raw logits witout passing through
## the sigmoid.

loss_fn=nn.BCEWithLogitsLoss()
optimizer=torch.optim.SGD(params=model1_0.parameters(), lr=0.1)

def accuracy_fn(y_true,y_pred):
    correct=torch.eq(y_true, y_pred).sum()
    acc=correct/len(y_pred)
    acc=acc*100
    return acc

y_logits=model1_0(X_test).to(device)
y_pred_probs=torch.sigmoid(y_logits)
y_preds=torch.round(y_pred_probs)

y_pred_labels=torch.round(torch.sigmoid(model1_0(X_test).to(device))[:5])
print(y_pred_labels.squeeze())

epochs=50
for epoch in range(epochs):
    ## enable change of gradients
    model1_0.train()

    y_logits=model1_0(X_train).squeeze()
    y_pred=torch.round(torch.sigmoid(y_logits))

    loss=loss_fn(y_logits,y_train)
    acc=accuracy_fn(y_true=y_train, y_pred=y_pred)

    optimizer.zero_grad()

    loss.backward()

    optimizer.step()

    ## turnoff
    model1_0.eval()

    ## smae as torch.no_grad()
    with torch.inference_mode():
        test_logits=model1_0(X_test).squeeze()
        test_pred=torch.round(torch.sigmoid(test_logits))

        test_loss=loss_fn(test_logits,y_test)
        test_acc=accuracy_fn(y_true=y_test,
                             y_pred=test_pred)

        if epoch % 10 == 0:
            print(f"Epoch: {epoch} | Loss: {loss:.5f}, Accuracy: {acc:.2f}% | Test loss: {test_loss:.5f}, Test acc: {test_acc:.2f}%")

import requests
from pathlib import Path

if Path("helper_functions.py").is_file():
      print("helper_functions.py already exists, skipping download")
else:
  print("Downloading helper_functions.py")
  request = requests.get("https://raw.githubusercontent.com/mrdbourke/pytorch-deep-learning/main/helper_functions.py")
  with open("helper_functions.py", "wb") as f:
    f.write(request.content)

from helper_functions import plot_predictions, plot_decision_boundary

def plotDecisions():
    plt.figure(figsize=(12,6))
    #nrows #ncols
    plt.subplot(1,2,1)
    plt.title("Train")
    plot_decision_boundary(model1_0,X_train,y_train)
    plt.subplot(1,2,2)
    plt.title("test")
    plot_decision_boundary(model1_0,X_test,y_test)
    plt.show()
plotDecisions()

##


import os
import sys

from direct.showbase.ShowBase import ShowBase
from panda3d.core import load_prc_file_data

from .Cube import Cube
from .Menu import Menu
from .LevelLoader import Level
from .CamControl import CamControl
from .GUI import GUI

class GameControl(ShowBase):
    """
    controlling the game itself, menu, editors level selection... it's sorta a fake-fsm.
    did i mention i dont like fsm's and prefer totaly whicked logic instead?
    """
    def __init__(self):

        load_prc_file_data("", "textures-power-2 none")
        load_prc_file_data("", "win-size 1600 900")
        # load_prc_file_data("", "fullscreen #t")
        load_prc_file_data("", "window-title cuboid")
        load_prc_file_data("", "icon-filename res/icon.ico")

        # I found openal works better for me
        load_prc_file_data("", "audio-library-name p3openal_audio")

         # ------ Begin of render pipeline code ------

        # Insert the pipeline path to the system path, this is required to be
        # able to import the pipeline classes
        pipeline_path = "../../"

        # Just a special case for my development setup, so I don't accidentally
        # commit a wrong path. You can remove this in your own programs.
        if not os.path.isfile(os.path.join(pipeline_path, "setup.py")):
            pipeline_path = "../../RenderPipeline/"

        sys.path.insert(0, pipeline_path)

        # Use the utility script to import the render pipeline classes
        from rpcore import RenderPipeline

        self.render_pipeline = RenderPipeline()
        # self.render_pipeline.mount_mgr.mount()
        # self.render_pipeline.load_settings("/$$rpconfig/pipeline.yaml")
        # self.render_pipeline.settings["pipeline.display_debugger"] = False
        # self.render_pipeline.set_empty_loading_screen()
        self.render_pipeline.create(self)

        # [Optional] use the default skybox, you can use your own skybox as well
        # self.render_pipeline.create_default_skybox()

        # ------ End of render pipeline code, thats it! ------

        # Set time of day
        self.render_pipeline.daytime_mgr.time = 0.812

        self.menu = Menu(self)
        self.level = Level(self)
        self.cube = Cube(self.level)
        self.camControl = CamControl(self.cube)
        self.gui = GUI(self)
        self.menu.showMenu()
        base.accept("i",self.camControl.zoomIn)
        base.accept("o",self.camControl.zoomOut)

    def startGame(self,level=0):
        #debug purpose only: to directly play a certian lvl number
        from sys import argv
        if len(argv) >1:
            level = int(argv[1])

        self.menu.hideMenu()
        self.level.loadLevel(level)
        self.cube.resetCube()
        self.cube.resetStats()
        self.cube.enableGameControl()
        base.accept("escape", self.pauseGame)

    def pauseGame(self):
        self.cube.disableGameControl()
        self.menu.showMenu()
        self.menu.showResume()
        #base.accept("escape", self.resumeGame )

    def resumeGame(self):
        self.menu.hideMenu()
        self.menu.hideResume()
        self.cube.enableGameControl()
        base.accept("escape", self.pauseGame)

    def levelEnd(self):
        self.cube.disableGameControl()
        self.menu.showMenu()


#coding="utf-8"
import os

from utils.spider_util import *
logger = get_logger("toutiao_caijing.log")
def parse_article_detail(detail_url):
    print('parse_detail_url is ' + detail_url)
    doc = pq(detail_url)
    return doc('div[class="article"]')('p')

def init_index_html(url):
    index_html = execute_script(url)
    with open('test.index', 'w', encoding="utf-8") as f:
        f.write(index_html)
    return index_html

if __name__ == '__main__':
    index_url = "https://www.toutiao.com/ch/news_finance/"
    index_file = 'test.index'
    # index_html = get_search_result_by_url(xinlang_caijing)
    if os.path.exists(index_file):
        logger.info("Already has index html in local path")
        with open(index_file, encoding="utf-8") as f:
            index_html = f.read()
    else:
        logger.info("First time load index")
        index_html = init_index_html(index_url)
    doc = pq(index_html)
    for item in doc('div[class="y-left index-content"]')('a[class="link title"]').items():
        logger.info("标题:\t" + item.text() + ' url: \t' + item.attr("href"))

    exit(0)
    for item in doc('div[class="item-inner"]')('a[class="title"]').items():
        print("标题:\t" + item.text() + ' url: \t' + item.attr("href"))
        # detail_html = get_search_result_by_url(item.attr("href"))
        # doc = pq(detail_html)
        # logging.info(doc('div[id="artibody"]').text())


# 可视化dBZ与降雨量正相关关系

import os
import numpy as np
import matplotlib.pyplot as plt

path = "/media/disk8T/zhn/dataset/NJU_CPOL_kdpRain/"
# path = '/media/disk8T/zhn/dataset/NJU_CPOL_update2308/dBZ/3.0km/'

file_names = sorted(os.listdir(path))

min_ = float('inf')
max_ = float('-inf')

arrays = []
for idx, file in enumerate(file_names):
    if idx !=70:
        continue
    new_path = path + file
    print(new_path)  # 找到一段连续降雨的记录
    frams = sorted([f for f in os.listdir(new_path) if f.endswith('.npy')])
    for fram in frams:
        new_new_path = new_path + '/' + fram
        data = np.load(new_new_path)
        # 生成1000个示例的256x256的随机数组（你可以替换成你的数据加载逻辑）

        if np.max(data) > max_:
            max_ = np.max(data)
        if np.min(data) < min_:
            min_ = np.min(data)

        arrays.append(data)

data = arrays[0:10]

fig, axes = plt.subplots(2, 5, figsize=(12, 6))
plt.subplots_adjust(wspace=0.3, hspace=0.3)  # 调整子图之间的间距

# 定义颜色区间和颜色映射
color_intervals = np.arange(0, 71, 10)  # 定义颜色区间为[0-10]和[10-20]
cmap = plt.get_cmap('viridis', len(color_intervals) - 1)  # 使用'viridis'颜色映射，根据颜色区间的数量选择颜色

from matplotlib.colors import BoundaryNorm
# 创建颜色标准化器
norm = BoundaryNorm(color_intervals, cmap.N, clip=True)

# 提取并绘制每个热力图
for i in range(10):
    row, col = i // 5, i % 5
    ax = axes[row, col]
    heatmap_data = data[i]  # 从你的NumPy数组中提取第i个热力图数据
    im = ax.imshow(heatmap_data, cmap=cmap, norm=norm, origin='lower', extent=[0, 256, 0, 256])
    ax.set_title(f'Heatmap {i + 1}')

# 添加颜色条
cbar = fig.colorbar(im, ax=axes.ravel().tolist(), orientation='vertical', ticks=color_intervals)
cbar.set_label('Color Intervals')

# 显示图形
plt.savefig('rain.png')


import numpy as np
import h5py
import os
from tqdm import tqdm
import utils
from imresize import imresize
os.environ['KMP_DUPLICATE_LIB_OK'] = 'TRUE'

def gen_traindata(config):
    scale = config["scale"]
    stride = config["stride"]
    h5savepath = config["hrDir"] + f'_train_SRCNNx{scale}.h5'
    hrDir = config["hrDir"] + '/'
    size_input = config["size_input"]
    size_label = config["size_label"]
    padding = int(abs(size_input - size_label) / 2)

    h5_file = h5py.File(h5savepath, 'w')
    imgList = os.listdir(hrDir)
    lr_subimgs = []
    hr_subimgs = []
    with tqdm(total=len(imgList)) as t:
        for imgName in imgList:
            hrIMG = utils.loadIMG_crop(hrDir + imgName, scale)
            hr_y = utils.img2ycbcr(hrIMG)[..., 0]
            lr = imresize(hr_y, 1 / scale, 'bicubic')
            lr = imresize(lr, scale, 'bicubic').astype(np.float32)

            for r in range(0, hr_y.shape[0] - size_input + 1, stride):  # hr.height
                for c in range(0, hr_y.shape[1] - size_input + 1, stride):  # hr.width
                    lr_subimgs.append(lr[r: r + size_input, c: c + size_input])
                    hr_subimgs.append(hr_y[r + padding: r + padding + size_label, c + padding: c + padding + size_label])
            t.update(1)

    lr_subimgs = np.array(lr_subimgs)
    hr_subimgs = np.array(hr_subimgs)

    h5_file.create_dataset('data', data=lr_subimgs)
    h5_file.create_dataset('label', data=hr_subimgs)

    h5_file.close()

def gen_valdata(config):
    scale = config["scale"]
    h5savepath = config["hrDir"] + f'_val_SRCNNx{scale}.h5'
    hrDir = config["hrDir"] + '/'
    size_input = config["size_input"]
    size_label = config["size_label"]
    padding = int(abs(size_input - size_label) / 2)

    h5_file = h5py.File(h5savepath, 'w')
    lr_group = h5_file.create_group('data')
    hr_group = h5_file.create_group('label')
    imgList = os.listdir(hrDir)
    for i, imgName in enumerate(imgList):
        hrIMG = utils.loadIMG_crop(hrDir + imgName, scale)
        hr_y = utils.img2ycbcr(hrIMG)[..., 0]

        data = imresize(hr_y, 1 / scale, 'bicubic')
        data = imresize(data, scale, 'bicubic')
        data = data.astype(np.float32)
        label = hr_y[padding: -padding, padding: -padding]
        lr_group.create_dataset(str(i), data=data)
        hr_group.create_dataset(str(i), data=label)

    h5_file.close()

if __name__ == '__main__':
    # config = {'hrDir': '../datasets/T91_aug', 'scale': 3, "stride": 14, "size_input": 33, "size_label": 21}
    config = {'hrDir': '../datasets/T91_aug', 'scale': 2, "stride": 14, "size_input": 22, "size_label": 10}
    gen_traindata(config)
    config['hrDir'] = '../datasets/Set5'
    gen_valdata(config)


from algorithm.pair_selector import *
from algorithm.crossover import *
from algorithm.mutator import *
from algorithm.elitism import *

class GeneticAlgorithm:
    def __init__(self, population):
        self.children = list()
        self.pair_selector = PairSelector()
        self.crossover = Crossover()
        self.mutator = Mutator()

        self.children = self.create_children(population)

        if ELITISM:
            self.children.extend(self.add_elites(population))

    # Create the children to be added to the new population.
    def create_children(self, population):
        children = list()
        pairings = self.calc_pairings()
        for i in range(pairings):
            pair = self.pair_selector.select(population)
            new_children = self.crossover.crossover(pair)
            for child in new_children:
                children.append(self.mutator.mutate(child))
        return children

    # Calculate how many pairings there will be.
    # This is dependant on Elitism, as to not increase the initial population size.
    def calc_pairings(self):
        pairings = POPULATION_SIZE / 2
        if ELITISM:
            pairings -= 1 * (ELITISM_COUNT / 2)
        return int(pairings)

    # Add the relevant amount of elites to the new generation.
    def add_elites(self, population):
        elitism = Elitism()
        return elitism.get_highest_x_elites(population, ELITISM_COUNT)


import pyspark.sql
from pyspark.sql import SparkSession

def null_filter(data: pyspark.sql.DataFrame) -> pyspark.sql.DataFrame:
    """
    Filters null values from the dataframe.

    :param data: dataframe to clean
    :return: pyspark.sql.DataFrame
    """
    return data.na.drop()

def main():
    spark = SparkSession. \
        builder. \
        appName("NullFilter"). \
        getOrCreate()
    data = spark.read.csv("../test/data/sample_data_1.csv", header="true")
    data = null_filter(data)
    data.show()

if __name__ == '__main__':
    main()


import requests
import re
import os
import random
import time
from fake_useragent import UserAgent

import winreg
def get_desktop():
    key = winreg.OpenKey(winreg.HKEY_CURRENT_USER,
                          r'Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders',)
    return winreg.QueryValueEx(key, "Desktop")[0]

# 存储得位置，桌面上创建一个下载文件
saveImagPath = get_desktop() +"\\" + "下载文件"
os.path.exists(saveImagPath)
if not os.path.exists(saveImagPath):
    os.mkdir(saveImagPath)

saveImagFile = saveImagPath

ua=UserAgent()

for j in range(1,237):
    t = 'http://www.mangareader.net/doraemon'+ '/'+ str(j)
    headers = {'User-Agent': ua.random}
    headers.update({'Cookie': '__cfduid=dffbec8ee6e5ebd0d1d60a22bbc10c7d01587445599; BB_plg=pm; _ga=GA1.2.2106140754.1587445603; _gid=GA1.2.456110379.1587445603; bbl=5; fjulakwal=1'})
    headers.update({'Host': 'www.mangareader.net'})
    headers.update({'Connection': 'keep-alive'})
    headers.update({'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9'})
    headers.update({'Accept-Encoding': 'gzip, deflate'})
    headers.update({'Accept-Language': 'zh-CN,zh;q=0.9'})
    headers.update({'Upgrade-Insecure-Requests': '1'})
    response = requests.get(t, headers=headers)
    saveimagLog = get_desktop() + "\\" + "下载文件" + "\\Log{}.txt".format(j)
    folog = open(saveimagLog, "w")
    if response.status_code ==200:
        html = response.text
        pag = re.findall('</select> of (.*?)
', html)
        a = int(pag[0])
        log = ('第{}章节有{}个图片\r'.format(j,a))
        print(log)
        folog.write(log)

        for i in range(1, int(pag[0])+1):

            time.sleep(4)
            I = t + '/' + str(i)
            # 获得每一页的地址

            response = requests.get(I, headers=headers)
            if response.status_code == 200:
                html = response.text

                # 获取每个图片的地址
                urls = re.findall('', html)
                # 获取图片和名称
                for url in urls:
                    print(url)

                    name = url.split('-')[-1]

                    headImagUrl = {}
                    headImagUrl.update({'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9'})
                    headImagUrl.update({'Host': 'i5.imggur.net'})
                    headImagUrl.update({'If-Modified-Since': 'Sat, 30 Oct 2010 22:44:22 GMT'})
                    headImagUrl.update({'If-None-Match': "4ccc9fc6-1b01c"})
                    headImagUrl.update({'Upgrade-Insecure-Requests': '1'})
                    headImagUrl.update({'User-Agent':ua.random})

                    response = requests.get(url, headers=headImagUrl)
                # 将图片写入文件夹
                    if response.status_code == 200:
                        information = '第 {} 章节中第 {} 个图片得地址是 {} ,文件写入成功'.format(j, i, url)
                        print(information)
                        folog.write(information + "\r")

                        saveImagFile = saveImagPath +"\\" + str(j) +"_" + name
                        with open(saveImagFile, 'wb') as f:
                            f.write(response.content)
                    else:
                        information = '第{}章节中第{}个图片没有读到，错误信息{},文件写入失败，请手动下载,图片地址是{}'.format(j, i, response.status_code, url)
                        print(information)
                        folog.write(information + "\r")
            else:
                information = '第{}章节中第{}个图片没有读到，错误信息{},文件写入失败，请手动下载'.format(j, i, response.status_code)
                print(information)
                folog.write(information + "\r")

    else:
        log ='第{}章访问出错，请自己手动访问，或者重新执行程序'.format(j)
        folog.write(log)

    folog.close()


import cv2
import numpy as np
import math

fr = 15;

def obj_robot_collision_border(arr):
    shape = arr.shape
    print("obj_robot_collision_border : " + str(shape))
    for i in range(shape[0]-1):
        for j in range(shape[1]):
            if (arr[i][j] - arr[i+1][j]) == 1:
                for k in range(15):
                    if i-k >= 0:
                        arr[i-k][j] = 0

    for i in range(shape[0]-1):
        for j in range(shape[1]):
            if (arr[shape[0]-1 -i][j] - arr[shape[0]-1-i-1][j]) == 1:
                for k in range(15):
                    if shape[0]-1-i+k < shape[0]:
                        arr[shape[0]-1-(i-k)][j] = 0

    for j in range(shape[1]-1):
        for i in range(shape[0]):
            if(arr[i][j] - arr[i][j+1]) == 1:
                for k in range(15):
                    if j-k > 0:
                        arr[i][j-k] = 0

    for j in range(shape[1]-1):
        for i in range(shape[0]):
            if(arr[i][shape[1]-1-j-1] - arr[i][shape[1]-1-j]) == -1:
                for k in range(15):
                    if shape[1]-1-j+k < shape[1]:
                        arr[i][shape[1]-1-j+k] = 0

    return arr

def array_shrinking(arr):
    shape = arr.shape
    print("array_shrinking : Recved size: " + str(shape))
    subarr = np.zeros([int(shape[0]/fr), int(shape[1]/fr)], dtype=int)
    for i in range(int(shape[0]/fr)):
        for j in range(int(shape[1]/fr)):
            tmp = arr[fr*i : fr*(i+1) , fr*j : fr*(j+1) ]
            #subarr[i,j] = sum(sum(tmp))
            subarr[i,j] = 1 if sum(sum(tmp)) == fr*fr else 0

    return subarr

def convert_img_to_array():

    f = open("/home/vishwas/Desktop/Office_Map/office_map_5cm.txt", 'w')

    image = cv2.imread("/home/vishwas/Desktop/Office_Map/combined_office_map.png", 0)
    image = image / 255

    A = image.shape
    print("Original size" + str(A))

    print(math.ceil(A[0]/fr))
    print(int(A[0]/fr))

    f.write("{{")
    for i in range(A[0]):
        for j in range(A[1]):
            f.write("%d, " % image[i][j])
        f.write("}, \n {")

    f.write("} \n\n\n\n")

    dflt_img = obj_robot_collision_border(image)

    if (math.ceil(A[0]/fr) != int(A[0]/fr)):
        row = ( A[0] + ( fr - (A[0]%fr) ) )
    else:
        row = A[0]

    if (math.ceil(A[1]/fr) != int(A[1]/fr)):
        col = ( A[1] + ( fr - (A[1]%fr) ) )
    else:
        col = A[1]

    grew_img = np.zeros([row, col], dtype=int)

    for i in range(A[0]):
        for j in range(A[1]):
            grew_img[i, j] = dflt_img[i][j]

    img = array_shrinking(grew_img)

    A = img.shape
    print("Modified size" + str(A))

    f.write("{{")
    for i in range(A[0]):
        for j in range(A[1]):
            f.write("%d, " % img[i][j])
        f.write("}, \n {")

    f.write("}")

if __name__ == "__main__":
    convert_img_to_array()


from ppip_function import pip, plants, ppip
from tkinter import Tk, Canvas, Frame, BOTH

class Vegetable:

    family = "plant"
    def __init__(self, name, distance):
        self.name = name
        self.distance = distance

carrot = Vegetable("Carrot", 5)
potato = Vegetable("Potato", 3)
basil = Vegetable("Basil", 7)

print(carrot.distance)
ppip(100,10, 2)

distance = 2
width = 10
length = 100

bounds=[0,width,length,0]

class Example(Frame):

    def __init__(self):
        super().__init__()

        self.initUI()

    def initUI(self):

        self.master.title("Colours")
        self.pack(fill=BOTH,expand=1)

        canvas = Canvas(self)
        canvas.create_rectangle(0,5*width,5*length,0,
        outline="#fb0", fill="#fb0")
        for plant in plants:
            canvas.create_bitmap(5*plant["x"],5*plant["y"],bitmap="warning")
        canvas.pack(fill=BOTH, expand=1)

def main():
    root = Tk()
    ex = Example()
    root.mainloop()

if __name__ == '__main__':
    main()


import logging
from typing import TYPE_CHECKING, List

from prospector.finder import FileFinder
from prospector.message import Location, Message
from prospector.tools.base import ToolBase

if TYPE_CHECKING:
    from prospector.config import ProspectorConfig

# HACK: pyroma configures logging in its __init__.py so by importing,
# it will change existing logging configuration to DEBUG which causes
# problems with other 3rd party modules as everything now logs to
# stdout...
_old = logging.basicConfig
try:
    logging.basicConfig = lambda **k: None
    from pyroma import projectdata, ratings

    # if pyroma doesn't exist, will raise an ImportError and be caught "upstream"
finally:
    # always restore logging.basicConfig
    logging.basicConfig = _old

PYROMA_ALL_CODES = {
    "Name": "PYR01",
    "Version": "PYR02",
    "VersionIsString": "PYR03",
    "PEPVersion": "PYR04",
    "Description": "PYR05",
    "LongDescription": "PYR06",
    "Classifiers": "PYR07",
    "PythonVersion": "PYR08",
    "Keywords": "PYR09",
    "Author": "PYR10",
    "AuthorEmail": "PYR11",
    "Url": "PYR12",
    "License": "PYR13",
    "LicenceClassifier": "PYR14",
    "ZipSafe": "PYR15",
    "SDist": "PYR16",
    "PackageDocs": "PYR17",
    "ValidREST": "PYR18",
    "BusFactor": "PYR19",
}

PYROMA_CODES = {}

def _copy_codes():
    for name, code in PYROMA_ALL_CODES.items():
        if hasattr(ratings, name):
            PYROMA_CODES[getattr(ratings, name)] = code

_copy_codes()

PYROMA_TEST_CLASSES = [t.__class__ for t in ratings.ALL_TESTS]

class PyromaTool(ToolBase):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.ignore_codes = ()

    def configure(self, prospector_config: "ProspectorConfig", found_files: FileFinder):
        self.ignore_codes = prospector_config.get_disabled_messages("pyroma")

    def run(self, found_files: FileFinder) -> List[Message]:
        messages = []
        for directory in found_files.directories:
            # just list directories which are not ignored, but find any `setup.py` ourselves
            # as that is excluded by default
            for filepath in directory.iterdir():
                if filepath.is_dir() or filepath.name != "setup.py":
                    continue

                data = projectdata.get_data(directory.resolve())

                all_tests = [m() for m in PYROMA_TEST_CLASSES]
                for test in all_tests:
                    code = PYROMA_CODES.get(test.__class__, "PYRUNKNOWN")

                    if code in self.ignore_codes:
                        continue

                    passed = test.test(data)
                    if passed is False:  # passed can be True, False or None...
                        loc = Location(filepath, "setup", None, -1, -1)
                        msg = Message("pyroma", code, loc, test.message())
                        messages.append(msg)

        return messages