from django.core.urlresolvers import reverse
from django.db import models
from TwitterSearch import TwitterSearch, TwitterSearchOrder, TwitterSearchException
from django.conf import settings

# Create your models here.
from urllib import quote_plus

class EntryQueue(models.Model):
    origin = models.CharField(max_length=200)
    origin_id = models.BigIntegerField(blank=True)
    message = models.TextField(blank=True)
    author_id = models.BigIntegerField(blank=True)
    author = models.CharField(max_length=200, blank=True)
    areas = models.ManyToManyField('Area')
    analysis = models.BooleanField(default=False)
    datetime = models.DateTimeField(null=True)
    lat = models.FloatField(blank=True, null=True)
    long = models.FloatField(blank=True, null=True)
    class Meta:
        unique_together = ('origin', 'origin_id',)

    def classification(self):
        classif = self.entryclassification_set.all()[0].label
        return classif

class EntryClassification(models.Model):
    entry = models.ForeignKey('EntryQueue')
    positive = models.FloatField(null=True)
    negative = models.FloatField(null=True)
    neutral = models.FloatField(null=True)
    label = models.CharField(max_length=20, blank=True)

class ApiRequests(models.Model):
    name = models.CharField(max_length=20)
    url = models.TextField()
    counter = models.IntegerField()
    until = models.DateTimeField()

def save_tweets(area, origin, tweet):
    result = None
    coordinates_key = 'coordinates'
    # try:
    #     import ipdb;ipdb.set_trace()
    if coordinates_key in tweet and tweet[coordinates_key] and 'type' in tweet[coordinates_key] and tweet[coordinates_key]['type'] == 'Point' and coordinates_key in tweet[coordinates_key] and tweet[coordinates_key][coordinates_key]:
        coord_lat = tweet[coordinates_key][coordinates_key][1]
        coord_long = tweet[coordinates_key][coordinates_key][0]
        eq, created = EntryQueue.objects.get_or_create(
            origin=origin.name,
            origin_id=tweet['id'],
            defaults={
                'message': tweet['text'],
                'author_id': tweet['user']['id'],
                'lat': coord_lat,
                'long': coord_long
            }
        )
        eq.author = tweet['user']['screen_name']
        eq.save()
        area.entryqueue_set.add(eq)
        result = tweet['id']
    else:
        eq, created = EntryQueue.objects.get_or_create(
            origin=origin.name,
            origin_id=tweet['id'],
            defaults={
                'message': tweet['text'],
                'author_id': tweet['user']['id']
            }
        )
        eq.author = tweet['user']['screen_name']
        eq.save()
        area.entryqueue_set.add(eq)
        result = tweet['id']
    print( '@%s tweeted: %s' % ( tweet['user']['screen_name'], tweet['text'] ) )
    # except Exception as ex:
    #     print ex.message
        #print( '@%s tweeted: %s' % ( tweet['user']['screen_name'], tweet['text'] ) )
    return result

class Area(models.Model):
    description = models.CharField(max_length=200)
    lat = models.FloatField(blank=True)#
    long = models.FloatField(blank=True)#
    rad = models.PositiveIntegerField(blank=True)

    def __unicode__(self):
        return self.description

class SourceForArea(models.Model):
    name = models.CharField(max_length=30)
    area = models.ForeignKey('Area')
    since_id = models.BigIntegerField(null=True, blank=True)
    max_id = models.BigIntegerField(null=True, blank=True)

    def fetch_new_twitter_entries(self):
        # read since_id
        pass

    def fetch_twitter_entries(self):
        origin = self
        max_id = origin.max_id
        since_id = None
        area = origin.area
        try:
            count = 50
            tso = TwitterSearchOrder() # create a TwitterSearchOrder object
            tso.setKeywords(['']) # let's define all words we would like to have a look for
            tso.setResultType('recent')
            if origin.max_id:
                tso.setMaxID(origin.max_id-1)  # as per twitter docs
            tso.setLanguage('en') # we want to see German tweets only
            tso.setGeocode(latitude=area.lat, longitude=area.long, radius=area.rad, km=True)
            tso.setCount(count) # please dear Mr Twitter, only give us 7 results per page
            tso.setIncludeEntities(False) # and don't give us all those entity information

            # it's about time to create a TwitterSearch object with our secret tokens
            ts = TwitterSearch(
                **settings.TWITTER
            )
            total = 0
            for tweet in ts.searchTweetsIterable(tso): # this is where the fun actually starts :)

                max_id = save_tweets(area, origin, tweet)
                if not since_id:
                    since_id = max_id

                #import ipdb;ipdb.set_trace()
                #print( '@%s tweeted: %s' % ( tweet['user']['screen_name'], tweet['text'] ) )

                total += 1

                if total >= 50:
                    break

        except TwitterSearchException as e: # take care of all those ugly errors if there are some
            print(e)
        # except:
        #     pass
        finally:
            origin.max_id = max_id
            origin.save()


TIME_UNITS = {
    "minutes": 60,
    "hours": 60 * 60,
    "days": 60 * 60 * 24,
    "weeks": 60 * 60 * 24 * 7,
    "months": 60 * 60 * 24 * 30,
}

def get_seconds(amount: int, unit: str) -> int:
    return amount * TIME_UNITS[unit]

def get_amount_and_unit(seconds: int) -> tuple[int, str]:
    print(seconds)
    possible_answers: list[tuple[int, str]] = []
    for unit, to_divide in TIME_UNITS.items():
        computed = seconds / to_divide
        if computed.is_integer() and computed >= 1:
            possible_answers.append((int(computed), unit))

    return min(possible_answers)


#!/usr/bin/python3

import sys

def safe_print_integer_err(value):
    """prints integer with "{:d}".format(0

    If ValueError message exist, a correspoding message is printed to
    standard error

    Arg:
    value (int): the integer to print

    Return:
    If TypeError or ValurError Exist, return false else true
    """

    try:
        print("{:d}".format(value))
        return (True)
    except (TypeError, ValueError):
        print("Exception: {}".format(sys.exc_info()[1]), file=sys.stderr)
        return (False)


def extractAtravelingpeachWordpressCom(item):
	'''
	Parser for 'atravelingpeach.wordpress.com'
	'''

	vol, chp, frag, postfix = extractVolChapterFragmentPostfix(item['title'])
	if not (chp or vol) or "preview" in item['title'].lower():
		return None

	if item['title'].startswith("Chapter ") and item['tags'] == ['Uncategorized']:
			return buildReleaseMessageWithType(item, 'After the Cannon Fodder’s Rebirth', vol, chp, frag=frag, postfix=postfix, tl_type='translated')

	tagmap = [
		('PRC',       'PRC',                      'translated'),
		('Loiterous', 'Loiterous',                'oel'),
	]

	for tagname, name, tl_type in tagmap:
		if tagname in item['tags']:
			return buildReleaseMessageWithType(item, name, vol, chp, frag=frag, postfix=postfix, tl_type=tl_type)

	return False


######################################
#   @file   correct_pixel_value.py
#   @author Tomomasa Uchida
#   @date   2019/02/07
######################################

import numpy as np
from matplotlib import pyplot as plt
from matplotlib import cycler
import matplotlib.gridspec as gridspec
import matplotlib.patches as pat
import cv2
import subprocess
import sys
import statistics

# Graph settings
plt.style.use('seaborn-white')
colors = cycler('color', ['#EE6666', '#3388BB', '#9988DD', '#EECC55', '#88BB44', '#FFBBBB'])
plt.rc('axes', facecolor='#E6E6E6', edgecolor='none', axisbelow=True, grid=False, prop_cycle=colors)
plt.rc('grid', color='w', linestyle='solid')
plt.rc('patch', edgecolor='#E6E6E6')
plt.rc('lines', linewidth=2)
plt.rcParams["mathtext.fontset"] = "stix"
plt.rcParams["mathtext.rm"] = "Times New Roman"

# Message
print("===================================")
print("     Correct Pixel Value")
print("      author : Tomomasa Uchida")
print("      date   : 2019/02/07")
print("===================================")

# Check arguments
args = sys.argv
if len(args) != 3:
    print("\nUSAGE        : $ python correct_pixel_value.py [input_image_data] [input_image_data(LR=1)]")
    print("Example      : $ python correct_pixel_value.py [input_image.bmp] [input_image_LR1.bmp]]")
    #raise Exception
    sys.exit()

# Set initial parameter
p_init      = 1.0
p_interval  = 0.01
print("\nInput image data (args[1])       :", args[1])
print("Input image data(LR=1) (args[2]) :", args[2])
print("p_init                           :", p_init)
print("p_interval                       :", p_interval)

# Read Input Image
def readImage(_img_name):
    # read input image
    img_BGR = cv2.imread(_img_name)

    # convert color BGR to RGB
    img_RGB = cv2.cvtColor(img_BGR, cv2.COLOR_BGR2RGB)

    return img_RGB

# RGB Histogram
def rgbHist(_img_rgb, _ax, _title):
    R_nonzero = _img_rgb[:,:,0][_img_rgb[:,:,0] > 0]
    G_nonzero = _img_rgb[:,:,1][_img_rgb[:,:,1] > 0]
    B_nonzero = _img_rgb[:,:,2][_img_rgb[:,:,2] > 0]
    _ax.hist(R_nonzero.ravel(), bins=50, color='r', alpha=0.5, label="R")
    _ax.hist(G_nonzero.ravel(), bins=50, color='g', alpha=0.5, label="G")
    _ax.hist(B_nonzero.ravel(), bins=50, color='b', alpha=0.5, label="B")
    _ax.legend()

    _ax.set_title('Histogram ('+_title+')')
    _ax.set_xlim([-5,260])

    return _ax

# Grayscale Histogram
def grayscaleHist(_img_gray, _ax, _title):
    img_Gray_nonzero = _img_gray[_img_gray > 0]
    _ax.hist(img_Gray_nonzero.ravel(), bins=50, color='black', alpha=1.0)

    _ax.set_title('Histogram ('+_title+')')
    _ax.set_xlim([-5,260])

    return _ax

# Histograms of Input image(LR=1), Input image and Corrected image
def comparativeHist(_img_in_rgb_LR1, _img_in_rgb, _img_out_rgb, _ax, _y_max):
    # Convert RGB to Grayscale
    img_in_Gray_LR1         = cv2.cvtColor(_img_in_rgb_LR1, cv2.COLOR_RGB2GRAY)
    img_in_Gray_LR1_nonzero = img_in_Gray_LR1[img_in_Gray_LR1 > 0]
    img_in_Gray             = cv2.cvtColor(_img_in_rgb, cv2.COLOR_RGB2GRAY)
    img_in_Gray_nonzero     = img_in_Gray[img_in_Gray > 0]
    img_out_Gray            = cv2.cvtColor(_img_out_rgb, cv2.COLOR_RGB2GRAY)
    img_out_Gray_nonzero    = img_out_Gray[img_out_Gray > 0]

    # input image(LR=1)
    mean_in_LR1 = int(np.mean(img_in_Gray_LR1_nonzero))
    _ax.hist(img_in_Gray_LR1_nonzero.ravel(), bins=50, alpha=0.5, label="Input image ($L_{\mathrm{R}}=1$)", color='#1F77B4')
    _ax.axvline(mean_in_LR1, color='#1F77B4')
    _ax.text(mean_in_LR1+5, _y_max*0.8, "mean:"+str(mean_in_LR1), color='#1F77B4', fontsize='12')

    # input image
    mean_in = int(np.mean(img_in_Gray_nonzero))
    _ax.hist(img_in_Gray_nonzero.ravel(), bins=50, alpha=0.5, label="Input image", color='#FF7E0F')
    _ax.axvline(mean_in, color='#FF7E0F')
    _ax.text(mean_in+5, _y_max*0.6, "mean:"+str(mean_in), color='#FF7E0F', fontsize='12')

    # corrected image
    mean_out = int(np.mean(img_out_Gray_nonzero))
    _ax.hist(img_out_Gray_nonzero.ravel(), bins=50, alpha=0.5, label="Corrected image", color='#2C9F2C')
    _ax.axvline(mean_out, color='#2C9F2C')
    _ax.text(mean_out+5, _y_max*0.7, "mean:"+str(mean_out), color='#2C9F2C', fontsize='12')

    _ax.set_title('Comparative histogram')
    _ax.set_xlabel("Pixel value")
    _ax.set_ylabel("Number of pixels")
    _ax.legend(fontsize='12')

    return _ax

# Create Histogram
def createHistogram(_img_in_RGB_LR1, _img_in_RGB, _img_corrected_RGB, _standard_pixel_value_LR1):
    # Convert RGB to Grayscale
    img_in_Gray_LR1     = cv2.cvtColor(_img_in_RGB_LR1, cv2.COLOR_RGB2GRAY)
    img_in_Gray         = cv2.cvtColor(_img_in_RGB, cv2.COLOR_RGB2GRAY)
    img_corrected_Gray  = cv2.cvtColor(_img_corrected_RGB, cv2.COLOR_RGB2GRAY)

    fig = plt.figure(figsize=(10, 6)) # figsize=(width, height)
    gs = gridspec.GridSpec(2,3)

    # Input image(LR=1)
    ax1 = fig.add_subplot(gs[0,0])
    ax1.set_title('Input image ($L_{\mathrm{R}}=1$)')
    ax1.imshow(_img_in_RGB_LR1)
    ax1.set_xticks([]), ax1.set_yticks([])

    # Input image
    ax2 = fig.add_subplot(gs[0,1])
    ax2.set_title('Input image')
    ax2.imshow(_img_in_RGB)
    ax2.set_xticks([]), ax2.set_yticks([])

    # Corrected image
    ax3 = fig.add_subplot(gs[0,2])
    ax3.set_title('Corrected image')
    ax3.imshow(_img_corrected_RGB)
    ax3.set_xticks([]), ax3.set_yticks([])

    # Histogram(input image(LR=1))
    ax4 = fig.add_subplot(gs[1,0])
    # ax4 = grayscaleHist(img_in_Gray_LR1, ax4, "Input image ($L_{\mathrm{R}}=1$)")
    ax4 = rgbHist(_img_in_RGB_LR1, ax4, "Input image ($L_{\mathrm{R}}=1$)")

    # Histogram(input image)
    ax5 = fig.add_subplot(gs[1,1])
    # ax5 = grayscaleHist(img_in_Gray, ax5, "Input image")
    ax5 = rgbHist(_img_in_RGB, ax5, "Input image")

    # Histogram(output image)
    ax6 = fig.add_subplot(gs[1,2])
    # ax6 = grayscaleHist(img_corrected_Gray, ax6, "Corrected image")
    ax6 = rgbHist(_img_corrected_RGB, ax6, "Corrected image")

    # Unify ylim b/w input image and corrected image
    hist_in_LR1,    bins_in_LR1     = np.histogram(img_in_Gray_LR1[img_in_Gray_LR1>0],      50)
    hist_in,        bins_in         = np.histogram(img_in_Gray[img_in_Gray>0],              50)
    hist_corrected, bins_corrected  = np.histogram(img_corrected_Gray[img_corrected_Gray>0],50)
    list_max = [max(hist_in_LR1), max(hist_in), max(hist_corrected)]
    ax4.set_ylim([0, max(list_max)*1.1])
    ax5.set_ylim([0, max(list_max)*1.1])
    ax6.set_ylim([0, max(list_max)*1.1])

    # # Histograms(Input(LR1), Input, Corrected)
    # ax7 = fig.add_subplot(gs[2,:])
    # ax7 = comparativeHist(_img_in_RGB_LR1, _img_in_RGB, _img_corrected_RGB, ax7, max(list_max)*1.1)
    # ax7.set_ylim([0, max(list_max)*1.1])

    # Draw text
    x = (_standard_pixel_value_LR1+max_pixel_value_LR1)*0.5 - 100
    text = "["+str(_standard_pixel_value_LR1)+", "+str(max_pixel_value_LR1)+"]\n→ "+str(ratio_of_reference_section*100)+"(%)"
    ax4.text(x, max(list_max)*1.1*0.5, text, color='black', fontsize='12')
    ax6.text(x, max(list_max)*1.1*0.5, text, color='black', fontsize='12')

    # Draw reference section
    rect = plt.Rectangle((_standard_pixel_value_LR1, 0), max_pixel_value_LR1-_standard_pixel_value_LR1, max(list_max)*1.1, fc='black', alpha=0.3)
    ax4.add_patch(rect)
    rect = plt.Rectangle((_standard_pixel_value_LR1, 0), max_pixel_value_LR1-_standard_pixel_value_LR1, max(list_max)*1.1, fc='black', alpha=0.3)
    ax6.add_patch(rect)

# Correct Pixel Value for Each RGB
def correct_pixel_value(_rgb_img, _correct_param):
    corrected_img_RGB = np.empty((_rgb_img.shape[0], _rgb_img.shape[1], 3), dtype=np.uint8)

    # Apply correction
    corrected_img_RGB[:, :, 0] = cv2.multiply(_rgb_img[:, :, 0], _correct_param) # R
    corrected_img_RGB[:, :, 1] = cv2.multiply(_rgb_img[:, :, 1], _correct_param) # G
    corrected_img_RGB[:, :, 2] = cv2.multiply(_rgb_img[:, :, 2], _correct_param) # B

    return corrected_img_RGB

def preProcess():
    print("Input image (RGB)                :", img_in_RGB.shape) # （height, width, channel）

    # Calc all number of pixels of the input image
    N_all = img_in_RGB.shape[0] * img_in_RGB.shape[1]
    print("N_all                            :", N_all, "(pixels)")

    # Calc number of pixels that pixel value is not 0
    img_in_Gray     = cv2.cvtColor(img_in_RGB, cv2.COLOR_RGB2GRAY)
    N_all_nonzero   = np.sum(img_in_Gray > 0)
    print("N_all_nonzero                    :", N_all_nonzero, "(pixels)")

    # Calc max pixel value of the input image(LR=1)
    img_in_Gray_LR1     = cv2.cvtColor(img_in_RGB_LR1, cv2.COLOR_RGB2GRAY)
    N_all_nonzero_LR1   = np.sum(img_in_Gray_LR1 > 0)
    max_pixel_value_LR1 = np.max(img_in_Gray_LR1)
    print("Max pixel value (LR=1)           :", max_pixel_value_LR1, "(pixel value)")

    # Calc ratio of the max pixel value (LR=1)
    num_max_pixel_value_LR1   = np.sum(img_in_Gray_LR1 == max_pixel_value_LR1)
    print("Num. of max pixel value (LR=1)   :", num_max_pixel_value_LR1, "(pixels)")
    ratio_max_pixel_value_LR1 = num_max_pixel_value_LR1 / N_all_nonzero_LR1
    # ratio_max_pixel_value_LR1 = round(ratio_max_pixel_value, 4)
    ratio_max_pixel_value_LR1 = round(ratio_max_pixel_value_LR1, 8)
    print("Ratio of max pixel value (LR=1)  :", round(ratio_max_pixel_value_LR1*100, 2), "(%)")

    # Calc most frequent pixel value (LR=1)
    img_in_Gray_nonzero_LR1         = img_in_Gray_LR1[img_in_Gray_LR1 > 0]
    bincount = np.bincount(img_in_Gray_nonzero_LR1)
    most_frequent_pixel_value_LR1   = np.argmax( bincount )
    print("Most frequent pixel value (LR=1) :", most_frequent_pixel_value_LR1, "(pixel value)")

    return N_all_nonzero, N_all_nonzero_LR1, img_in_Gray_LR1, max_pixel_value_LR1, ratio_max_pixel_value_LR1

def determineCorrectParam(_ratio_of_reference_section):
    print("Ratio of reference section       :", _ratio_of_reference_section*100, "(%)")

    # Determine standard pixel value in the input image(LR=1)
    tmp_reference_section         = 0.0
    standard_pixel_value_LR1      = max_pixel_value_LR1
    while tmp_reference_section < _ratio_of_reference_section:
        # Temporarily calc
        sum_pixels_in_section     = np.sum( (standard_pixel_value_LR1 <= img_in_Gray_LR1) )
        tmp_reference_section     = sum_pixels_in_section / N_all_nonzero_LR1

        # Next pixel value
        standard_pixel_value_LR1 -= 1

    print("Standard pixel value (LR=1)      :", standard_pixel_value_LR1, "(pixel value)")
    print("Reference section                :", standard_pixel_value_LR1, "~", max_pixel_value_LR1, "(pixel value)")
    print("Ratio of reference section       :", round(tmp_reference_section*100, 2), "(%)")

    # Determine parameter
    p = p_init
    tmp_ratio = 0.0
    while tmp_ratio < _ratio_of_reference_section:
        # Temporarily, correct pixel value of the input image with p
        tmp_corrected_img_RGB   = correct_pixel_value(img_in_RGB, p)
        tmp_corrected_img_Gray  = cv2.cvtColor(tmp_corrected_img_RGB, cv2.COLOR_RGB2GRAY)

        # Then, calc ratio of max pixel value(LR=1)
        tmp_sum_pixels_in_section = np.sum(standard_pixel_value_LR1 <= tmp_corrected_img_Gray)
        tmp_ratio = tmp_sum_pixels_in_section / N_all_nonzero

        # Update parameter
        p += p_interval

    p_final = round(p, 2)

    return p_final, standard_pixel_value_LR1

def correctPixelValue(_p_final, _standard_pixel_value_LR1):
    # Create corrected image
    img_corrected_RGB  = correct_pixel_value(img_in_RGB, _p_final)
    img_corrected_Gray = cv2.cvtColor(img_corrected_RGB, cv2.COLOR_RGB2GRAY)

    print("p_final                          :", _p_final)
    print("Ratio of num. of pixels to 255   :", round(np.sum(img_corrected_Gray==255) / N_all_nonzero * 100, 2), "(%)")

    # Create figure
    createHistogram(img_in_RGB_LR1, img_in_RGB, img_corrected_RGB, _standard_pixel_value_LR1)

    return img_corrected_RGB

# Save figure and images
def saveFigureAndImages(_p_final, _img_in_RGB, _img_corrected_RGB):
    img_in_name = args[1]
    save_path = img_in_name[:-4]

    fig_name = save_path+"_figure_"+str(_p_final)+".png"
    plt.savefig(fig_name)
    # plt.show()

    # convert color RGB to BGR
    # img_in_BGR          = cv2.cvtColor(_img_in_RGB,         cv2.COLOR_RGB2BGR)
    img_out_BGR         = cv2.cvtColor(_img_corrected_RGB,  cv2.COLOR_RGB2BGR)
    # input_img_name      = "images/input.bmp"
    corrected_img_name  = save_path+"_corrected_"+str(_p_final)+".bmp"
    # cv2.imwrite(input_img_name, img_in_BGR)
    cv2.imwrite(corrected_img_name, img_out_BGR)

    execCommand( corrected_img_name )

# Exec. command
def execCommand(_corrected_img_name):
    preview_command = ['open', _corrected_img_name]
    try:
	    res = subprocess.check_call(preview_command)

    except:
	    print("ERROR")

if __name__ == "__main__":
    # Read two input images
    img_in_RGB      = readImage(args[1])
    img_in_RGB_LR1  = readImage(args[2])

    print("\n\n====================================")
    print(" STEP1 : Get max pixel value (LR=1)")
    print("====================================")
    N_all_nonzero, N_all_nonzero_LR1, img_in_Gray_LR1, max_pixel_value_LR1, ratio_max_pixel_value_LR1 = preProcess()

    print("\n\n================================================")
    print(" STEP2 : Search for standard pixel value (LR=1)")
    print("================================================")
    ratio_of_reference_section = 0.01 # 1(%)
    p_final, standard_pixel_value_LR1 = determineCorrectParam(ratio_of_reference_section)

    print("\n\n=============================")
    print(" STEP3 : Correct pixel value")
    print("=============================")
    img_corrected_RGB = correctPixelValue(p_final, standard_pixel_value_LR1)

    # Save figure and images
    saveFigureAndImages(p_final, img_in_RGB, img_corrected_RGB)


from django.core.management.base import BaseCommand
from reviews.models import Title, Category, Genre, Comment, Review
from users.models import User
import csv

MODEL_FILE = {
    Genre: 'static/data/genre.csv',
    Category: 'static/data/category.csv',
    Title: 'static/data/titles.csv',
    User: 'static/data/users.csv',
    Review: 'static/data/review.csv',
    Comment: 'static/data/comments.csv'
}

class Command(BaseCommand):
    help = ''

    def handle(self, *args, **kwargs):
        for model, dir in MODEL_FILE.items():
            with open(dir, encoding='utf-8-sig') as csv_file:
                reader = csv.DictReader(csv_file, delimiter=',')
                for row in reader:
                    model.objects.get_or_create(**row)


import asyncio
import logging
from datetime import datetime, timedelta
from typing import List

from telethon import TelegramClient

from core import misc
from modules.db_model.context import Context
from modules.db_model.smw import SmwChat
from modules.nailerine.wipers.smw import smw, utils
from modules.utils import aiowrap

log = logging.getLogger(__name__)

class SmwTask:
    __slots__ = ['_running']

    def __init__(self):
        self._running = False

    async def task(self):
        if self._running:
            return

        self._running = True

        while await utils.chats_available():
            chats: List[SmwChat] = await utils.get_chats()
            for chat in chats:
                if datetime.utcnow() >= chat.next_date:
                    result = await self.wipe(chat)
                    await self.on_wipe(chat)
                    next_dt = await utils.get_next_date(chat.id)
                    log.warning('smw task has sent %s deletion requests in %s', result, chat.id)
                    await self.notify(chat, result, next_dt)
            await asyncio.sleep(timedelta(minutes=30).total_seconds())
        self._running = False

    async def wipe(self, smw_chat: SmwChat) -> int:
        client: TelegramClient = misc.clients[smw_chat.client_name]
        result = await smw.wipe_seen_messages(client, smw_chat.id, own=False)

        return result

    @aiowrap
    def on_wipe(self, smw_chat: SmwChat):
        with Context() as c:
            smw_chat: SmwChat = c.query(SmwChat).get(smw_chat.id)
            smw_chat.next_date += timedelta(days=1)

    async def notify(self, smw_chat: SmwChat, result: int, next_dt: datetime):
        client: TelegramClient = misc.clients[smw_chat.client_name]
        s_next_dt = next_dt.strftime(utils.fmt)
        await client.send_message(smw_chat.id,
                                  f'```Nailerine auto SMW, {result} delete requests sent.\n'
                                  f'Next wipe is scheduled on {s_next_dt}.```')


##
# A sublist is a list that is part of a larger list. A sublist may be a list containing a
# single element, multiple elements, or even no elements at all. For example, [1],
# [2], [3] and [4] are all sublists of [1, 2, 3, 4]. The list [2, 3] is also a
# sublist of [1, 2, 3, 4], but [2, 4] is not a sublist [1, 2, 3, 4] because
# the elements 2 and 4 are not adjacent in the longer list. The empty list is a sublist of
# any list. As a result, [] is a sublist of [1, 2, 3, 4]. A list is a sublist of itself,
# meaning that [1, 2, 3, 4] is also a sublist of [1, 2, 3, 4].
# In this exercise you will create a function, isSublist, that determines whether
# or not one list is a sublist of another. Your function should take two lists, larger
# and smaller, as its only parameters. It should return True if and only if smaller
# is a sublist of larger. Write a main program that demonstrates your function.
"""
check if a list is a sublist of another
@param: list: a list to search in
@param: list: a list to check
@return: bool: true if the list is sublist of the other, false if not
"""
def isSublist(larger, smaller):

    # set result for special cases
    if smaller == [] or smaller == larger:

        return True

    else:

        # loop on smaller list and perform checking for each element if it is present in larger list
        for el in smaller:

            if el in larger:

                # set starting and ending position to slice larger list in smaller one
                start_pos = larger.index(el)
                end_pos = start_pos + (len(smaller))

                # return true if sliced larger is same as smaller list
                if smaller == larger[start_pos:end_pos]:
                    return True

        return False

def main():

    t = [1, 23, -4, 3.45, 908]
    t2 = [23, -4, 3.45, 23]
    t3 = [-4, 3.45]
    t4 = [1, 23]
    t5 = [23, 3.45]

    print("La lista {} {} sottolista di {}".format( t2 ,"È" if isSublist(t, t2) else "NON È", t ))
    print("La lista {} {} sottolista di {}".format( t3 ,"È" if isSublist(t, t3) else "NON È", t ))
    print("La lista {} {} sottolista di {}".format( t4 ,"È" if isSublist(t, t4) else "NON È", t ))
    print("La lista {} {} sottolista di {}".format( t5 ,"È" if isSublist(t, t5) else "NON È", t ))
    print("La lista {} {} sottolista di {}".format( [] ,"È" if isSublist(t, []) else "NON È", t ))

main()


def happyNumber(num):
    # initialise fast and slow pointers
    slow = num
    fast = num
    # iterate until slow and fast equal. if 1, return true, else false
    while True:
        slow = squareSum(slow)
        fast = squareSum(squareSum(fast))

        if slow == fast:
            break

    if slow == 1:
        return True
    return False

def squareSum(num):
    sum = 0
    num = str(num)
    for i in range(len(num)):
        sum += (int(num[i]) ** 2)
    return sum

print(happyNumber(19)) # true
print(happyNumber(2)) # false


#from flask_bootstrap import Bootstrap
import sys
import os, os.path

from flask import Flask, render_template, redirect, url_for, request

from flask_bootstrap import Bootstrap
from flask_wtf import FlaskForm
from wtforms import StringField, SubmitField
from wtforms.validators import DataRequired

import whoosh
from whoosh.index import create_in
from whoosh.index import open_dir
from whoosh.fields import *
from whoosh.qparser import QueryParser
from whoosh.qparser import MultifieldParser
from whoosh import qparser

import pandas as pd
from whoosh.fields import Schema, TEXT
from whoosh import index
import sys
import csv

app = Flask(__name__)

app.config['SECRET_KEY'] = 'NOJERRYSALLOWEDbcSHBXUXs2123'

bootstrap =Bootstrap(app)

@app.route('/', methods=['GET', 'POST'])
def index():

	return render_template('welcome_page.html')

@app.route('/results/', methods=['GET', 'POST'])
def results():
	global mysearch
	if request.method == 'POST':
		data = request.form
	else:
		data = request.args

	keywordquery = data.get('searchterm')

	print('Keyword Query is: ' + keywordquery)

	#titles, description = mysearch.search(keywordquery)

	item, description, picture, productType, brand, size, price, url, onsale, discount = mysearch.index_search(keywordquery)

	return render_template('results.html', query=keywordquery, len = len(item), item =item, description= description, picture =picture, productType = productType, brand = brand, size = size, price =price, url =url, onsale = onsale, discount =discount)

#approutes=============================================================================

class NameForm(FlaskForm):
    name = StringField('Which actor is your favorite?', validators=[DataRequired()])
    submit = SubmitField('Submit')

# import data into pandas df and create index schema

class wooshSearch(object):

	def __init__(self):
		super(wooshSearch,self).__init__()

	def index(self):

		with open(r"SearchEngineData2.csv") as csv_file:
			csv_reader = csv.reader(csv_file, delimiter=',')
			df = pd.DataFrame([csv_reader], index=None)
			df.head()
		schema = Schema(item=TEXT(stored=True), description=TEXT(stored=True), picture=TEXT(stored = True),productType=TEXT(stored = True),brand=TEXT(stored = True),size=TEXT(stored = True),price=TEXT(stored = True),url=TEXT(stored = True),onsale=TEXT(stored = True),discount=TEXT(stored = True))

		ix = create_in('exampleIndex', schema)

		# Imports stories from pandas df

		for i in range(len(list(df))):		#go the length of the index

			for val in list(df[i]):

				writer = ix.writer() 		#add documents

				writer.add_document(item =(val[0]),	#val[] are the positions in the index piping it to the schema
						   description =(val[1]),
						   picture =(val[2]),
						   productType =(val[3]),
						   brand =(val[4]),
						   size =(val[5]),
						   price =(val[6]),
						   url =(val[7]),
						   onsale =(val[8]),
						   discount =(val[9]))

				writer.commit()
				self.ix = ix

				print ("writing doc")

										#commit the document

	# creates index searcher

	def index_search(self,queryEntered):

		item = list()
		description = list()
		picture= list()
		productType= list()
		brand= list()
		size= list()
		price= list()
		url= list()
		onsale= list()
		discount= list()

		ix = open_dir('exampleIndex')
		schema = ix.schema
		# Create query parser that looks through designated fields in index
		og = qparser.OrGroup.factory(0.9)
		mp = qparser.MultifieldParser(['item', 'description','picture','productType','brand','size','price','url','onsale','discount'], schema =self.ix.schema, group = og)

		# This is the user query
		#query = input("Please enter a Title, Year, Rating, IMDB tag, or some type of key word description:")
		q = mp.parse(queryEntered)
		#threshold = int(input("How many results would you like?:"))
		# Actual searcher, prints top 10 hits
		with ix.searcher() as s:
			results = s.search(q, limit = None)	#user threshold
			print("Search Results: ")
			try:
				for i in results:
					#print("\n RESULT #:",i+1 )			#prints resuts
					#print(results[i])
					#print("\n")

					item.append(i['item'])
					description.append(i['description'])
					picture.append(i['picture'])
					productType.append(i['productType'])
					brand.append(i['brand'])
					size.append(i['size'])
					price.append(i['price'])
					url.append(i['url'])
					onsale.append(i['onsale'])
					discount.append(i['discount'])

			except IndexError:							#if it doesnt find x number of results it catches the error
				pass									#program goes into the infinite while loop for another query
		return  item, description, picture, productType, brand, size, price, url, onsale, discount

if __name__ == '__main__':

	global mysearch

	mysearch = wooshSearch()

	mysearch.index()

	#print("\n\n Welcome to the Snowboard search engine")

	#wooshSearch().index_search("Snowboard_Index", ['item', 'description','picture','productType','brand','size','price','url','onsale','discount'],keywordquery)

	app.run(debug=True)


from flask import Flask, render_template, request, redirect
import mysql.connector

app = Flask(__name__)

# Configure MySQL connection
db = mysql.connector.connect(
    host="localhost",
    user="root",
    database="dbsorya_db"
)

# Create a cursor to interact with the database
cursor = db.cursor()

# Define the data model
class InventoryItem:
    def __init__(self, id, name, quantity, price):
        self.id = id
        self.name = name
        self.quantity = quantity
        self.price = price

# Home page to display all inventory items
@app.route("/")
def home():
    # Fetch all items from the database
    cursor.execute("SELECT * FROM inventory")
    rows = cursor.fetchall()

    # Create InventoryItem objects for each row
    inventory_items = []
    for row in rows:
        item = InventoryItem(row[0], row[1], row[2], row[3])
        inventory_items.append(item)

    # Render the template with the inventory data
    return render_template("home.html", inventory=inventory_items)

# Add new item to the inventory
@app.route("/add", methods=["GET", "POST"])
def add_item():
    if request.method == "POST":
        name = request.form["name"]
        quantity = int(request.form["quantity"])
        price = float(request.form["price"])

        # Insert the new item into the database
        cursor.execute("INSERT INTO inventory (name, quantity, price) VALUES (%s, %s, %s)", (name, quantity, price))
        db.commit()

        return redirect("/")
    else:
        return render_template("add.html")

# Edit an existing item in the inventory
@app.route("/edit/<int:item_id>", methods=["GET", "POST"])
def edit_item(item_id):
    if request.method == "POST":
        name = request.form["name"]
        quantity = int(request.form["quantity"])
        price = float(request.form["price"])

        # Update the item in the database
        cursor.execute("UPDATE inventory SET name=%s, quantity=%s, price=%s WHERE id=%s", (name, quantity, price, item_id))
        db.commit()

        return redirect("/")
    else:
        # Fetch the item from the database
        cursor.execute("SELECT * FROM inventory WHERE id = %s", (item_id,))
        row = cursor.fetchone()
        item = InventoryItem(row[0], row[1], row[2], row[3])

        return render_template("edit.html", item=item)

# Delete an item from the inventory
@app.route("/delete/<int:item_id>")
def delete_item(item_id):
    # Delete the item from the database
    cursor.execute("DELETE FROM inventory WHERE id = %s", (item_id,))
    db.commit()

    return redirect("/")

if __name__ == "__main__":
    app.run(debug=True)


from typing import List, Tuple

import numpy as np
import numpy.typing as npt

Games = npt.NDArray[np.int64]

def main(input_lines: List[str]) -> Tuple[int, int]:
    game_rounds: Games = np.empty(shape=(len(input_lines), 2), dtype=np.int64)
    for i, line in enumerate(input_lines):
        for j, char in enumerate(line.split(" ")):
            val: int = -1
            if char in ["A", "X"]:
                val = 0
            elif char in ["B", "Y"]:
                val = 1
            elif char in ["C", "Z"]:
                val = 2
            game_rounds[i, j] = val

    part1_answer: int = compute_score(*part1_get_responses_outcomes(game_rounds))
    part2_answer: int = compute_score(*part2_get_responses_outcomes(game_rounds))
    return part1_answer, part2_answer

def compute_score(responses: Games, outcomes: Games) -> int:
    scores: Games = (responses + 1) + outcomes * 3
    return int(np.sum(scores))

def part1_get_responses_outcomes(game_rounds: Games) -> Tuple[Games, Games]:
    responses: Games = game_rounds[:, 1]
    outcomes: Games = np.mod(responses - game_rounds[:, 0] + 1, 3)
    return responses, outcomes

def part2_get_responses_outcomes(game_rounds: Games) -> Tuple[Games, Games]:
    outcomes: Games = game_rounds[:, 1]
    responses: Games = np.mod(outcomes + game_rounds[:, 0] - 1, 3)
    return responses, outcomes


"""empty message

Revision ID: 6d8e9e4138bf
Revises: 445667ce6268
Create Date: 2016-03-03 10:36:03.205829

"""

# revision identifiers, used by Alembic.
revision = '6d8e9e4138bf'
down_revision = '445667ce6268'

from alembic import op
import app
import sqlalchemy as sa

def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.add_column('provas', sa.Column('data_inicio', sa.DateTime(), nullable=True))
    op.add_column('provas', sa.Column('tempo_execucao', sa.Integer(), nullable=True))
    ### end Alembic commands ###

def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('provas', 'tempo_execucao')
    op.drop_column('provas', 'data_inicio')
    ### end Alembic commands ###


# Medium
# https://leetcode.com/problems/asteroid-collision/
# TC:O(N)
# SC:O(N)
class Solution:
    def asteroidCollision(self, asteroids: List[int]) -> List[int]:
        stack = []
        for curr in asteroids:
            while stack and curr < 0 < stack[-1]:
                if -curr > stack[-1]:
                    stack.pop()
                    continue
                elif -curr == stack[-1]:
                    stack.pop()
                break
            else:
                stack.append(curr)
        return stack


import string
import random
import matplotlib.pyplot as plt
import pandas as pd

# Читаем данные из файла и сохраняем их в списке
with open('names.txt', 'r') as f:
    names = [name.strip().lower() for name in f.readlines()]

# Создаем список букв, которые мы будем использовать для генерации имен
alphabet = list(string.ascii_lowercase) + ['^', '$']

# Создаем словарь, в котором будем хранить частоты каждого биграмма
bigram_counts = {}

# Проходимся по каждому имени в списке и считаем частоты каждого биграмма
for name in names:
    # Добавляем начальный и конечный символы к имени
    name = '^' + name + '$'

    # Разбиваем имя на биграммы
    bigrams = [name[i:i+2] for i in range(len(name)-1)]

    # Считаем частоты каждого биграмма
    for bigram in bigrams:
        if bigram in bigram_counts:
            bigram_counts[bigram] += 1
        else:
            bigram_counts[bigram] = 1

# Вычисляем общее количество биграмм
total_bigrams = sum(bigram_counts.values())

# Создаем словарь, в котором будем хранить вероятности каждого биграмма
bigram_probs = {}

# Вычисляем вероятности каждого биграмма
for bigram, count in bigram_counts.items():
    bigram_probs[bigram] = count / total_bigrams

def generate_name():
    # Выбираем случайную первую букву из списка алфавита
    first_letter = random.choice(alphabet[:-2]) # Исключаем "^" и "$"

    # Инициализируем имя начальной буквой
    name = first_letter

    # Пока последняя буква имени не является конечным символом "$"
    while name[-1] != '$':
        # Получаем все возможные биграммы для последней буквы имени
        possible_bigrams = [bigram for bigram in bigram_probs.keys() if bigram.startswith(name[-1])]

        # Вычисляем вероятности для каждого возможного биграмма
        probabilities = [bigram_probs[bigram] for bigram in possible_bigrams]

        # Выбираем случайный биграмм на основе вероятностей
        next_bigram = random.choices(possible_bigrams, weights=probabilities)[0]

        # Добавляем следующую букву к имени
        name += next_bigram[1]

    # Удаляем начальный и конечный символы из имени и возвращаем его
    return name[:-1]

# Выводим сгенерированное
print(generate_name())

# Создаем пустой DataFrame для хранения данных о вероятностях биграмм
df = pd.DataFrame(columns=alphabet, index=alphabet)

# Заполняем DataFrame вероятностями для каждого биграмма
for bigram, prob in bigram_probs.items():
    df.loc[bigram[0], bigram[1]] = prob

# Заполняем пропущенные значения нулями
df.fillna(0, inplace=True)

# Выводим таблицу
print(df)

# Создаем список биграмм и соответствующих вероятностей
bigrams = list(bigram_probs.keys())
probs = list(bigram_probs.values())

# Рисуем график
plt.bar(bigrams, probs)
plt.title("Bigram probabilities")
plt.xlabel("Bigram")
plt.ylabel("Probability")
plt.show()

# Создаем heatmap на основе DataFrame
plt.imshow(df.values, cmap='viridis')

# Настраиваем оси и заголовок
plt.xticks(range(len(df.columns)), df.columns)
plt.yticks(range(len(df.index)), df.index)
plt.xlabel('Second letter')
plt.ylabel('First letter')
plt.title('Bigram probabilities')

# Выводим картинку
plt.show()


# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, val=0, left=None, right=None):
#         self.val = val
#         self.left = left
#         self.right = right
class Solution:
    def rightSideView(self, root: Optional[TreeNode]) -> List[int]:
        _dict = defaultdict(list)
        ans = []

        def breadth_first(root, level):
            if not root:
                return

            _dict[level].append(root.val)
            breadth_first(root.left, level + 1)
            breadth_first(root.right, level + 1)

        breadth_first(root, 0)

        for key in _dict:
            ans.append(_dict[key][-1])

        return ans


# importing libararies
import cv2
from datetime import datetime

def record():
    # define video capture object
    cap = cv2.VideoCapture(0)

    # define the codec and create videowriter object
    fourcc = cv2.VideoWriter_fourcc(*'XVID')
    store = cv2.VideoWriter(f'recording/{datetime.now().strftime("%H-%M-%S")}.avi', fourcc, 20.0, (640, 480))

    # loop runs if capturing has been initialized
    while True:

        # capture the video frame
        # by frame
        # ret checks return at each frame
        ret, frame = cap.read()

        # using cv2.putText() method
        cv2.putText(frame,f'{datetime.now().strftime("%D-%H-%M-%S")}', (50, 50), cv2.FONT_HERSHEY_COMPLEX, 0.6, (255, 0, 255), 2)

        # output the frame
        store.write(frame)

        # display the resulting frame
        cv2.imshow('Webcam', frame)

        # the 'q' button is set as the
        # quitting button
        if cv2.waitKey(1) & 0xFF == ord('q'):

            break

    # after the loop release the cap object
    cap.release()
    # destroy all the windows
    cv2.destroyAllWindows()


from datetime import datetime, timezone
from enum import Enum
from typing import List, Tuple

import numpy as np
import pykalman
import statsmodels.api as sm
from influxdb import InfluxDBClient

#
# Alerts
#

def get_last_alert_status_and_timestamp(influxdb: InfluxDBClient) -> Tuple[bool, datetime]:
    results = list(influxdb.query(
        'SELECT * FROM temperatures.autogen.alert ORDER BY time DESC LIMIT 1').get_points())
    if not results:
        return False, datetime.utcfromtimestamp(0).replace(tzinfo=timezone.utc)
    return results[-1]['status'] != 'off', _parse_timestamp_str(results[-1]['time'])

def save_alert_status(influxdb: InfluxDBClient, is_on: bool) -> None:
    influxdb.write_points([f'alert,status={"on" if is_on else "off"} value=0'], protocol='line', time_precision='ms')

#
# Watchers
#

def set_watcher_enabled(influxdb: InfluxDBClient, chat_id: str, enabled: bool) -> None:
    influxdb.write_points([f'watcher,status={"on" if enabled else "off"},chat_id={chat_id} value=0'],
                          protocol='line', time_precision='ms')

def get_watcher_enabled(influxdb: InfluxDBClient, chat_id: str):
    results = list(influxdb.query(
        'SELECT * FROM temperatures.autogen.watcher'
        f' WHERE chat_id=\'{chat_id}\' ORDER BY time DESC LIMIT 1').get_points())
    return not results or results[-1]['status'] != 'off'

#
# Measurements
#

def save_measurements(influxdb: InfluxDBClient, measurements: List[Tuple[str, float]]) -> None:
    # Write raw data points
    now = datetime.now().timestamp()
    latest_clock_time = None
    temperature_timestamps = []
    for key, temperature in measurements:
        epoch, clock_time = key.split('_')
        clock_time = int(epoch) * (2 ** 32) + int(clock_time)
        if latest_clock_time is None:
            latest_clock_time = clock_time
        timestamp = int(now * 1000 - (latest_clock_time - clock_time))
        temperature_timestamps.append((temperature, timestamp))
    temperature_timestamps.reverse()
    influxdb.write_points([f'temperature value={temp} {ts}' for temp, ts in temperature_timestamps],
                          protocol='line', time_precision='ms')

    # Updating kalman filter data
    mean, cov = get_kalman_mean_cov(influxdb)
    kf = pykalman.AdditiveUnscentedKalmanFilter()
    lines = []
    mean, cov = [mean], [cov]
    for temp, ts in temperature_timestamps:
        mean, cov = kf.filter_update(mean, cov, temp)
        lines.append(f'temperature_filtered value={mean.item()} {ts}')
    save_kalman_mean_cov(influxdb, mean.item(), cov.item())
    influxdb.write_points(lines, protocol='line', time_precision='ms')

def get_kalman_mean_cov(influxdb: InfluxDBClient) -> Tuple[float, float]:
    results = list(influxdb.query(
        'SELECT * FROM temperatures.autogen.kalman ORDER BY time DESC LIMIT 1').get_points())
    if results:
        mean = results[-1]['mean']
        cov = results[-1]['cov']
    else:
        # Initializing Kalman filter
        temperatures = list(influxdb.query(
            'SELECT value FROM temperatures.autogen.temperature WHERE time > now()-120m'
        ).get_points())
        kf = pykalman.AdditiveUnscentedKalmanFilter()
        filtered_state_means, filtered_state_covariances = kf.filter([t['value'] for t in temperatures])
        mean = filtered_state_means[-1].item()
        cov = filtered_state_covariances[-1].item()
        save_kalman_mean_cov(influxdb, mean, cov)
    return mean, cov

def save_kalman_mean_cov(influxdb: InfluxDBClient, mean: float, cov: float) -> None:
    influxdb.write_points([f'kalman mean={mean},cov={cov}'], protocol='line', time_precision='ms')

def get_last_measurement_timestamp(influxdb: InfluxDBClient) -> datetime:
    results = list(influxdb.query(
        'SELECT * FROM temperatures.autogen.temperature ORDER BY time DESC LIMIT 1').get_points())
    if not results:
        return datetime.utcfromtimestamp(0).replace(tzinfo=timezone.utc)
    return _parse_timestamp_str(results[-1]['time'])

#
# Status
#

def get_current_temperature(influxdb: InfluxDBClient) -> float:
    results = list(influxdb.query(
        'SELECT * FROM temperatures.autogen.temperature_filtered ORDER BY time DESC LIMIT 1').get_points())
    if not results:
        return 0
    return _measurement_to_temperature(results[-1]['value'])

class FurnaceStatus(Enum):
    HEATING_UP = 'HEATING_UP'
    COOLING_DOWN = 'COOLING_DOWN'
    NOT_TRENDING = 'NOT_TRENDING'

def get_furnace_status(influxdb: InfluxDBClient) -> FurnaceStatus:
    temps = list(influxdb.query(
        'SELECT value FROM temperatures.autogen.temperature_filtered WHERE time > now()-3m'
    ).get_points())
    if temps:
        ys = np.array([e['value'] for e in temps])
        xs = sm.add_constant(np.array(range(ys.size)), prepend=False)
        result = sm.OLS(ys, xs).fit()
        if result.rsquared >= 0.8:
            return FurnaceStatus.COOLING_DOWN if result.conf_int()[0].mean() < 0 else \
                FurnaceStatus.HEATING_UP if result.conf_int()[0].mean() > 0 else FurnaceStatus.NOT_TRENDING
    return FurnaceStatus.NOT_TRENDING

#
# Congrats
#

def get_last_congrat_timestamp(influxdb: InfluxDBClient) -> datetime:
    results = list(influxdb.query(
        'SELECT * FROM temperatures.autogen.congrat ORDER BY time DESC LIMIT 1').get_points())
    if not results:
        return datetime.utcfromtimestamp(0).replace(tzinfo=timezone.utc)
    return _parse_timestamp_str(results[-1]['time'])

def save_congrat(influxdb: InfluxDBClient, temperature: float) -> None:
    influxdb.write_points([f'congrat value={temperature}'], protocol='line', time_precision='ms')

#
# Misc
#

def _parse_timestamp_str(timestamp_str: str) -> datetime:
    format_str = '%Y-%m-%dT%H:%M:%S.%fZ' if len(timestamp_str) > 20 else '%Y-%m-%dT%H:%M:%SZ'
    return datetime.strptime(timestamp_str, format_str).replace(tzinfo=timezone.utc)

def _measurement_to_temperature(measurement: float) -> float:
    return measurement


import json
from defines_json import JSON_FILE

class ReadJson():
    key_search = "title"

    def check_title_json_file(self, titles, dir= JSON_FILE):
        """
            Read Json file and search for given title in it
            param: title,
            param: dir(optional).
        """
        json_file = json.loads(open(dir).read())

        for item in json_file:
            for title in titles:
                if title.lower() in item[self.key_search].lower():
                    print(item)


import datetime
import os
import pygame
import pygame.camera
import pygame.image
import time

INTERVAL = 10.0  # 撮影間隔
IMAGE_WIDTH = 1280  # 写真の幅
IMAGE_HEIGHT = 720  # 写真の高さ
base_dir = '/home/pi/Documents/Projects/timelapse'  # プログラム用ディレクトリー名
data_dir = base_dir + '/data'  # データ用ディレクトリー名
if not os.path.exists(data_dir):  # データ用ディレクトリーがない場合は作成
    os.mkdir(data_dir)

# カメラ用意
pygame.init()
pygame.camera.init()
camera_list = pygame.camera.list_cameras()
camera = pygame.camera.Camera(camera_list[0], (IMAGE_WIDTH, IMAGE_HEIGHT))

# 撮影
for cnt in range(150):  # 150回撮影したら終了
    now = datetime.datetime.now()  # 現在時刻の取得
    now_str = now.strftime('%Y%m%d%H%M%S')  # 年月日時分秒の文字列の取得
    file_name = data_dir + '/' + now_str + '.jpg'  # 写真ファイル名の生成
    print(str(cnt) + ':' + file_name)
    camera.start()
    img = camera.get_image()  # カメラ撮影
    pygame.image.save(img, file_name)  # ファイル保存
    camera.stop()
    td = datetime.datetime.now() - now  # コマンド実行後の時刻取得
    micro_td = (td / datetime.timedelta(microseconds=1)) / 1000000.0  # コマンド実行にかかった秒数算出
    time.sleep(INTERVAL - micro_td)  # コマンド実行分も考慮してINTERVAL秒休止


from src.crawler.base_crawler import BaseCrawler
from bs4 import BeautifulSoup
import pandas as pd
from pprint import pprint
import re
import json
import os
import requests
from PIL import Image
import requests
import copy
import torchvision.transforms as transforms
from torchvision.transforms import ToTensor,Grayscale,ToPILImage
"""
#時系列review
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException,StaleElementReferenceException
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
import datetime
"""

def one_to_three(img):
    if img.shape[0]!=3:
        transform=transforms.Compose(
            [
                transforms.ToPILImage(),
                Grayscale(num_output_channels=3)
            ]
        )
        return transform(img)
    else:
        transform=transforms.ToPILImage()
        return transform(img)

transform=transforms.Compose(
                    [
                        transforms.ToTensor(),
                        transforms.Lambda(one_to_three),
                        transforms.Resize([300,256]),
                        transforms.ToTensor()
                    ])

#データ保存用ディレクトリ
di="data/Feature_data"
idi="data/image"
jidi="data/jimage"

#時系列review
firefox='/home/kwk/share20A/dentsu/Sugawara/example/fire/usr/bin/firefox'
driver=r'/home/kwk/share/dentsu/Hisamitsu/lib/geckodriver'

def getHTMLText(url):
    headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36'}
    r = requests.get(url, headers=headers)
    r.raise_for_status()
    return r.text

def load_image(file_name,option="eng",idi=idi,jidi=jidi):
    if option == "jap":
        idi = jidi
    elif option != "eng":
        print("please choose eng or jap")
        raise Exception

    try:
        im = Image.open(idi+"/"+file_name+".jpg")
        image= transform(im)

        im.close()
        return image
    except Exception as e:
        print(e,"please load image of "+file_name)

class Feature_Crawler(BaseCrawler):
    def __init__(self,start=0,last=0,order_list=None,order_name=None,info=[],movie_only=True,di=di,idi=idi,jidi=jidi,firefox=firefox,driver=driver):
        super().__init__()
        self.idi=idi
        self.jidi=jidi
        self.di=di
        self.firefox=firefox
        self.driver=driver
        self.start=start
        self.last=last
        self.order_list=order_list
        self.order_name=order_name
        self.movie_only=movie_only

        #欲しい特徴の『名前』1 こちらは『処理』と順を揃える
        self.feature_list=["titleId",'image_exist','j_image_exist','story_line','domestic_money','international_money','full_money']
        #他['domestic_rate','international_rate',"dates","stars","reviews"]

        # 特徴の『名前』2 順不同
        self.all_feature=[  'contentRating',  'description','datePublished', 'keywords']
        #'name','review',@context', '@type', 'url','image','actor','director','creator','genre','duration'

        # 特徴の『名前』3 順不同
        self.detail_feature=['aggregateRating']
        self.feature_dict={'aggregateRating':['ratingCount','bestRating','worstRating','ratingValue']}
        #'trailer'

        self.feature_list.extend(self.all_feature)
        for f in self.detail_feature:
            self.feature_list.extend(self.feature_dict[f])
        self.load_csv()

    def get_feature(self, movie_id):
        try:
            os.makedirs(self.idi)
        except FileExistsError:
            pass
        try:
            os.makedirs(self.jidi)
        except FileExistsError:
            pass
        res = self.get_response('https://www.imdb.com/title/' + movie_id + '/')
        soup = BeautifulSoup(markup=res.content, features='html.parser')
        try:
            json_data = soup.find('script', {'type' : 'application/ld+json'})
        except Exception as e:
            return None
        parsed_json = json.loads(json_data.contents[0])
        kind = soup.find("a",href="/search/title?genres=movie&explore=title_type,genres")

        if parsed_json["@type"]=="Movie" and self.movie_only:

            #欲しい特徴を得る『処理』1(名前と順を揃える)
            features=[]

            #画像
            img_url = parsed_json.get('image')
            try:
                res_img = self.get_response(img_url)
                self.save_image(movie_id,res_img)
            except Exception as e:
                img_url=None
            features.append(img_url)

            #日本画像
            try:
                j_im="exist"
                self.j_getPoster(soup.find("title").text.split("(")[0],movie_id)
            except Exception as e:
                j_im=None
            if not os.path.exists(self.jidi+"/"+movie_id+".jpg"):
                j_im=None
            features.append(j_im)

            #story
            story_data=soup.find("div",{'class':"inline canwrap"})
            if story_data is None:
                story=None
            else:
                story=str(story_data.getText())[6:-2]
            features.append(story)

            #money
            try:
                box_url = "https://www.boxofficemojo.com/title/"+movie_id
                box_html = getHTMLText(box_url)
                box_soup = BeautifulSoup(box_html, 'html.parser')
                percent = box_soup.find_all('span', attrs = {'class':'percent'})
                money = box_soup.find_all('span', attrs = {'class':'money'})
                p = [pe.text for pe in percent]
                m = [float(mo.text.replace(",","")[1:]) for mo in money]
                if m[0]==m[1] and (p[0]=='–' or p[1]=='–'):
                    if p[0]=='–':
                        result = [None,m[0],m[1]]
                    elif p[1]=='–':
                        result = [m[0],None,m[1]]
                else:
                    result = [m[0],m[1],m[2]]
                #money rate
                #features.extend(p)
                features.extend(result)
            except Exception as e:
                #features.extend([None,None])
                features.extend([None,None,None])
            """"
            #時系列review

            try:
                features.extend(self.get_reviews(movie_id))
            except Exception as e:

                features.extend([None]*3)
            """

            #『処理』2 順不同
            for name in self.all_feature:
                try:
                    features.append(parsed_json[name])
                except Exception as e:
                    features.append(None)

            #『処理』3 順不同
            for name in self.detail_feature:
                try:
                    detail=eval(str(parsed_json[name]))
                    for d_name in self.feature_dict[name]:
                        features.append(float(detail[d_name]))
                except Exception as e:
                    features.extend([None]*len(self.feature_dict[name]))

            return features
        else:
            return None

    def get_reviews(self, movie_id):
        try:
            os.makedirs(self.di)
        except FileExistsError:
            pass
        options = Options()
        options.headless = True
        options.binary=FirefoxBinary(self.firefox)
        driver = webdriver.Firefox(executable_path=self.driver,options=options)
        driver.get('https://www.imdb.com/title/' + movie_id + '/reviews')
        while True:
            try:
                showmore=WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH ,'//*[@id="load-more-trigger"]')))
                showmore.click()
            except TimeoutException:
                break
            except StaleElementReferenceException:
                break
        soup = BeautifulSoup(driver.page_source,'html.parser')
        dates=[]
        stars=[]
        reviews=[]

        for review_container in soup.find_all('div',{'class':'lister-item-content'}):
            #star
            try:
                star = review_container.find('span',{'class':'rating-other-user-rating'}).find('span').getText()
            except:
                star = "none"

            #review
            try:
                review = review_container.find('div',{'class':'text show-more__control'}).getText()
            except:
                review = "none"
            #date
            review_date = review_container.find('span',{'class':'review-date'}).getText()
            formated_date = datetime.datetime.strptime(review_date, '%d %B %Y').strftime('%Y/%m/%d') #YYYY/mm/dd HH:MM +0000に変換
            dates.append(formated_date)
            stars.append(star)
            reviews.append(reviews)

            #json_data = soup.find_all('div',{'class':'text show-more__control'})

            #print(json_data[0])
        return  [dates,stars,reviews]

    def j_getPoster(self, kw, imdbNo):
        root=self.jidi
        #キーワードを検索
        url = 'https://filmarks.com/search/movies?q=' + kw
        html = getHTMLText(url)
        soup = BeautifulSoup(html, 'html.parser')
        a = soup.find('a', attrs = {'class':'p-content-cassette__readmore'})
        url = 'https://filmarks.com/' + a['href']
        #映画の詳細ページを取得
        html = getHTMLText(url)
        soup = BeautifulSoup(html, 'html.parser')
        div = soup.find('div', attrs = {'class':'c-content c-content--large'})
        for i in div.descendants:
            if i.name == 'img':
                url = i['src']
                if url[-3:] == 'svg':
                    return
                break
        #ポスターのリンクから画像を保存
        r = requests.get(url)
        file = root + "/"+ imdbNo + '.jpg'
        with open(file, 'wb') as f:
            f.write(r.content)
            f.close()

    def save_image(self, file_name, res_img):
        if not os.path.exists(self.idi+"/"+file_name+".jpg"):
            try:
                with open(self.idi+"/"+file_name+".jpg", 'wb') as img_file:
                    img_file.write(res_img.content)
                print('Save image for ' + file_name)
            except Exception as e:
                print('Saving image failure for ' + file_name)

    def movie_num(self,n):
        return "tt"+(7-len(str(n)))*"0"+str(n)

    @property
    def feature_di(self):
        if self.order_name is None:
            return self.di+"/"+"_s"+str(self.start)+"_l"+str(self.last)+".csv"
        #"_".join(self.feature_list)+
        else:
            return self.di+"/"+self.order_name+".csv"

    def save_as_csv(self):
        try:
            os.makedirs(self.di)
        except FileExistsError:
            pass
        pd_list=[]
        search=None
        if self.order_name is None:
            search=range(self.start,self.last+1,1)
        else:
            search=self.order_list

        for i in search:
            try:
                if self.order_name is None:
                    movie_id=self.movie_num(i)
                else:
                    movie_id=i
                feature=self.get_feature(movie_id)
                id_feature=[movie_id]
                if feature !=  None:
                    id_feature.extend(feature)
                    pd_list.append(id_feature)
            except Exception as e:
                print(e)
                continue
        pd_list=pd.DataFrame(pd_list,columns=self.feature_list)
        pd_list.to_csv(self.feature_di)

        return pd_list

    def load_csv(self):
        if not os.path.exists(self.feature_di):
            self.feature=self.save_as_csv()
        self.feature=pd.read_csv(self.feature_di,index_col=1)
        self.feature=self.feature.drop('Unnamed: 0',axis=1)


#начало функции
def add(a, b):  # функцию зовут add, она принимает 2 аргумента а и b
       return a + b # а здесь функция возвратит результат а+b

y= add (1, 2) # у примет то что возвратит функция add в которую мы передали аргументы а=1 b=2
print ('y = ',y) # выводим значение y

z= add (int(input("a= ")), 2) # а тут передадим в а то что введет пользователь
print ('z = ',z) # выводим значение y

variable= add(500,-500)
print ('zvariable = ',variable)

a = int(input("a= "))#Ввод 1го числа.
b = int(input("b= "))#Ввод 2го числа .
c = int(input("c= "))  #Ввод 3го числа.

input_data = { 'a' : 0,
               'b' : 0,
               'c' : 0
}


import math

ACKLEY_MIN = -32.768  #-32.768 -4
ACKLEY_MAX = 32.768   #32.768 4
ACKLEY_STEP = 1 #0.2
ACKLEY_SIGMA = 6

LEVY_MIN = -10
LEVY_MAX = 10
LEVY_STEP = 0.2
LEVY_SIGMA = 2

GRIEWANK_MIN = -10    #-600
GRIEWANK_MAX = 10     #600
GRIEWANK_STEP = 0.2
GRIEWANK_SIGMA = 2

RASTRIGIN_MIN = -5.12
RASTRIGIN_MAX = 5.12
RASTRIGIN_STEP = 0.1
RASTRIGIN_SIGMA = 1

SCHWEFEL_MIN = -500
SCHWEFEL_MAX = 500
SCHWEFEL_STEP = 5
SCHWEFEL_SIGMA = 100

ZAKHAROV_MIN = -10
ZAKHAROV_MAX = 10
ZAKHAROV_STEP = 0.2
ZAKHAROV_SIGMA = 2

SPHERE_MIN = -5.12
SPHERE_MAX = 5.12
SPHERE_STEP = 0.1
SPHERE_SIGMA = 1

MICHALEWICZ_MIN = 0
MICHALEWICZ_MAX = math.pi
MICHALEWICZ_STEP = 0.1
MICHALEWICZ_SIGMA = 0.3

ROSENBROCK_MIN = -10
ROSENBROCK_MAX = 10
ROSENBROCK_STEP = 0.2
ROSENBROCK_SIGMA = 2

ranges = {
    "ackley": (ACKLEY_MIN, ACKLEY_STEP, ACKLEY_MAX),
    "levy": (LEVY_MIN, LEVY_STEP, LEVY_MAX),
    "griewank": (GRIEWANK_MIN, GRIEWANK_STEP, GRIEWANK_MAX),
    "schwefel":  (SCHWEFEL_MIN, SCHWEFEL_STEP, SCHWEFEL_MAX),
    "zakharov": (ZAKHAROV_MIN, ZAKHAROV_STEP, ZAKHAROV_MAX),
    "sphere": (SPHERE_MIN, SPHERE_STEP, SPHERE_MAX),
    "michalewicz": (MICHALEWICZ_MIN, MICHALEWICZ_STEP, MICHALEWICZ_MAX),
    "rastrigin": (RASTRIGIN_MIN, RASTRIGIN_STEP, RASTRIGIN_MAX),
    "rosenbrock": (ROSENBROCK_MIN, ROSENBROCK_STEP, ROSENBROCK_MAX),
}

sigmas = {
    "ackley": ACKLEY_SIGMA,
    "levy": LEVY_SIGMA,
    "griewank": GRIEWANK_SIGMA,
    "schwefel": SCHWEFEL_SIGMA,
    "zakharov": ZAKHAROV_SIGMA,
    "sphere": SPHERE_SIGMA,
    "michalewicz": MICHALEWICZ_SIGMA,
    "rastrigin": RASTRIGIN_SIGMA,
    "rosenbrock": ROSENBROCK_SIGMA,
}

def get_min_range(name_function):
    if ranges.get(name_function):
        return ranges[name_function][0]
    return 0

def get_max_range(name_function):
    if ranges.get(name_function):
        return ranges[name_function][2]
    return 0

def get_range(name_function):
    if ranges.get(name_function):
        return ranges[name_function][1]
    return 0

def get_ranges(name_function):
    if ranges.get(name_function):
        return ranges[name_function]
    return (0, 0, 0)

def get_sigma(name_function):
    if sigmas.get(name_function):
        return sigmas[name_function]
    return 0

X=16


from django.shortcuts import render, redirect, HttpResponseRedirect
from django.views.generic import ListView

# local
from mainapp.forms import PackageForm
from mainapp.models import Package

class PackageList(ListView):
    model = Package
    context_object_name = "package_list"
    template_name = 'main/package/list.html'

    def get_queryset(self):
        query = self.request.GET.get("s")
        if query:
            return Package.objects.filter(name__startswith=query)
        else:
            return Package.objects.order_by("-id")

    def get_context_data(self, *args, **kwargs):
        data = super().get_context_data(**kwargs)
        data['page_title'] = 'Authors'
        data['form'] = PackageForm()
        return data

def add_package(request):
    package_form = PackageForm()

    if request.method == "POST":
        package_form = PackageForm(request.POST)
        if package_form.is_valid():
            package_form.save()

    context = {
        'form': package_form
    }
    return render(request, 'main/package/add.html', context)

def edit_package(request, id):
    package = Package.objects.get(pk=id)
    package_form = PackageForm(instance=package)

    if request.method == "POST":
        package_form = PackageForm(instance=package, data=request.POST)
        package_form.save()
    context = {
        'form': package_form
    }
    return render(request, 'main/package/add.html', context)

def delete_package(request, id):
    if request.method == "POST":
        package = Package.objects.get(pk=id)
        package.delete()
    return HttpResponseRedirect(request.META.get('HTTP_REFERER'))


# 1 多线程编程

# 线程概念和标准库threading

'''线程是操作系统调度的基本单位,负责执行包含在进程地址空间中的代码并反问其中的资源,当一个进程被创建时
操作系统会自动位置创建一个线程,通常称为主线程.一个进程可以包含多个线程,主线程根据需要在动态创建其他
子线程,操作系统为每个线程保存单独的寄存器环境和单独的对战,但是他们共享进程的地址空间,对象句柄,代码,数据和其他资源.
线程总是在某个进程的上下文中被创建,运行和结束,不可以脱离进程而独立存在.但允许属于同一个进程的多个线程之间进行数据共享
和同步控制.一般而言,除主线程'''

'''标准库threading是python支持多线程编程的重要模块,该模块实在底层模块_thread的基础上开发的更高层次的线程
编程接口,提供了大量的方法和类来支持多线程编程'''

"""
active_count(),activeCount()        返回当前处于alive状态的Thread对象数量
current_thread(),currentThread()    返回当前Thread对象
get_indent()                     返回当前线程的线程标识符,线程标识符是一个非负整数,这个整数本身没有并没有特殊含义
                                 只是用来标识线程,可能会被循环利用
enumerate()                      返回处于alive状态的所有Thread对象列表
local()                          线程局部数据类
main_thread()                    返回主线程对象,即启动python解释器的线程对象
stack_size([size])               返回创建线程时使用的栈的大小,如果指定size参数,则用来制定后续创建的线程使用的栈的大小,
                                 size必须是0(表示使用系统默认值)或大于32K的正整数
setprofile(func)                 设置之后每个线程启动之前都会把func函数传递给sys.setprofile()
settrace(func)                   设置之后每个线程启动之前都会把func函数传递给sys.settrace()
TIMEOUT_MAX                      线程同步获取锁时的最大允许等待时间
Thread                           线程类,用于创建和管理线程
Event                            事件类,用于线程同步
Condition                        条件类,用于线程同步
Lock,RLock                       锁类,用于线程同步
Semaphore,BoundedSemaphore       信号量类,用于现场同步
Timer                            用于在指定时间之后调用一个函数
"""

# 线程对象
'''标准库中的threading的Thread类用来创建和管理线程对象,支持使用两种法来创建
    1. 直接使用Thread类实例化一个线程对象并传递一个可调用参数作为对象
    2. 继承Thread类并在派生类中重写__init__()和run()方法
   创建了线程对象以后,可以调用其start()方法来启动,该方法自动调用该类对象,此时该线程处于alive状态,直至线程的run()方法
   运行结束.'''

# Thread对象的主要成员
"""
start()             自动调用run()方法,启动线程,执行线程代码,每个线程只能启动一次
run()               线程代码,用来实现线程的功能与业务逻辑,可以在子类中重写该方法来自定义线程的行为
__init__(self,group=None,target=None,name=None,args=(),kwarg=None,verbose=None)
                    构造方法
name                读取或设置线程的名字
ident               线程标识,非0数字或None(线程未被启动)
is_alive(),isAlive()    测试线程是否处于alive状态
daemon              布尔值,表示线程是否为守护线程
join(timeout=None)  等待线程结束或超时返回
"""

'''另外,threading还支持Timer类来创建定时启动的线程,调用线程的start()方法之后,线程会在指定的时间(单位是秒)
之后再调用线程函数.'''
import threading

def demo(v):
    print(v)

t = threading.Timer(3, demo, args=(5,))  # 创建线程
t.start()  # 启动线程,3s之后调用demo函数
t.cancel()  # 如果仍在等待时间到达,则取消

# 1.join([timeout])
'''阻塞当前线程,等待被调线程结束或超时后再执行当前线程的后续代码,参数timeout用来指定最长等待时间,
单位是秒.方法join()返回后被调用isAlive()方法,如果得到True则说明线程仍在运行并且join()方法,是因为超时
而返回的,如果返回False,则说明join方法()是因为线程运行结束而返回的.一个线程可以调用多次join()方法(如果
线程已经结束,join()会立即返回),但不允许对当前线程调用join(),否则会抛出异常'''

from threading import Thread
import time

# def func1(x, y):
#     for i in range(x, y):
#         print(i, end=' ')
#     print()
#     time.sleep(10)  # 等待10s
#
#
# t1 = Thread(target=func1, args=(15, 20))  # 创建线程对象,args时传递给函数的参数
# t1.start()  # 启动线程
# t1.join(5)  # 等待线程t1运行结束或等待5s
# t2 = Thread(target=func1, args=(5, 10))
# t2.start()
# print('\n')

"""
首先输出15,19这5个整数,然后程序暂停5s以后继续输出5~9这5个整数,如果将join(5)注释掉,两个线程的输出结果会重叠在一起,
这是因为两个线程冰法运行,而不是等待一个结束以后再运行第二个.如果把time.sleep(10)这一行注释掉再运行,会发现两个程序的输出之间
没有时间间隔,这是因为线程对象的join()方法当线程运行结束或超时之后返回,虽然指定了超时时间为5s,而实际上线程函数瞬间就执行结束了
"""

# is_alive()
'''用来测试线程是否处于运行状态,如果仍在运行则返回True,如果尚未启动或运行则返回False'''

# def func1():
#     time.sleep(10)
#
#
# t1 = Thread(target=func1, args=())
# print('t1:', t1.is_alive())  # 线程还未运行,返回False
# t1.start()
# print('t1:', t1.is_alive())  # 线程还在运行,返回True
# t1.join(5)  # join()方法因超时而结束
# print('t1:', t1.is_alive())  # 线程还在运行,返回True
# t1.join()  # 等待线程结束
# print('t1:', t1.is_alive())  # 线程已结束,返回False

# daemon属性
'''在多线程编程中,如果子线程需要访问主线程中的某个资源(比如某个变量),当退出程序时主线程结束后这些资源将不再存在,子线程
继续运行时会因为无法访问资源而引发异常导致崩溃.需要一种机制保证主线程结束时可以同时结束子线程,或者使主线程等待子线程运行
结束后再结束,线程的deamon属性能够满足该需求
    在程序运行过程中有一个主线程,若在主线程中穿件了子线程,当当主线程结束时根据子线程daemon属性值不同会发生一下两种情况之一
        1.如果某个子线程的daemon属性为False,主线程结束时会检测该子线程是否结束,如果该子线程还在运行,则主线程会等待到它完成
        之后再退出
        2.如果某个子线程的daemon属性为True,主线程结束时不会对这个子线程进行检查而直接退出,同时所有ddaemon值为True的子线程
        将随主线程一起结束,无论是否运行完成
    属性deamon默认值为False,如果需要修改,必须在调用start()方法启动线程之前设置.另外,上述并不适用于IDLE环境中的交互模式或脚本'
    运行模式,因为在该环境中的主线程只有在退出python IDLE时才终止.'''

# class MyThread(threading.Thread):  # 继承Thread类,创建自定义线程类
#     def __init__(self, num, threadname):
#         # threading.Thread.__init__(self, name=threadname)
#         super(MyThread, self).__init__(name=threadname)
#         self.num = num
#
#     def run(self) -> None:
#         time.sleep(self.num)
#         print(self.num)
#
#
# t1 = MyThread(1, 't1')  # 创建自定义线程类对象,daemon默认值为False
#
# t2 = MyThread(5, 't2')
# t2.daemon = True  # 设置线程对象的daemon属性为True
# print(t1.daemon)
# print(t2.daemon)
# t1.start()  # 启动线程
# t2.start()

# Lock/RLock对象

'''Lock是比较低级的同步原语,当被锁定以后不属于特定的线程.一个锁有两种状态:locked和unlocked,刚创建的
Lock对象处于unlocked状态.如果锁处于unlocked状态,acquire()方法将其修改为locked并立即返回;如果锁已处于
locked状态,则阻塞当前线程并等待其他线程释放锁,然后将其修改为locked并立即返回.release()方法用来将锁的
状态由locked修改为unlocked并立即返回.如果锁状态本身已经是unlocked,调用该方法将会抛出异常
    可重入锁RLock对象也是一种常用的线程同步原语,可被同一个线程acquire()多次,.当处于locked状态时,某线程
拥有该锁;当处于unlocked状态时,该锁不属于任何线程.RLock对象的acquire()/release()调用对可以嵌套,仅当最后一个
或者最外层release()执行结束后,锁才会被设置为unlocked状态'''

# 自定义线程类
# class MyThread(threading.Thread):
#     def __init__(self):
#         # threading.Thread.__init__(self)
#         super(MyThread, self).__init__()
#
#     #  重写run()方法
#     def run(self) -> None:
#         global x
#         # 获取锁,如果成功则进入临界区
#         lock.acquire()
#         x = x + 3
#         print(x)
#         # 退出临界区,释放锁
#         lock.release()
#
#
# lock = threading.RLock()
# # 也可以使用Lock类实现加锁和线程同步
# # lock = threading.Lock()
#
# # 存放多个线程的列表
# t1 = []
# for i in range(10):
# #     创建线程并添加列表
#     t = MyThread()
#     t1.append(t)
#
# # 多个线程互斥访问的变量
# x = 0
# # 启动列表中的所有线程
# for i in t1:
#     i.start()

"""需要注意的是,多线程同步时如果需要获得多个锁才能进入临界区的话,可能会发生死锁,在多线程编程时一定要注意
并认真检查和避免这种情况"""

# 以下代码就可能发生死锁,类似于"哲学家就餐问题"

class MyThread1(threading.Thread):
    def __init__(self):
        super(MyThread1, self).__init__()

    def run(self) -> None:
        lock1.acquire()  # 获取一个锁
        lock2.acquire()  # 获取另一个锁
        # 实际功能代码,(略)
        lock2.release()
        lock1.release()

class MyThread2(threading.Thread):
    def __init__(self):
        super(MyThread2, self).__init__()

    def run(self) -> None:
        lock1.acquire()  # 获取一个锁
        lock2.acquire()  # 获取另一个锁
        # 实际功能代码,(略)
        lock2.release()
        lock1.release()

lock1 = threading.RLock()
lock2 = threading.RLock()
t1 = MyThread1()
t2 = MyThread2()
t1.start()
t2.start()

# %%
# condition对象
'''使用condition对象可以在某些事件触发后才处理数据或执行特定的功能代码,可以用于不同线程之间的通信或通知,已实现更高级的
同步.condition对象除了具有acquire和release方法之外,还有wait,wait_for,notify,notify_all等方法'''
''' wait(timeout=None)方法释放锁,并阻塞当前线程知道超时或其他线程针对同一个condition对象调用notify/notify_all方法.
被唤醒的线程会重新尝试取锁并在成功获取锁之后结束wait方法,然后继续执行
    wait_for(predicate,timout=None)方法阻塞,当前线程直到超时或条件得到满足
    notify(n=1)唤醒或等待该condition对象的一个或多个线程,该方法不负责释放锁
    notify_all()方法会唤醒等待该condition的所有线程
'''

"""通过生产者-消费者问题演示condition的用法,程序中生产者产生线程和消费者线程共享一个列表,生产者在列表尾部添加元素,
消费者从列表收不获取并删除元素,如果列表长度达到了20表示已满,生产者等待,如果列表已空则消费者等待"""

from random import randint
from time import sleep

# 自定义生产者线程类
class Producer(threading.Thread):
    def __init__(self, threadName):
        super(Producer, self).__init__(name=threadName)

    def run(self) -> None:
        global x
        while True:
            # 获取锁
            con.acquire()
            # 假设共享列表最多能容纳20个晕啊素
            if len(x) == 20:
                # 表示列表已满,生产者等待
                con.wait()
                print('Producer is waiting')
            else:
                print('Producer:', end='')
                # 产生新元素,添加至共享列表
                x.append(randint(1, 1000))
                print(x)
                sleep(1)
                # 唤醒等待条件的线程
                con.notify()
            # 释放锁
            con.release()

class Consumer(threading.Thread):
    def __init__(self, threadName):
        super(Consumer, self).__init__(name=threadName)

    def run(self) -> None:
        global x
        while True:
            con.acquire()
            if not x:
                # 等待
                con.wait()
                print('Consumer is waiting')
            else:
                print(x.pop(0))
                print(x)
                sleep(2)
                con.notify()
            con.release()

# 创造线程condition对象以及生产者,消费者线程
con = threading.Condition()
c = Consumer('Consumer')
x = []
p = Producer('Producer')
p.start()
c.start()
p.join()
c.join()

# %%
# Queue对象
'''queue模块提供的Queue类实现多线程编程所需要的锁原语,是线程安全的,不需要额外的同步机制,尤其适合需要在多个
线程之间信息交换的场合.Queue类对象的get()和put()方法都支持一个超时参数timeout,调用该方法时如果超时参数会
报出异常'''

# 使用Queue类实现多线程同步,模拟生产者/消费者问题
import queue
import threading
import time

class Producer(threading.Thread):
    def __init__(self, threadName):
        threading.Thread.__init__(self, name=threadName)

    def run(self) -> None:
        global myqueue
        # 在队列尾部追加元素
        time.sleep(1)
        try:
            myqueue.put(self.getName(), timeout=1)
            print(self.getName(), ' put', self.getName(), ' to queue')
        except:
            pass

class Consumer(threading.Thread):
    def __init__(self, threadName):
        super(Consumer, self).__init__(name=threadName)

    def run(self) -> None:
        global myqueue

        # 在队列首部获取元素
        time.sleep(0.1)
        try:
            print(self.getName(), ' get', myqueue.put(timeout=1.1), ' from queue')
        except:
            pass

myqueue = queue.Queue(5)
plist = []
clist = []
for i in range(10):
    p = Producer('Producer' + str(i))
    plist.append(p)
    c = Consumer('Consumer' + str(i))
    clist.append(c)

# 依次启动生产者和消费者线程
for p, c in zip(plist, clist):
    p.start()
    c.start()

# %%
# Event对象
"""Event对象是一种简单的线程通信技术,一个线程设置event对象,另一个线程等待Event对象.Event对象的set()方法可以设置Event对象内部的
信号标志为真;clear()方法可以清除Event对象内部的信号标志,将其设置为假;isSet()方法用来判断其内部信号标志的状态;wait()方法在其内部
信号状态为真时会立刻执行并返回,若Event对象的信号标志为假,wait()方法就一直等待至超时或者内部信号状态为真."""
import threading

class MyThread(threading.Thread):
    def __init__(self, threadName):
        super(MyThread, self).__init__(name=threadName)

    def run(self) -> None:
        global myevent
        # 根据Event对象是否已设置做出不同的响应
        if myevent.isSet():
            # 清除标志
            myevent.clear()
            # 等待
            myevent.wait()
            print(self.getName() + ' set')
        else:
            print(self.getName() + ' not set')
            # 设置标志
            myevent.set()

myevent = threading.Event()
# 设置标志
for i in range(10):
    t = MyThread(str(i))
    t.start()

# %%
# Semaphore与BoundedSemaphore
'''
Semaphore对象维护着一个内部计数器,调用acquire()方法时该计数器-1,调用release()方法时+1,适用于需要控制特定资源的并发访问线程数量
的场合,调用acquire()方法时,如果计数器已经为0则阻塞当前线程直到有其他线程调用了release()方法,所以计数器的值永远不会小于0.
Semaphore对象可以调用任意次release()方法(如果真的出现这种情况,很可能是有bug),而BoundedSemaphore对象可以保证计数器的值不会超过特定
的值,与Lock/RLock,Condition对象一样,Semaphore和BoundedSemaphore对象也支持上下文管理协议,支持with关键字
'''

# 使用BoundedSemaphore对象限制特定资源的并发访问线程数量
import time

def worker(value):
    with sema:
        print(value)
        time.sleep(6)

# 同一时刻最多允许2个线程访问特定资源
sema = threading.BoundedSemaphore(2)
for i in range(10):
    t = threading.Thread(target=worker, args=(i,))
    t.start()

# %%
# Barrier对象
'''Barrier对象常用来实现这样的线程同步,多个线程运行到某个时间点以后每个线程都需要等着其他线程都准备好以后再同时进行下一步工作.
类似赛马需要先用栅栏拦住,每个试图穿过栅栏的选手都需要说明自己准备好了,当所有选手都表示准备好以后,栅栏打开,所有选手同时冲出栅栏'''

import random
import threading
import time

def worker(arg):
    # 假设每个线程需要不同的时间来完成准备工作
    time.sleep(random.randint(1, 20))
    # 假设已知任何线程的准备工作最多需要20s
    # 每个线程调用wait(),返回值不一样
    r = b.wait(20)
    if r == 0:
        print(arg)

def printok():
    print('ok')

# 允许3个线程等待
# 如果调用wait()时没有指定超时时间,默认为20s
b = threading.Barrier(parties=3, action=printok, timeout=20)

# 创建并启动3个线程,线程数量必须与Barrier对象的parties一致
for i in range(3):
    t = threading.Thread(target=worker, args=(i,))
    t.start()

# 下面的代码模拟一个类似的场景,服务器启动时需要一定的时间,在服务器做好准备工作之前不允许客户端发起连接请求
# b = threading.Barrier(parties=2,timeout=5)
# def server():
#     # 启动服务器,准备接收客户端的连接,代码略
#     b.wait()
#     while True:
#         # 接收客户端连接,处理客户端请求,代码略
#
# def client():
#     # 等待服务器启动
#     b.wait()
#     while True:
#         # 建立连接,和服务器进行通信
#
# # 分别建立并启动服务器线程和客户端线程
# threading.Thread(target=server).start()
# threading.Thread(target=client).start()


import os
import sys
import time
import logging
logging.getLogger().setLevel(logging.INFO)

import torch
import torch.nn as nn
import torch.optim as optim
import torch.utils.data as DATA

# from tensorboardX import SummaryWriter

MY_DIRNAME = os.path.dirname(os.path.abspath(__file__)) # 获取本文件所处的绝对路径，解决相对路径的问题
sys.path.insert(0, os.path.join(MY_DIRNAME, '..'))

from nets.model_main import ModelMain
from nets.YOLO_loss import YOLOLoss
from common.coco_dataset import COCODataset

''' 配置表 '''
anchors = [[[116, 90], [156, 198], [373, 326]],
           [[30, 61], [62, 45], [59, 119]],
           [[10, 13], [16, 30], [33, 23]]]
classes = 80

batch_size = 10
train_path = "../data/coco/trainvalno5k.txt"
epochs = 100
img_h, img_w = 416, 416
lr_freeze_backbone = False
other_lr = 0.01
backbone_lr = 0.001
optimizer_type = "sgd"
optimizer_weight_decay = 4e-05
lr_decay_step = 20
lr_decay_gamma = 0.1
pretrain_snapshot = ""#"F:/DeepLearningPytorch/M9_YOLOv3_Full_Reproduce/checkpoint/run0/model@0_9000.pth"
evaluate_type = ""
export_onnx = False
# checkpoint_path = "F:/DeepLearningPytorch/M9_YOLOv3_Full_Reproduce/checkpoint/model.pth"
''' 配置表 '''

def train():
    global_step = 0
    is_training = True

    # Load and Initialize Network
    net = ModelMain(is_training)
    net.train(is_training)

    # Optimizer and Lr
    optimizer = _get_optimizer(net)
    lr_scheduler = optim.lr_scheduler.StepLR(
        optimizer,
        step_size=lr_decay_step, #20
        gamma=lr_decay_gamma) # 0.1

    # Set Data Paraller:
    net = nn.DataParallel(net)
    net = net.cuda()
    logging.info("Net of Cuda is Done!")

    # Restore pretrain model 从预训练模型中恢复
    if pretrain_snapshot:
        logging.info("Load pretrained weights from {}".format(pretrain_snapshot))
        state_dic = torch.load(pretrain_snapshot)
        net.load_state_dict(state_dic)

    yolo_losses = []
    for i in range(3):
        yolo_losses.append(YOLOLoss(anchors[i], classes, (img_w, img_h)).cuda())
    print('YOLO_Losses: \n', yolo_losses)

    # DataLoader
    train_data_loader = DATA.DataLoader(dataset=COCODataset(train_path, (img_w, img_h), is_training=True),
                                        batch_size=batch_size,shuffle=True, pin_memory=False)
    # Start the training loop
    logging.info("Start training......")
    for epoch in range(epochs):
        for step, samples in enumerate(train_data_loader):
            images, labels = samples['image'].cuda(), samples["label"].cuda()
            start_time = time.time()
            global_step += 1

            # Forward & Backward
            optimizer.zero_grad()
            outputs = net(images)
            losses_name = ["total_loss", "x","y", "w", "h", "conf", "cls"]
            losses = [[]] * len(losses_name) # [[]] ---> [[], [], [], [], [], [], []]
            for i in range(3): # YOLO 3 scales
                _loss_item = yolo_losses[i](outputs[i], labels)
                for j, l in enumerate(_loss_item):
                    # print('j: ', j, 'l: ', l) j: index(0-6); l内容: 总loss, x, y, w, h, conf, cls
                    losses[j].append(l)
            losses = [sum(l) for l in losses]
            loss = losses[0] # losses[0]为总Loss
            conf = losses[5]
            loss.backward()
            optimizer.step()

            if step>0 and step % 10 == 0:
                _loss = loss.item()
                _conf = conf.item()
                duration = float(time.time()-start_time) # 总用时
                example_per_second = batch_size / duration # 每个样本用时
                lr = optimizer.param_groups[0]['lr']
                logging.info(
                    "epoch [%.3d] iter = %d loss = %.2f conf = %.2f example/sec = %.3f lr = %.5f " %
                    (epoch, step, _loss, _conf, example_per_second, lr)
                )
            if step >= 0 and step % 1000 == 0:
                # net.train(False)
                _save_checkpoint(net.state_dict(), epoch, step)
                # net.train(True)

        lr_scheduler.step()

    _save_checkpoint(net.state_dict(), 100, 9999)
    logging.info("Bye~")

def _save_checkpoint(state_dict, epoch, step, evaluate_func = None):
    checkpoint_path = '%s%s%s%s%s' % ('F:/DeepLearningPytorch/M9_YOLOv3_Full_Reproduce/checkpoint/model@', str(epoch),'_', str(step), '.pth')
    torch.save(state_dict, checkpoint_path)
    logging.info("Model checkpoint saved to %s" % checkpoint_path)

def _get_optimizer(net):
    logging.info("Get optimizer: ")
    optimizer = None

    # Assign different lr for each layer
    params =None
    base_params = list(map(id, net.backbone.parameters()))
    logits_params = filter(lambda p: id(p) not  in base_params, net.parameters())

    if not lr_freeze_backbone: # Default: False
        params = [
            {"params": logits_params, "lr": other_lr}, # 0.01
            {"params": net.backbone.parameters(), "lr": backbone_lr}, # 0.001
        ]
    else:
        logging.info("freeze backbone's parameters.")
        for p in net.backbone.parameters():
            p.requires_grad = False
        params = [
            {"params": logits_params, "lr": other_lr},
        ]
    # Initialize optimizer class
    if optimizer_type == "adam":
        optimizer = optim.Adam(params, weight_decay=optimizer_weight_decay)
    elif optimizer_type == "amsgrad":
        optimizer = optim.Adam(params, weight_decay=optimizer_weight_decay, amsgrad=True)
    elif optimizer_type == "rmsprop":
        optimizer = optim.RMSprop(params, weight_decay=optimizer_weight_decay)
    else:
        logging.info("Using SGD Optimizer.")
        optimizer = optim.SGD(params, momentum=0.9, weight_decay=optimizer_weight_decay,
                              nesterov=False)
    return optimizer

if __name__ == "__main__":
    train()


"""
Shop Calculator
The program allows the user to enter the number of items and the price of each different item.
Then the program computes and displays the total price of those items.
if the total price is over $100, then a 10% discount is applied to that total
"""
ItemList: list[float] = []
price_output: float = 0
total = 0

item_num = int(input("Number of items:"))
while item_num < 0:
    print("Invalid number of items!")
    item_num = int(input("Number of items:"))

for i in range(item_num):
    price = float(input("Price of items:"))
    ItemList.append(price)
total = sum(ItemList)

if total < 0:
    print("Invalid price of items!")
elif total > 100:
    price_output = total * 0.9
else:
    price_output = total

print(f"Total price for 3 items is ${price_output:.2f}")
# print(ItemList)
# print(price_output)
# print(total)


from django import forms
from django.contrib.auth.forms import UserCreationForm
from django.contrib.auth.models import User
from .models import Userprofile

# form for user profile
class UserprofileForm(forms.ModelForm):
    def __init__(self, *args, **kwargs):
        super(UserprofileForm, self).__init__(*args, **kwargs)

        self.fields['address'].widget.attrs['class'] = 'input'
        self.fields['postcode'].widget.attrs['class'] = 'input'
        self.fields['city'].widget.attrs['class'] = 'input'
        self.fields['phone'].widget.attrs['class'] = 'input'

    class Meta:
        model = Userprofile
        fields = '__all__'
        exclude = ('user',)

# user sign up form
class SignUpForm(UserCreationForm):
    first_name = forms.CharField(max_length=50, required=True)
    last_name = forms.CharField(max_length=50, required=True)
    email = forms.EmailField(max_length=255, required=True)

    # function to improve signup form asethetic
    def __init__(self, *args, **kwargs):
        super(SignUpForm, self).__init__(*args, **kwargs)

        self.fields['username'].widget.attrs['class'] = 'input'
        self.fields['email'].widget.attrs['class'] = 'input'
        self.fields['password1'].widget.attrs['class'] = 'input'
        self.fields['password2'].widget.attrs['class'] = 'input'
        self.fields['first_name'].widget.attrs['class'] = 'input'
        self.fields['last_name'].widget.attrs['class'] = 'input'

    # meta data for signup form
    class Meta:
        model = User
        fields = ['username', 'first_name', 'last_name', 'email', 'password1', 'password2']

# creates an edit account form using  a model
class EditAccountForm(forms.ModelForm):

    # form metadata options
    class Meta:
        # using Post model
        model = Userprofile

        # fields that will be used for the form
        fields = ('address', 'postcode', 'city', 'phone')

        # basic controls/styling for the form fields
        widgets = {
            'address': forms.TextInput(attrs={'class': 'form-control',
                                            'placeholder':
                                            'Enter Your Full Address'}),
            'postcode': forms.Textarea(attrs={'class': 'form-control',
                                          'placeholder':
                                          'Enter your Postcode'}),
            'city': forms.Textarea(attrs={'class': 'form-control',
                                             'placeholder':
                                             'Enter Your City'}),
            'phone': forms.Textarea(attrs={'class': 'form-control',
                                             'placeholder':
                                             'Enter Your Phone Number'}),
        }

# creates an delete number form using  a model
class DeleteForm(forms.ModelForm):

    # form metadata options
    class Meta:
        # using Post model
        model = Userprofile

        # fields that will be used for the form
        fields = ('address', 'postcode', 'city',
                  'phone')

        # basic controls/styling for the form fields
        widgets = {
            'address': forms.TextInput(attrs={'class': 'form-control',
                                            'placeholder':
                                            'Enter Your Full Address'}),
            'postcode': forms.Textarea(attrs={'class': 'form-control',
                                          'placeholder':
                                          'Enter your Postcode'}),
            'city': forms.Textarea(attrs={'class': 'form-control',
                                             'placeholder':
                                             'Enter Your City'}),
            'phone': forms.Textarea(attrs={'class': 'form-control',
                                             'placeholder':
                                             'Enter Your Phone Number'}),
        }


import os
from components.page import Page
from components.type import Type
import services.file_manager as fm

DIRECTORY = fm.DIRECTORY["meta"]

def createType(args):
    t = Type()
    t.name = args[0]
    t.state = 1
    t.numOfFields = int(args[1])
    t.fieldNames = args[2:] + [""] * (10 - int(t.numOfFields))

    if not fm.isAvailableFileExist("meta"):
        fm.createFile("meta")

    fileList = fm.getFiles("meta")

    for fileName in fileList:
        f = open(DIRECTORY + fileName, "r+b")
        for i in range(0, fm.MAX_NUM_PAGES):
            p = Page(f, i, "meta")
            for j in range(0, Page.MAX_NUM_OF_ELEMENTS["meta"]):
                if p.elements[j].state == 0:
                    p.elements[j] = t
                    p.numOfElements += 1
                    if p.numOfElements == Page.MAX_NUM_OF_ELEMENTS["meta"]:
                        p.state = 1
                    p.write(f, i)
                    f.close()
                    return
                elif t.name < p.elements[j].name:
                    t, p.elements[j] = p.elements[j], t
            p.write(f, i)
        f.close()

def deleteType(args):
    fileList = fm.getFiles("meta")
    typeToBeDeleted = args[0]
    t = Type()
    updateHeaderFlag = True
    for fileName in reversed(fileList):
        f = open(DIRECTORY + fileName, "r+b")
        for i in reversed(range(0, fm.MAX_NUM_PAGES)):
            p = Page(f, i, "meta")
            if updateHeaderFlag == True and p.numOfElements > 0:
                updateHeaderFlag = False
                p.numOfElements -= 1
                p.state = 0
            for j in reversed(range(0, Page.MAX_NUM_OF_ELEMENTS["meta"])):
                if p.elements[j].state != 0:
                    t, p.elements[j] = p.elements[j], t
                    if t.name == typeToBeDeleted:
                        p.write(f, i)
                        f.close()
                        fm.deleteAllFiles(typeToBeDeleted)
                        if fm.isEmpty("meta", fileList[-1]):
                            fm.deleteFile("meta", fileList[-1])
                        return
            p.write(f, i)
        f.close()

def listType(args):
    fileList = fm.getFiles("meta")
    for fileName in fileList:
        f = open(DIRECTORY + fileName, "r+b")
        for i in range(0, fm.MAX_NUM_PAGES):
            p = Page(f, i, "meta")
            for j in range(0, Page.MAX_NUM_OF_ELEMENTS["meta"]):
                if p.elements[j].state != 0:
                    print(p.elements[j].name)
        f.close()

CMPE 321  Introduction to Database Systems/project 2 - Database Management System/src/operations/ddl.py


import scrapy
import sqlite3
import pandas as pd
from tqdm import tqdm
from scrapy.loader import ItemLoader
from orv2.items import RestItem, RevItem, RestLoader, RevLoader
from orv2.spiders import reg
from orv2.spiders.utils import info_handler

class orv2(scrapy.Spider):
    name = 'orv2'
    allowed_domains = ['openrice.com']

    def __init__(self):
        '''
        Initilization
        '''
        self.url = 'https://www.openrice.com/en/hongkong/r-openrice-r{:s}/reviews'

    def start_requests(self):
        '''
        Run the crawling.
        '''
        ## Iteration
        conn = sqlite3.connect("Meta/ORID.sqlite")
        while True:
            restID = pd.read_sql("SELECT ID FROM rid WHERE state=1 LIMIT 1", conn)["ID"]
            if restID.empty:
                conn.close()
            else:
                print("ID: {:d} |".format(restID[0]), end=".")
                yield scrapy.Request(self.url.format(str(restID[0])))
                conn.execute("UPDATE rid SET State=0 WHERE ID={:d}".format(restID[0]))
                conn.commit()

    def parse(self, response):
        review_list = response.xpath("//div[@itemprop='review']")
        for review in review_list:
            yield self._parse_rev(review)

        if response.xpath("//div[@class='or-sprite common_pagination_more_r_desktop']"):
            next_url = response.xpath("//div[@class='or-sprite common_pagination_more_r_desktop']/../@href").get()
            yield scrapy.Request('https://www.openrice.com/' + next_url)
        else:
            yield self._parse_rest(response)

    def _parse_rest(self, response):
        loader = RestLoader(item = RestItem(), response=response)

        loader.add_xpath('ID', "//div[@itemprop='itemReviewed']/div/div/@data-poi-id")
        loader.add_xpath('TT', "//ul[@class='breadcrumb']/li[5]/a/span/text()")
        loader.add_xpath('NM', "//div[@class='poi-name']/h1/span/text()")
        loader.add_xpath('RT', "//div[@class='header-score']/text()")
        loader.add_xpath('BM', "//div[@class='header-bookmark-count js-header-bookmark-count']/@data-count")
        loader.add_xpath('DT', "//ul[@class='breadcrumb']/li[4]/a/span/text()")
        loader.add_xpath('PR', "//div[@itemprop='priceRange']/a/@href", re=reg.PR_REG)
        loader.add_xpath('TP', "//div[@class='header-poi-categories dot-separator']/a/@href", re=reg.TP_REG)
        loader.add_xpath('RC', "//span[@itemprop='reviewCount']/text()")
        loader.add_xpath('OS', "//div[@class='score-div'][1]/text()")
        loader.add_xpath('OO', "//div[@class='score-div'][2]/text()")
        loader.add_xpath('OC', "//div[@class='score-div'][3]/text()")

        for aspect in response.xpath("//div[@class='header-score-details-right-item']"):
            loader.add_value('A' + aspect.xpath("div[@class='header-score-details-right-item-title']/text()").re(reg.AS_REG)[0],
                                aspect.xpath("div[2]/@class").re(reg.ASR_REG))
        return loader.load_item()

    def _parse_rev(self, response):
        loader = RevLoader(item = RevItem(), selector=response)

        loader.add_xpath('ID', "@data-review-id")
        loader.add_xpath('RI', "@data-poi-id")
        loader.add_xpath('UI', "div//a[@itemprop='author']", re=(reg.UI_REG))
        loader.add_xpath('DA', "div//span[@itemprop='datepublished']/text()")
        loader.add_xpath('OR', "div//div[@class='left-header']/div/@class", re=reg.OR_REG)
        loader.add_xpath('VC', "div//span[@class='view-count']/text()", re=reg.VC_REG)
        loader.add_xpath('TT', "div//div[@class='review-title']/a/text()")
        loader.add_xpath('BD', "div//section[@class='review-container']")
        loader.add_xpath('TE', "div//div[@class='review-title']/a/text()", re=reg.EMOJI_REG)
        loader.add_xpath('CE', "div//section[@class='review-container']/div[@class[contains(., 'write-re-icon')]]/@class", re=reg.CE_REG)
        loader.add_xpath('BE', "div//section[@class='review-container']/text()", re=reg.EMOJI_REG)
        loader.add_xpath('UR', "div//a[@data-is-photo='true']/@data-shorten-url", re=reg.UR_REG)

        for aspect in response.xpath("div//section[@class='sr2-review-list2-detailed-rating-section detail']/div[@class='subject']"):
            loader.add_value('A' + aspect.xpath("div[1]/text()").re(reg.AS_REG)[0], aspect.xpath("div[2]/span/@class").re(reg.STAR_REG).count('y'))

        for (attr, value) in info_handler(response.xpath("div//section[@class='info-section detail']/section[@class='info info-row']")):
            loader.add_value(attr, value)

        return loader.load_item()


#!/usr/bin/env python3

import pandas as pd
import numpy as np
import os
import argparse
from log_parser.DFParser import *

def main(file_list):
    gps_meta = []

    for filename in file_list:
        log = DFLog(filename)

        # Plan - get first three messages
        # Get max value of GPS1 delta
        firmware = log.tables["MSG"]["Message"].iloc[0]
        os = log.tables["MSG"]["Message"].iloc[1]
        hardware = log.tables["MSG"]["Message"].iloc[2]

        max_delay = log.tables['GPA']['Delta'].max()

        gps_meta.append([firmware, os, hardware, max_delay])

    gps_meta_df = pd.DataFrame(data=gps_meta, columns=["firmware", "os", "hardware", "max_delay"])
    return gps_meta_df

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("output", help="The new log file to output into")
    parser.add_argument("input_folder", help="The folder containing log files")
    args = parser.parse_args()
    folder = args.input_folder
    file_list = [os.path.join(folder, f) for f in os.listdir(folder) if f.endswith(".bin")]
    df = main(file_list)
    df.to_csv(args.output, index=False)


# -*- coding: utf-8 -*-
"""
Created on 19:13 2022/9/24

@author: shengrihui
"""
import matplotlib.pyplot as plt
import numpy as np

x = np.linspace(-10, 10, 1000)
# print(x)
y = 1 / (1 + np.exp(-x))
g1 = np.exp(-x) / (np.exp(-x) + 1) ** 2
g2 = y * (1 - y)
plt.plot(x, g1)
# plt.plot(x, y)
plt.show()
plt.plot(x, g2)
plt.show()


import numpy as np
import pylab as pl
import matplotlib.pyplot as mpl
import pandas as pd
import glob
import sys
import os
filenames = glob.glob('C:/cygwin64/home\siu851591172/mos2_defect\*.dat')
mpl.rcParams['font.family'] = 'serif'
mpl.rcParams['mathtext.fontset'] = 'dejavuserif'
for f in filenames:

    save_name = os.path.split(f)[-1]
    save_name = save_name.split('.')[0].split('_')
    delim = ' '
    save_delim = '_'
    final_save = save_delim.join(save_name)
    title = delim.join(save_name).title()
    if 'H3' in title:
        y_max = 10.5
    else:
        y_max = 10
    print(y_max)
    df = pd.read_csv(f,header=None,sep='\s+')
    charge_den = df.iloc[:,1]
    position = df.iloc[:,0]
    position = position/1.88973
    plot = mpl.plot(charge_den,position,c='r',linewidth=2)
    mpl.tight_layout(pad=4.5)
    mpl.ticklabel_format(style='sci',axis='x',scilimits=(0,0),useMathText=True)
    #mpl.title(label=title,fontsize='xx-large')
    left_lim = -1*max(charge_den)
    right_lim = max(charge_den)
    mpl.xlim(left_lim,right_lim)
    mpl.ylim(3.5,y_max)
    mpl.grid(b=True,dashes=(5,1),alpha=0.7)
    mpl.tick_params(axis='both',labelsize='large',direction='in',grid_linewidth=0.5)
    mpl.xticks([left_lim, left_lim/2,0,right_lim/2,right_lim])
    mpl.xlabel(r'$\rho$ (e/$\AA$)',fontsize='xx-large',fontname='serif')
    mpl.ylabel(r'Z ($\AA$)',fontsize='xx-large')
    mpl.axvline(0,linewidth=.5,color='black',dashes=(7,3))
    mpl.savefig(final_save,optimize=True)
    mpl.show()
    mpl.clf()