#mass import
import pandas as pd
import urllib.request
import json
import numpy as np
import matplotlib.pyplot as plt
from datetime import datetime, timedelta
from sklearn.cluster import KMeans
pd.options.display.max_rows = 999

#for getting the date 5 days ago
daydif = str(datetime.today() - timedelta(days=5))
dayref = str(daydif[0:10])
today = str(datetime.today().strftime('%Y-%m-%d'))

#for retrieving the data from the APIs
link = ["https://environment.data.gov.uk/flood-monitoring/id/stations/46160/readings?since=",dayref, "&_sorted&parameter=rainfall"]
final = ""
final = final.join(link)
#print(final)
web = urllib.request.Request(final)
response = urllib.request.urlopen(web)
the_page = response.read()
jason = json.loads(the_page)
link1 = ["http://environment.data.gov.uk/flood-monitoring/id/measures/46126-level-stage-i-15_min-m/readings?since=",dayref]
final1 = ""
final1 = final1.join(link1)
#print(final1)
web1 = urllib.request.Request(final1)
response1 = urllib.request.urlopen(web1)
the_page1 = response1.read()
jason1 = json.loads(the_page1)

#creates dataframes for each API
df1 = pd.DataFrame(jason1["items"])
df1 = df1.sort_values('dateTime', ascending = True)
df = pd.DataFrame(jason["items"])
df = df.sort_values('dateTime', ascending = True)

#merged table containing level and fall, plots a graph
a = pd.merge(df, df1, on = 'dateTime', how = 'left')
b = a[['dateTime', 'value_x', 'value_y']].copy()
b = b.rename(columns = {"dateTime" : "Date/Time", "value_x" : "Rainfall", "value_y" : "River Level"})

#Calculates hourly results
c = b[['Rainfall', 'River Level']]
d = c['River Level'].groupby(c.index//4).mean()
d = d.diff()
e = c.groupby(c.index//4)['Rainfall'].sum()
hourly = pd.concat([d, e], axis = 1)
drip = hourly['Rainfall'].max()
drip = int(drip * 10)
calc = []
tester = pd.DataFrame()
for i in range (0, drip+1):
  x = i/10
  s = hourly.Rainfall.eq(x)
  out = pd.DataFrame()
  out['River'] = hourly.loc[s.shift(1, axis = 0) | s.shift(2, axis = 0), 'River Level']
  runner = len(out)
  out['Rain'] = x
  tester = pd.concat([tester, out])
tester = tester.dropna()

#Machine learning : Kmeans clustering
reg = KMeans(n_clusters = 2, random_state = 0).fit(tester[['River']])

#Producing a graph of the regression
plt.scatter(tester['Rain'], tester['River'], color = 'Blue', marker = '+')
plt.plot(tester['Rain'], reg.predict(tester[['Rain']]), color = 'Red')
plt.show()

#Producing an example prediction
pred = pd.DataFrame()
#print(pred)
end = len(b.index)
for i in range(1, end):
  prod = pd.DataFrame()
#print(b)
for row in b.itertuples():
  temp = pd.DataFrame()
  temp['RDelta'] = reg.predict([[row.Rainfall]])
  temp['Rainfall'] = row.Rainfall
  pred = pd.concat([pred, temp])
endriver = pd.DataFrame()
riverstart = b['River Level'].iloc[0]
for row in pred.itertuples():
  rtemp = pd.DataFrame({'RPred': [riverstart]})
  endriver = pd.concat([endriver, rtemp], ignore_index = True)
  riverstart = riverstart + row.RDelta

#Producing a dataframe with the prediction and all other data together
fin = pd.merge(b, endriver, left_index = True, right_index = True)
print(fin)

#Plotting a graph of expected riverlevel and the actual river level
plt.plot(fin['Date/Time'],fin['River Level'],label = 'River Level')
plt.plot(fin['Date/Time'],fin['Rainfall'], label = 'Rainfall')
plt.plot(fin['Date/Time'],fin['RPred'], label = 'Predicted')
plt.locator_params(axis = 'Date/Time', nbins = 10)
plt.xticks(rotation = 'vertical')
ax = plt.gca()
ax.set_xticks(ax.get_xticks()[::48])
plt.show()


import textract
import re
import os
import requests
from bs4 import BeautifulSoup
import time
import random

# ===================================== get paper url =====================================
urls = [
    'https://sj.ctu.edu.vn/ql/docgia/nam-2015/loaichuyensan-2/xuatban-782.html',
    'https://sj.ctu.edu.vn/ql/docgia/nam-2017/loaichuyensan-2/xuatban-1222/chuyensan-250.html',
    'https://sj.ctu.edu.vn/ql/docgia/nam-2011/loaichuyensan-2/xuatban-182.html',
    'https://sj.ctu.edu.vn/ql/docgia/nam-2013/loaichuyensan-2/xuatban-442/chuyensan-250.html',
    'https://sj.ctu.edu.vn/ql/docgia/nam-2020/loaichuyensan-2/xuatban-2002.html'
    'https://sj.ctu.edu.vn/ql/docgia/nam-2018/loaichuyensan-2/xuatban-1402.html',
    'https://sj.ctu.edu.vn/ql/docgia/nam-2018/loaichuyensan-2/xuatban-1522.html'
]
paper_url = []
for url in urls:
    page = requests.get(url)
    data = BeautifulSoup(page.content, 'html.parser')
    elements = data.select('.div-left.chitiet.grid_05')
    for e in elements:
        paper_url.append(str(e.parent.get("href")))

# ===================================== download pdf file =====================================
def download_file(writefile, url):
    r = requests.get(url, allow_redirects=True)
    with open(writefile, 'wb') as f:
        f.write(r.content)

pdf_folder = '/content/drive/MyDrive/data/dhct'
raw_text_folder = '/content/drive/MyDrive/data/dhct_raw_txt'
clean_text_folder = '/content/drive/MyDrive/data/dhct_clean'

for i, url in enumerate(paper_url):
    sleep_time = random.randint(1, 5)
    print(f'file {i} -- sleep in {sleep_time}s')
    file = os.path.join(pdf_folder, f'paper_{i}.pdf')
    download_file(file, url)
    time.sleep(sleep_time)

# ===================================== convert pdf to raw txt =====================================
def convert_pdf_to_raw_txt(pdf_file, txt_file):
    text = textract.process(pdf_file, language='eng')
    text = text.decode('utf-8')
    with open(txt_file, 'w', encoding='utf-8') as f:
        f.write(text)

# convert pdf to raw txt file. Raw mean there are still invalid characters
for file in os.listdir(pdf_folder):
    file_name = file.split('.')[0] + '.txt'
    pdf_file = os.path.join(pdf_folder, file)
    txt_file = os.path.join(raw_text_folder, file_name)
    convert_pdf_to_raw_txt(pdf_file, txt_file)

# ===================================== clean raw data =====================================
from  util.shared import read_file, write_to_file
from .text_preprocessor import TextPreprocessor

processor = TextPreprocessor()

# clean raw text
for file in os.listdir(raw_text_folder):
    text = read_file(os.path.join(raw_text_folder, file))
    text = processor.remove_invalid_unicode(text)
    text = re.sub('Tap chi Khoa hoc Trương Đai hoc Cân Thơ', 'Tạp chí Khoa học Trường Đại học Cần Thơ', text)
    text = re.sub('Trương Đai hoc Cân Thơ', 'Trường Đại học Cần Thơ', text)
    text = re.sub('Trương Đai hoc', 'Trường Đại học', text)
    text = re.sub('Tap chı Khoa hoc Trươ ng Đai hoc Cân Thơ', 'Tạp chí Khoa học Trường Đại học Cần Thơ', text)
    write_to_file(os.path.join(clean_text_folder, file), text)
    print(file)


from collections import deque
import sys

while True:
	try:
		a=str(input('Input final configuration: '))
		N=int(a.replace(' ',''))
		if ((N>87654321)|(N<12345678)):
			raise ValueError('Incorrect configuration, giving up...')
		break
	except ValueError:
		print('Incorrect configuration, giving up...')
		sys.exit()

final=[int(i) for i in str(N)]

def initial_state(N):
    initial_list = []
    for i in range(1, N + 1):
        locals()['d' + str(i)] = i
        name_list = [('d' + '%d' % i) for i in range(1, N + 1)]
    for i in range(N):
        initial_list.append(vars()[name_list[i]])
    return initial_list

def row_exchange(L):
    r_L = list(L)
    d8, d7, d6, d5, d4, d3, d2, d1 = r_L[0], r_L[1], r_L[2], r_L[3], r_L[4], r_L[5], r_L[6], r_L[7]
    r_L[0], r_L[1], r_L[2], r_L[3], r_L[4], r_L[5], r_L[6], r_L[7] = d1, d2, d3, d4, d5, d6, d7, d8
    return r_L

def right_circular_shift(L):
    r_L = list(L)
    d1, d2, d3, d4, d5, d6, d7, d8 = r_L[0], r_L[1], r_L[2], r_L[3], r_L[4], r_L[5], r_L[6], r_L[7]
    r_L[0], r_L[1], r_L[2], r_L[3], r_L[4], r_L[5], r_L[6], r_L[7] = d4, d1, d2, d3, d6, d7, d8, d5
    return r_L

def middle_clockwise_rotation(L):
    r_L = list(L)
    d1, d2, d3, d4, d5, d6, d7, d8 = r_L[0], r_L[1], r_L[2], r_L[3], r_L[4], r_L[5], r_L[6], r_L[7]
    r_L[0], r_L[1], r_L[2], r_L[3], r_L[4], r_L[5], r_L[6], r_L[7] = d1, d7, d2, d4, d5, d3, d6, d8
    return r_L

def Rubik_Rectangle(initial_list, final_conf_list):
    steps_need = 0
    created_list = list(initial_list)
    check_list = []
    created_total = []
    created_all=deque()
    created_all.append([1, 2, 3, 4, 5, 6, 7, 8])
    temp_created_all=[]
    Flag = False
    temp_created_all=[]
    temp_created_all.append([1, 2, 3, 4, 5, 6, 7, 8])

    don_t_know=set()
    don_t_know.add((1, 2, 3, 4, 5, 6, 7, 8))

    def List_compare(list_1, list_2, i):
        if list_1 == list_2:
            return True
        else:
            return False

    for i in range(23):
        #print('I',i)
        #print('LENTH OF ALL',len(created_all))
        steps_need = i
        if initial_list == final_conf_list:
            return steps_need
        else:
                for element in range(0, len(created_all)):
                    created_list = created_all[element]
                    if tuple(row_exchange(created_list)) not in don_t_know:
                        created_total.append(row_exchange(created_list))
                        check_list = row_exchange(created_list)
                        Flag = List_compare(check_list, final_conf_list, steps_need)
                        if Flag == True:
                            #print(check_list)
                            return steps_need + 1
                for element in range(0, len(created_all)):
                    created_list = created_all[element]
                    if tuple(right_circular_shift(created_list)) not in don_t_know:
                        created_total.append(right_circular_shift(created_list))
                        check_list = right_circular_shift(created_list)
                        Flag = List_compare(check_list, final_conf_list, steps_need)
                        if Flag == True:
                            #print(check_list)
                            return steps_need + 1
                for element in range(0, len(created_all)):
                    created_list = created_all[element]
                    if tuple(middle_clockwise_rotation(created_list)) not in don_t_know:
                        created_total.append(middle_clockwise_rotation(created_list))
                        check_list = middle_clockwise_rotation(created_list)
                        Flag = List_compare(check_list, final_conf_list, steps_need)
                        if Flag == True:
                            #print(check_list)
                            return steps_need + 1
                #print('------------------------------------------')
                #bug
                bug_L=[1,3,7,14,26,51,92,159,274,453,720,1115,1727,2603,3701,4729,5620,6240,5840,4492,2120,328,5,0]

                created_all1=deque(maxlen=bug_L[steps_need+1])

                for i in range(len(created_total)):
                     if tuple(created_total[i]) not in don_t_know:
                        created_all1.append(created_total[i])
                        don_t_know.add(tuple(created_total[i]))
                temp_created_all=list(created_all1)

                created_all=list(created_all1)

                for num in range(len(created_all)):
                    check_list = created_all[num]
                    Flag = List_compare(check_list, final_conf_list, steps_need)
                    if Flag == True:
                        #print(check_list)
                        return steps_need + 1
                created_total = list()

initial_list=[1,2,3,4,5,6,7,8]
#final = [1,5,3,2,4,6,7,8]

#print('step',Rubik_Rectangle(initial_list, final))
nb_of_stairs=Rubik_Rectangle(initial_list, final)
stair_or_stairs = 'step is' if nb_of_stairs <= 1 else 'steps are'

print(f'{nb_of_stairs} {stair_or_stairs} needed to reach the final configuration.')

Principles of Programming/Assignment/Assignment1/ASS question2.files/rubiks_rectangle.py


import json
import os

import cherrypy
from jinja2 import Environment, FileSystemLoader

# GET CURRENT DIRECTORY
from helper import get_redis_connection, get_sorted_list
from scrapper import main1

CUR_DIR = os.path.dirname(os.path.abspath(__file__))
env = Environment(loader=FileSystemLoader(CUR_DIR), trim_blocks=True)

class Index(object):
    @cherrypy.expose
    def index(self):
        template = env.get_template('templates/home.html')
        r = get_redis_connection()
        try:
            users = json.loads(r.get('ten_users'))
        except:
            main1()  # If file was not downloaded then download the latest file.
            users = json.loads(r.get('ten_users'))

        return template.render(users=users, name='JAYANTH')

@cherrypy.expose
class UserService(object):
    @cherrypy.tools.json_out()
    def POST(self, name):
        r = get_redis_connection()
        users = json.loads(r.get('users'))
        response = {}
        try:
            res = [user for user in users if name.lower() in user["name"].lower()]  # search by name (substring match)

            result = get_sorted_list(res, 'dict')

            response['success'] = True
            response['users'] = result[:10]
            response['length'] = res.__len__()
            return response
        except:
            response['success'] = False
            return response

if __name__ == '__main__':
    conf = {
        '/': {
            'tools.sessions.on': True,
            'tools.staticdir.root': os.path.abspath(os.getcwd())
        },
        '/get_users': {
            'request.dispatch': cherrypy.dispatch.MethodDispatcher(),
            'tools.response_headers.on': True,
        },
        '/static': {
            'tools.staticdir.on': True,
            'tools.staticdir.dir': './static'
        }
    }
    webapp = Index()
    webapp.get_users = UserService()
    cherrypy.server.socket_host = '0.0.0.0'
    cherrypy.quickstart(webapp, '/', conf)


from tensorflow.keras.models import load_model
from tensorflow.keras.preprocessing.image import ImageDataGenerator
import numpy as np
import cv2
import matplotlib.pyplot as plt
import matplotlib.image as mpimg

model = load_model("/home/yash/Desktop/PyImageSearch/checkpoints/emotion1.h5")
classifier = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')

img_size = 48

validation_dir = "/home/yash/Desktop/PyImageSearch/deep_survelliance_detector/fer2013/validation/"

validation_datagen = ImageDataGenerator(rescale=1./255)

validation_generator = validation_datagen.flow_from_directory(validation_dir,
								color_mode="grayscale",
								target_size=(img_size,img_size),
								batch_size=32,
								class_mode="categorical",
								shuffle=False)

#getting class labels
class_labels = validation_generator.class_indices
class_labels = {v:k for v,k in class_labels.items()}

#predicting
test_img = "/home/yash/Downloads/happy1.jpg"

def get_label(prediction):
	for key, val in class_labels.items():
		if prediction == val:
			return key
	return -1

def predict(test_img):
	img = cv2.imread(test_img,cv2.IMREAD_GRAYSCALE)
	faces = classifier.detectMultiScale(img,scaleFactor=1.2,minNeighbors=7)
	face = []
	for (x,y,w,h) in faces:
		roi_gray = img[y:y+h,x:x+w]
		roi = cv2.resize(roi_gray, (img_size,img_size), interpolation=cv2.INTER_AREA)
		face.append(roi)
	num_image = np.array(face, dtype=np.float32)
	num_image /= 255.0
	num_image = num_image.reshape(1,48,48,1)
	predicted = model.predict(num_image)[0] #returns a list of probabilities of diff classes
	pred = predicted.argmax() #getting the max value in the list
	label = get_label(pred)
	return label

pred_class = predict(test_img)
original_image = mpimg.imread(test_img)
plt.xlabel("Predicted: {0}".format(str(pred_class)))
plt.imshow(original_image)
plt.show()

Emotion and Gender Classification/Emotion Classification/recognition.py


import numpy as np
import math
import cv2

#input hsvColor:[h,s,v], ranges between (0-180, 0-255, 0-255)
#and output true hsvColor:[h,s,v]
def get_right_V(hsvColor):
    h = float(hsvColor[0])
    s = float(hsvColor[1])
    s1 = float(hsvColor[1])/255
    v1 = float(hsvColor[2])
    h60 = h / 60.0
    h60f = math.floor(h60)
    f = h60 - h60f
    if f<0.5:
        v = 3*v1*(1-f*s1)/(3-2*s1)
    elif f>=0.5:
        v = 3*v1*(1-s1+f*s1)/(3-2*s1)
    return [h, s, min(v,255)]

if __name__=="__main__":
    #input and ouput path for image.
    PATH_TO_WRONG_IMAGE = './wrong_image.jpg'
    PATH_TO_RIGHT_IMAGE = './right_image.jpg'

    image_wrong = cv2.imread(PATH_TO_WRONG_IMAGE)
    image_wrong_hsv = cv2.cvtColor(image_wrong, cv2.COLOR_BGR2HSV)
    shape = image_wrong.shape
    image_right_hsv = np.zeros(shape, dtype=np.uint8)

    #iterate over every pixel to change the V value.
    for row in range(shape[0]):
        for col in range(shape[1]):
            image_right_hsv[row][col] = get_right_V(image_wrong_hsv[row][col])

    image_right = cv2.cvtColor(image_right_hsv, cv2.COLOR_HSV2BGR)

    cv2.imwrite(PATH_TO_RIGHT_IMAGE, image_right)


import torch
import logging
from tqdm import tqdm
from schnetpack.src.schnetpack import properties

__all__ = ["TorchStructureLBFGS"]

class TorchStructureLBFGS(torch.optim.LBFGS):
    """
    LBFGS optimizer that allows for relaxation of multiple structures in parallel. The approximation of the inverse
    hessian is shared across the entire batch (all structures). Hence, it is recommended to use this optimizer
    preferably for batches of similar structures/compositions. In other cases, please utilize the ASELBFGS optimizer,
    which is particularly constructed for batches of different structures/compositions.

    This optimizer is an extension/adaptation of the torch.optim.LBFGS optimizer particularly designed for relaxation
    of atomic structures. In addition to the inherited features, this optimizer allows for fixing the positions of a
    set of atoms during the relaxation and a method to run the optimizer. Latter allows for setting a convergence
    criterium. Furthermore, we implemented a logging method that prints out the largest force in the system after each
    optimization iteration.
    """

    def __init__(
        self,
        model,
        model_inputs,
        fixed_atoms_mask,
        maxstep=None,
        logging_function=None,
        lr: float = 1.0,
        energy_key: str = "energy",
        position_key: str = properties.R,
    ):
        """
        Args:
            model (schnetpack.model.AtomisticModel): ml force field model
            model_inputs: input batch containing all structures
            fixed_atoms_mask (list(bool)): list of booleans indicating to atoms with positions fixed in space.
            maxstep (float): how far is a single atom allowed to move. (default: None)
            logging_function: function that logs the structure of the systems during the relaxation
            lr (float): learning rate (default: 1)
            energy_key (str): name of energies in model (default="energy")
            position_key (str): name of atomic positions in model (default="_positions")
        """

        self.model = model
        self.energy_key = energy_key
        self.position_key = position_key
        self.fixed_atoms_mask = fixed_atoms_mask
        self.model_inputs = model_inputs
        self.logging_function = logging_function
        self.fmax = None
        self.maxstep = maxstep

        R = self.model_inputs[self.position_key]
        R.requires_grad = True
        super().__init__(params=[R], lr=lr)

    def _gather_flat_grad(self):
        """override this function to allow for keeping atoms fixed during the relaxation"""
        views = []
        for p in self._params:
            if p.grad is None:
                view = p.new(p.numel()).zero_()
            elif p.grad.is_sparse:
                view = p.grad.to_dense().view(-1)
            else:
                view = p.grad.view(-1)
            views.append(view)
        flat_grad = torch.cat(views, 0)
        if self.fixed_atoms_mask is not None:
            flat_grad[self.fixed_atoms_mask] = 0.0
        self.flat_grad = flat_grad
        return flat_grad

    def _add_grad(self, step_size, update):
        offset = 0

        if self.maxstep is not None:
            step_size = self.determine_step_size(step_size, update)

        for p in self._params:
            numel = p.numel()
            # view as to avoid deprecated pointwise semantics
            p.add_(update[offset : offset + numel].view_as(p), alpha=step_size)
            offset += numel
        assert offset == self._numel()

    def determine_step_size(self, step_size, update):
        """Determine step to take according to maxstep

        Normalize all steps as the largest step. This way
        we still move along the eigendirection.
        """
        reshaped_update = update.view(-1, 3)
        steplengths = ((step_size * reshaped_update) ** 2).sum(1) ** 0.5
        longest_step = torch.max(steplengths)
        # check if any step in entire batch is greater than maxstep
        if longest_step >= self.maxstep:
            # rescale all steps
            logging.info("normalized integration step")
            step_size *= self.maxstep / longest_step
        return step_size

    def closure(self):
        results = self.model(self.model_inputs)
        self.zero_grad()
        loss = results[self.energy_key].sum()
        loss.backward()
        return loss

    def log(self, forces=None):
        """log relaxation results such as max force in the system"""
        if forces is None:
            forces = self.flat_grad.view(-1, 3)
        if not self.converged():
            logging.info("NOT CONVERGED")
        logging.info(
            "max. atomic force: {}".format(torch.sqrt((forces**2).sum(axis=1).max()))
        )

    def converged(self, forces=None):
        """Did the optimization converge?"""
        if forces is None:
            forces = self.flat_grad.view(-1, 3)
        return (forces**2).sum(axis=1).max() < self.fmax**2

    def run(self, fmax, max_opt_steps):
        """run relaxation"""
        self.fmax = fmax

        # optimization
        for opt_step in tqdm(range(max_opt_steps)):
            self.step(self.closure)

            # log structure
            if self.logging_function is not None:
                self.logging_function(opt_step)

            # stop optimization if max force is smaller than threshold
            if self.converged():
                break
        self.log()

    def get_relaxed_structure(self):
        return self.model_inputs[self.position_key]


import pickle
import torch
import argparse

from foresight.models import *
from foresight.pruners import *
from foresight.dataset import *
from foresight.weight_initializers import init_net

def get_num_classes(args):
    return 100 if args.dataset == 'cifar100' else 10 if args.dataset == 'cifar10' else 120

def parse_arguments():
    parser = argparse.ArgumentParser(description='Zero-cost Metrics for NAS-Bench-201')
    parser.add_argument('--api_loc', default='data/NAS-Bench-201-v1_0-e61699.pth',
                        type=str, help='path to API')
    parser.add_argument('--outdir', default='./',
                        type=str, help='output directory')
    parser.add_argument('--init_w_type', type=str, default='none', help='weight initialization (before pruning) type [none, xavier, kaiming, zero]')
    parser.add_argument('--init_b_type', type=str, default='none', help='bias initialization (before pruning) type [none, xavier, kaiming, zero]')
    parser.add_argument('--batch_size', default=64, type=int)
    parser.add_argument('--dataset', type=str, default='cifar10', help='dataset to use [cifar10, cifar100, ImageNet16-120]')
    parser.add_argument('--gpu', type=int, default=0, help='GPU index to work on')
    parser.add_argument('--num_data_workers', type=int, default=2, help='number of workers for dataloaders')
    parser.add_argument('--dataload', type=str, default='random', help='random or grasp supported')
    parser.add_argument('--dataload_info', type=int, default=1, help='number of batches to use for random dataload or number of samples per class for grasp dataload')
    parser.add_argument('--seed', type=int, default=42, help='pytorch manual seed')
    parser.add_argument('--write_freq', type=int, default=1, help='frequency of write to file')
    parser.add_argument('--start', type=int, default=0, help='start index')
    parser.add_argument('--end', type=int, default=0, help='end index')
    parser.add_argument('--noacc', default=False, action='store_true', help='avoid loading NASBench2 api an instead load a pickle file with tuple (index, arch_str)')
    args = parser.parse_args()
    args.device = torch.device("cuda:"+str(args.gpu) if torch.cuda.is_available() else "cpu")
    return args

if __name__ == '__main__':
    args = parse_arguments()

    if args.noacc:
        api = pickle.load(open(args.api_loc,'rb'))
    else:
        from nas_201_api import NASBench201API as API
        api = API(args.api_loc)

    torch.manual_seed(args.seed)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False

    train_loader, val_loader = get_cifar_dataloaders(args.batch_size, args.batch_size, args.dataset, args.num_data_workers)

    cached_res = []
    pre='cf' if 'cifar' in args.dataset else 'im'
    pfn=f'nb2_{pre}{get_num_classes(args)}_seed{args.seed}_dl{args.dataload}_dlinfo{args.dataload_info}_initw{args.init_w_type}_initb{args.init_b_type}.p'
    op = os.path.join(args.outdir,pfn)

    args.end = len(api) if args.end == 0 else args.end

    #loop over nasbench2 archs
    for i, arch_str in enumerate(api):

        if i < args.start:
            continue
        if i >= args.end:
            break

        res = {'i':i, 'arch':arch_str}

        net = nasbench2.get_model_from_arch_str(arch_str, get_num_classes(args))
        net.to(args.device)

        init_net(net, args.init_w_type, args.init_b_type)

        arch_str2 = nasbench2.get_arch_str_from_model(net)
        if arch_str != arch_str2:
            print(arch_str)
            print(arch_str2)
            raise ValueError

        measures = predictive.find_measures(net,
                                            train_loader,
                                            (args.dataload, args.dataload_info, get_num_classes(args)),
                                            args.device)

        res['logmeasures']= measures

        if not args.noacc:
            info = api.get_more_info(i, 'cifar10-valid' if args.dataset=='cifar10' else args.dataset, iepoch=None, hp='200', is_random=False)

            trainacc = info['train-accuracy']
            valacc   = info['valid-accuracy']
            testacc  = info['test-accuracy']

            res['trainacc']=trainacc
            res['valacc']=valacc
            res['testacc']=testacc

        #print(res)
        cached_res.append(res)

        #write to file
        if i % args.write_freq == 0 or i == len(api)-1 or i == 10:
            print(f'writing {len(cached_res)} results to {op}')
            pf=open(op, 'ab')
            for cr in cached_res:
                pickle.dump(cr, pf)
            pf.close()
            cached_res = []


from time import time
from gurobipy import Model

class MDSP:

    def __init__(self, d: list, filename: str, optimize=False, time_limit=3600):
        self.D = d
        self.B = sum(d)
        self.k = len(self.D)
        self.D_ = self.get_unique_distances()
        self.M = self.get_mult()
        self.P = list(range(self.B + 1))
        self.filename = filename

        if optimize:
            self.P = self.valid_points()

        self.model = Model()
        self.model.setParam('LogFile', f'{filename}.log')
        self.model.setParam('LogToConsole', 0)
        self.model.setParam('TimeLimit', time_limit)

    def solve(self):
        t1 = time()
        self.model.update()
        self.model.optimize()
        t2 = time()

        self.write_time_file(t1, t2)
        self.model.write(f'{self.filename}.lp')

        if self.model.status == 3:
            print('Infeasible')
        else:
            self.model.write(f'{self.filename}.sol')
            print('Obj: %s' % self.model.ObjVal)

    def get_unique_distances(self):
        d = list(set(self.D))
        d.sort()

        return d

    def get_mult(self):
        m = dict()

        for i in self.D_:
            m[i] = self.D.count(i)

        return m

    def valid_points(self):
        c = {0}

        for d in self.D:
            t = set()

            for p in c:
                t = t.union({p-d, p+d})

            c = c.union(t)

        p = [x for x in c if x >= 0]
        p.sort()

        return p

    def write_time_file(self, t1, t2):
        with open(f'{self.filename}.time', 'w') as file:
            file.write(f'{self.model.Status}\n')
            file.write(f'{t2 - t1}\n')
            file.close()


import pygame
from pygame.locals import *
import numpy as np
from conway import Life

class GameOfLife(Life):
    def __init__(self, width = 1600, height = 1000, cell_size = 5, speed = 10):
        Life.__init__(self, width // 10, height // 10)
        self.width = width
        self.height = height
        self.cell_size = cell_size

        # Устанавливаем размер окна
        self.screen_size = width, height
        # Создание нового окна
        self.screen = pygame.display.set_mode(self.screen_size)

        # Вычисляем количество ячеек по вертикали и горизонтали
        self.cell_width = self.width // self.cell_size
        self.cell_height = self.height // self.cell_size

        # Скорость протекания игры
        self.speed = speed
        self.Mboard()

    def draw_grid(self):

        for x in range(0, self.width, self.cell_size):
            pygame.draw.line(self.screen, pygame.Color('black'),
                (x, 0), (x, self.height))
        for y in range(0, self.height, self.cell_size):
            pygame.draw.line(self.screen, pygame.Color('black'),
                (0, y), (self.width, y))

    def cells(self):
        for row in range(self.rows):
            for column in range(self.columns):
                if self.board[row, column] == 1:
                    pygame.draw.rect(self.screen, (57, 255, 20), ((row * self.cell_size) + 1, (column * self.cell_size) +1, self.cell_size -1,
                                     self.cell_size -1))
                else:
                    pygame.draw.rect(self.screen, (255, 255, 255), ((row * self.cell_size) + 1, (column * self.cell_size) + 1, self.cell_size -1,
                                     self.cell_size -1))

    def run(self):
        pygame.init()
        clock = pygame.time.Clock()
        pygame.display.set_caption('Game of Life')
        self.screen.fill(pygame.Color('white'))
        running = True
        while running:
            for event in pygame.event.get():
                if event.type == QUIT:
                    running = False
            self.draw_grid()
            self.cells()
            self.cell()
            pygame.display.flip()
            clock.tick(self.speed)
        pygame.quit()

if __name__ == '__main__':
    game = GameOfLife()
    game.run()


#import libraries
import pandas as pd
import numpy as np
import bokeh
from bokeh.plotting import figure, output_file, show
from bokeh.models.tools import HoverTool
from bokeh.core.properties import value
from bokeh.models import ColumnDataSource, FactorRange
from bokeh.plotting import figure
import math
from bokeh.models import Range1d, LabelSet, Label

class Flight_Arrivals():
    def __init__(self):
        pass

    def flights(self):
        avion=pd.read_csv("fly_mia.csv",encoding="latin-1")
        a=avion
        a['est_arr_time'] = a['est_arr_time'].str.replace('?', '')
        a['est_arr_time']=a['est_arr_time'].str.replace(r"\(.*\)","")
        a=a[a.est_arr_time.str.contains('0')]

        sun1=a[a.est_arr_time.str.contains('Sun')]
        sun1['est_arr_time'] = sun1['est_arr_time'].str.replace('Sun', '2019-08-18')
        sun1['dep_time'] = sun1['dep_time'].str.replace('Sun', '2019-08-18')
        sat1=a[a.est_arr_time.str.contains('Sat')]
        sat1['est_arr_time'] = sat1['est_arr_time'].str.replace('Sat', '2019-08-17')
        sat1['dep_time'] = sat1['dep_time'].str.replace('Sat', '2019-08-17')
        fri1=a[a.est_arr_time.str.contains('Fri')]
        fri1['est_arr_time'] =fri1['est_arr_time'].str.replace('Fri', '2019-08-16')
        fri1['dep_time'] =fri1['dep_time'].str.replace('Fri', '2019-08-16')

        ok2=pd.concat([sun1,sat1,fri1],axis=0)
        ok2['dep_time'] =ok2['dep_time'].str.replace('Fri', '2019-08-16')
        ok2['dep_time'] =ok2['dep_time'].str.replace('Sat', '2019-08-17')

        ok2['dep_time']=pd.to_datetime(ok2['dep_time'])
        ok2['est_arr_time']=pd.to_datetime(ok2['est_arr_time'])

        ok2['flight_time']=ok2['est_arr_time']-ok2['dep_time']
        ok2['flight_time']=ok2['flight_time'].dt.total_seconds()
        ok2['flight_time']=ok2['flight_time']/60 #to minutes

        #airport time zones (departure zones)
        #1. cest
        cest=ok2[ok2.origin.str.contains('MAD|ZRH|BRU|MXP|CDG|DUS|FCO|VIE|FRA|Pisa|BCN|ZAZ|WAW|ORY|AMS')]
        cest['flight_time']=cest['flight_time']+360
        cest['flight_time'] = cest['flight_time'].apply(lambda x: 561 if x < 400 else x)

        #2.south american flights
        sa=ok2[ok2.origin.str.contains("GIG|FOR|COR|EZE|Dois de|BSB|GRU|REC|MVD|BEL|SNU")]
        sa['flight_time']=sa['flight_time']+60
        sa['flight_time']=sa['flight_time'].apply(lambda x: 451.5 if x<350 else x)
        otro=ok2[~ok2.origin.str.contains('MAD|ZRH|BRU|MXP|CDG|DUS|FCO|VIE|FRA|Pisa|BCN|ZAZ|WAW|ORY|AMS|GIG|FOR|COR|EZE|Dois de|BSB|GRU|REC|MVD|BEL|SNU')]
        todos=pd.concat([cest,sa,otro],axis=0)

        # percent of flights less one hour
        bins=[0,60,120,180,240,300,360,420,480,540,600,660]
        todos['flight_bins']=pd.cut(todos['flight_time'], bins)

        pct_time=todos['flight_bins'].value_counts()
        pct_time=pd.DataFrame(pct_time)
        pct_time.reset_index(level=0,inplace=True)
        pct_time['pct']=pct_time['flight_bins']/todos.shape[0]

        #ii. variance by origin
        vaR=todos.groupby('origin')['flight_time'].var()
        vaR.sort_values()

        #iii. arrives by part of the day
        tiempo=todos[["origin","est_arr_time"]]
        t=tiempo
        t['hours']=t['est_arr_time'].dt.hour
        t['minutes']=t['est_arr_time'].dt.minute

        mid_six=t[(t.hours>=0) & (t.hours<=6)]
        seven_twelve=t[(t.hours>=7) & (t.hours<=12)]
        one_six=t[(t.hours>=13) & (t.hours<=18)]
        seven_twelve1=t[(t.hours>=19) & (t.hours<=23)]

        #percent arrivals by time of the day
        mid_sixP=mid_six.shape[0]/t.shape[0]
        seven_twelveP=seven_twelve.shape[0]/t.shape[0]
        one_sixP=one_six.shape[0]/t.shape[0]
        seven_twelveP1=seven_twelve1.shape[0]/t.shape[0]

        #origin counts
        ori=t['origin'].value_counts()
        ori=pd.DataFrame(ori)
        ori.reset_index(level=0,inplace=True)
        ori.columns=['origin','total']

        #time between flights
        tX=todos
        tX.sort_values(['origin','dep_time'],inplace=True)
        tX['diff_dep']=tX['dep_time'].diff()
        mask=tX.origin !=tX.origin.shift(1)
        tX['diff_dep'][mask]=np.nan
        tX['diff_dep']=tX['diff_dep'].dt.total_seconds()
        tX['diff_dep']=tX['diff_dep']/60 #to minutes
        tX.iloc[0:10]
        tX=tX[~(tX.diff_dep==0)]

        takeoffs=tX.groupby('origin')['diff_dep'].median()
        takeoffs=takeoffs.sort_values()
        takeoffs=pd.DataFrame(takeoffs)
        take=takeoffs
        take=take[take.diff_dep>=1]
        take1=take[take.diff_dep<=80]

        s=t
        s=s.set_index('est_arr_time')
        s=s.loc['2019-08-17 00:00:00':'2019-08-17 23:59:59']

        #VIZ I
        #east coast time vs. cst,pdt, and mdt (comparing flight times)
        west_cent=tX[tX.origin.str.contains('LAX|SFO|LAS|SEA|SAN|SNU|DFW|MEX|MDW|MSY|CMW|MEM|ORD|TUL|MSP|MCI|STL|MID|IAH|VRA|PNS|GDL|MTY|KSAT|BHM|SCU|HOG|TLC|HSV')]
        east=tX[tX.origin.str.contains('NAS|PHI|Toron|Bahama|DCA|HAV|ORF|TPA|LGA|JAX|SAV|SDF|PIE|GGT|PLS|CVG|PIT|CHS|CLE|JFK|CAP|IND|DTW|KEY|CMH|BUF|RDU|SFB|MYEH|MYAM|CYUL|GSP|PBI|RIC|GSO|FMY|BDL|BWI|KTEB|ZSA|KMLB|KAPF|SGJ')]

        #length of flights
        wc=west_cent['flight_bins'].value_counts()
        wc=pd.DataFrame(wc)
        wc.columns=['flight_time']
        wc.reset_index(level=0,inplace=True)
        wc=wc.sort_values(by="index")
        wc=wc.set_index('index')

        ea=east['flight_bins'].value_counts()
        ea=pd.DataFrame(ea)
        ea.columns=['flight_time']
        ea.reset_index(level=0,inplace=True)
        ea=ea.sort_values(by="index")
        ea=ea.set_index('index')

        factors=[("0-60"),("60-120"),("120-180"), ("180-240"),("240-300"),("300-360"),("360-420"),("420-480"),("480-540"),("540-600"),("600-660")]
        regions=['east_time_zone','other_time_zone']
        east_data=ea.flight_time.tolist()
        west_data=wc.flight_time.tolist()

        source=ColumnDataSource(data=dict(x=factors,east_time_zone=east_data,other_time_zone=west_data,))

        p = figure(x_range=FactorRange(*factors), plot_height=250,toolbar_location=None, tools="")
        p.vbar_stack(regions, x='x', width=0.9, alpha=0.5, color=["orange", "purple"], source=source,legend=[value(x) for x in regions])

        p.y_range.start = 0
        p.y_range.end = 120
        p.x_range.range_padding = 0.1
        p.xaxis.major_label_orientation = 1
        p.xgrid.grid_line_color = None
        p.xaxis.axis_label='Flight Time (Minutes)'
        p.yaxis.axis_label='Frequency'
        p.legend.location = "top_right"
        p.legend.orientation = "horizontal"
        output_file("mia1.html")
        #show(p)

        #VIZ II (time between departures)
        source1=ColumnDataSource(take1)
        airports=source1.data['origin'].tolist()
        p1=figure(x_range=airports)
        p1.vbar_stack(stackers=['diff_dep'],x='origin',source=source1,width=0.5)
        p1.title.text='Time Between Flight Departures'
        p1.title.align="center"
        p1.title.text_color="orange"
        p1.xaxis.major_label_orientation = math.pi/4.25
        p1.xaxis.axis_label=''
        p1.yaxis.axis_label='Minutes'
        hover=HoverTool()
        hover.tooltips=[("Time Between Flights","@diff_dep minutes")]
        hover.mode='vline'
        p1.add_tools(hover)
        output_file("mia2.html")
        #show(p1)

        #VIZ III (what time of the day do flights arrive?)
        time_arr=['Midnight to 7 AM','7 AM to 1 PM','1 PM to 7 PM','7 PM to Midnight']
        counts=[mid_sixP,seven_twelveP1,one_sixP,seven_twelveP1]
        palette=['lavender','plum','darkviolet','indigo']

        source = ColumnDataSource(data=dict(time_arr=time_arr, counts=counts))

        p = figure(x_range=time_arr, plot_height=250, toolbar_location=None, title="When Do Flights to X Arrive?")
        p.vbar(x='time_arr', top='counts', width=0.5, source=source, color="teal",
        line_color='white')

        p.xgrid.grid_line_color = None
        p.y_range.start = 0.0
        p.y_range.end = 0.6
        p.xaxis.axis_label=""
        p.yaxis.major_label_overrides = {0:'0',0.1:'10%',0.2:'20%',0.3:'30%',0.4:'40%',0.5:'50%'}
        p.yaxis.axis_label="Total Flights"
        p.legend.orientation = "horizontal"
        p.legend.location = "top_center"
        p.title.align="center"
        output_file("mia3.html")
        #show(p)

        #VIZ IV (outlier flights time plot)
        top_diez=tX['origin'].value_counts()
        top_diez=pd.DataFrame(top_diez)
        top_diez.reset_index(level=0,inplace=True)
        air_names=top_diez.iloc[0:10]["index"]
        an=air_names
        an0=an.iloc[0]
        an1=an.iloc[1]
        an2=an.iloc[2]
        an3=an.iloc[3]
        an4=an.iloc[4]
        an5=an.iloc[5]
        an6=an.iloc[6]
        an7=an.iloc[7]
        an8=an.iloc[8]
        an9=an.iloc[9]

        sub_air=tX[(tX.origin==an0) | (tX.origin==an1) | (tX.origin==an2) | (tX.origin==an3) | (tX.origin==an4) | (tX.origin==an5) | (tX.origin==an6) | (tX.origin==an7) | (tX.origin==an8) | (tX.origin==an9)]
        df=pd.DataFrame(dict(flight_time=sub_air['flight_time'],group=sub_air['origin']))
        originS=df['group'].unique().tolist()
        groups=df.groupby('group')
        q1=groups.quantile(q=0.25)
        q2=groups.quantile(q=0.50)
        q3=groups.quantile(q=0.75)
        iqr=q3-q1
        upper=q3+1.5*iqr
        lower=q1-1.5*iqr

        #find outliers in each group
        def outliers(group):
            originS=group.name
            return group[(group.flight_time > upper.loc[originS]['flight_time']) | (group.flight_time < lower.loc[originS]['flight_time'])]['flight_time']
        out=groups.apply(outliers).dropna()

        #prepare outlier data for plotting
        if not out.empty:
            outx=[]
            outy=[]
            for keys in out.index:
                outx.append(keys[0])
                outy.append(out.loc[keys[0]].loc[keys[1]])

        p = figure(tools="", background_fill_color="#efefef", x_range=originS, toolbar_location=None)

        #if no outliers, shrink lengths of stems to be no longer than the minimums or maximums
        qmin=groups.quantile(q=0.00)
        qmax=groups.quantile(q=1.00)
        upper.score=[min([x,y]) for (x,y) in zip(list(qmax.loc[:,'flight_time']),upper.flight_time)]
        lower.score = [max([x,y]) for (x,y) in zip(list(qmin.loc[:,'flight_time']),lower.flight_time)]

        # stems
        p.segment(originS, upper.flight_time, originS, q3.flight_time, line_color="black")
        p.segment(originS, lower.flight_time, originS, q1.flight_time, line_color="black")

        # boxes
        p.vbar(originS, 0.7, q2.flight_time, q3.flight_time, fill_color="aqua", line_color="black")
        p.vbar(originS, 0.7, q1.flight_time, q2.flight_time, fill_color="maroon", line_color="black")

        # whiskers (almost-0 height rects simpler than segments)
        p.rect(originS, lower.flight_time, 0.2, 0.01, line_color="black")
        p.rect(originS,upper.flight_time, 0.2, 0.01, line_color="black")

        # outliers
        if not out.empty:
            p.circle(outx, outy, size=6, color="#F38630", fill_alpha=0.6)

        p.xgrid.grid_line_color = None
        p.ygrid.grid_line_color = "white"
        p.grid.grid_line_width = 2
        p.xaxis.major_label_text_font_size="12pt"
        p.xaxis.major_label_orientation = 3.5/2
        p.xaxis.axis_label = ''
        p.yaxis.axis_label = 'Flight Time (minutes)'
        p.title.text='Flights That Are Shorter or Longer Than Average'
        p.title.align="center"
        output_file('mia4x.html')
        #show(p)

        #VIZ V
        dep=tX['diff_dep'].tolist()
        time=tX['flight_time'].tolist()
        airports=tX['origin'].tolist()

        source=ColumnDataSource(data=dict(dep=dep,time=time,airports=airports))
        p=figure(title="Flight Time Vs. Time Between Departures",x_range=Range1d(0,1000))
        p.scatter(x="dep",y="time",size=4,source=source)
        p.xaxis[0].axis_label="Time Between Flights (Minutes)"
        p.yaxis[0].axis_label="Flight Time (Minutes)"

        labels = LabelSet(x='dep', y='time', text='airports', level='glyph',x_offset=5, y_offset=5, source=source, render_mode='canvas')

        p.add_layout(labels)
        show(p)

if __name__=='__main__':
    flights=Flight_Arrivals()
    flights.flights()


import pygame, threading

pygame.init()

white = (255, 255, 255)
green = (0, 255, 0)
blue = (0, 0, 128)
X = 400
Y = 400
display_surface = pygame.display.set_mode((X, Y ))
pygame.display.set_caption('Show Text')
font = pygame.font.Font('freesansbold.ttf', 32)
text = font.render('GeeksForGeeks', True, green, blue)
textRect = text.get_rect()
textRect.center = (X // 2, Y // 2)

def updateGUI():
    while True:
        display_surface.fill(white)
        display_surface.blit(text, textRect)

        for event in pygame.event.get() :
            if event.type == pygame.QUIT :
                pygame.quit()
                quit()
            pygame.display.update()

GUI = threading.Thread(target = updateGUI, args = ())
GUI.daemon = True
GUI.start()


from enum import Enum
from typing import List

import sqlalchemy as sa
from sqlalchemy import orm as so

from .base import BaseMixin, db, IdentityMixin, TimestampMixin

__all__ = ['Chat', 'ChatEntry']

class Chat(BaseMixin, IdentityMixin, TimestampMixin, db.Model):
    """Chat Model.

    Represents a chat conversation in the application.
    """

    __tablename__ = 'chats'

    title: so.Mapped[str] = so.mapped_column(
        sa.String(64),
        unique=False,
        index=True,
        nullable=False,
    )

    entry: so.Mapped[List['ChatEntry']] = so.relationship(
        back_populates='chat',
    )

    def teaser(self, length=None) -> str:
        """Return a teaser for the chat."""
        if not self.entry:
            return ''

        first_entry = min(self.entry, key=lambda x: x.created_at)
        length = length or 150

        if len(first_entry.content) > length:
            return first_entry.content[:length - 3] + '...'

        return first_entry.content[:length]

    @classmethod
    def create_new_chat(cls, title=None):
        """Create a new chat and return the new chat."""
        title = title or 'New chat'
        new_chat = cls.create(title=title)
        new_chat.save()
        return new_chat

class ChatEntry(BaseMixin, IdentityMixin, TimestampMixin, db.Model):
    """ChatEntry Model.

    Represents a single message in a Chat.
    """
    class Role(Enum):
        """Represents the role of a :class:`.ChatEntry`."""

        USER = 'user'
        ASSISTANT = 'assistant'

        def __str__(self) -> str:
            """Get a string representation of this role.

            :return: The name of this role.
            :rtype: str
            """
            return self.name.strip().lower()

    __tablename__ = 'chat_entries'

    content: so.Mapped[str] = so.mapped_column(
        sa.Text,
        nullable=False,
    )

    chat_id: so.Mapped[int] = so.mapped_column(
        sa.ForeignKey('chats.id'),
        nullable=False
    )

    role: so.Mapped[Role] = so.mapped_column(
        sa.Enum(
            Role,
            name='role_types',
            values_callable=lambda obj: [str(item.value) for item in obj]
        ),
        nullable=False,
    )

    chat: so.Mapped['Chat'] = so.relationship(
        back_populates='entry',
    )


'''This script contains the functions used to contruct and train the GAN.'''

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import os
import tensorflow as tf
# Change the environment variable TF_CPP_MIN_LOG_LEVEL to 2 to avoid the orderbooks about the compilation of the CUDA code
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
from data_utils import *
import argparse
import logging
# from tensorflow.keras.utils import plot_model
from model_utils import *
import math
import gc
from scipy.stats import wasserstein_distance
import sys

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='''Main script used to train the GAN.''')
    parser.add_argument("-l", "--log", default="info",
                        help=("Provide logging level. Example --log debug', default='info'"))
    parser.add_argument('-N', '--N_days', type=int, help='Number of the day to consider')
    parser.add_argument('-d', '--depth', help='Depth of the orderbook', type=int)
    parser.add_argument('-bs', '--batch_size', help='Batch size', type=int)
    parser.add_argument('-ld', '--latent_dim', help='Latent dimension', type=int)
    parser.add_argument('-nlg', '--n_layers_gen', help='Number of generator layers', type=int)
    parser.add_argument('-nld', '--n_layers_disc', help='Number of discriminator layers', type=int)
    parser.add_argument('-tg', '--type_gen', help='Type of generator model (conv, lstm, dense)', type=str)
    parser.add_argument('-td', '--type_disc', help='Type of discriminator model (conv, lstm, dense)', type=str)
    parser.add_argument('-sc', '--skip_connection', action='store_true', help='Use or not skip connections')
    parser.add_argument('-Tc', '--T_condition', help='Number of time steps to condition on', type=int, default=2)
    parser.add_argument('-Tg', '--T_gen', help='Number of time steps to generate', type=int, default=1)
    parser.add_argument('-ls', '--loss', help='Loss function (original, wasserstein)', type=str, default='original')
    parser.add_argument('-lo', '--load', help='Load a model. The job_id must be provided', type=int, default=0)

    args = parser.parse_args()
    levels = {'critical': logging.CRITICAL,
              'error': logging.ERROR,
              'warning': logging.WARNING,
              'info': logging.INFO,
              'debug': logging.DEBUG}

    if os.getenv("PBS_JOBID") != None:
        job_id = os.getenv("PBS_JOBID")
    else:
        job_id = os.getpid()

    logging.basicConfig(filename=f'train_{job_id}.log', format='%(message)s', level=levels[args.log])

    logger = tf.get_logger()
    logger.setLevel('ERROR')

    # Set the seed for TensorFlow to the number of the beast
    tf.random.set_seed(666)

    # Print the current date and time
    current_datetime = pd.Timestamp.now()
    formatted_datetime = current_datetime.strftime("%Y-%m-%d %H:%M:%S")
    logging.info(f"Current Date and Time:\n\t {formatted_datetime}")

    # Enable device placement logging
    tf.debugging.set_log_device_placement(True)

    # Load the data
    stock = 'MSFT'
    date = '2018-04-01_2018-04-30_5'
    total_depth = 5
    N = args.N_days
    depth = args.depth

    logging.info(f'Stock:\n\t{stock}')
    logging.info(f'Number of days:\n\t{N}')

    physical_devices = tf.config.experimental.list_physical_devices('GPU')
    if len(physical_devices) == 0:
        logging.info("No GPUs available.")
    else:
        logging.info("Available GPUs:")
        tf.config.experimental.set_memory_growth(physical_devices[0], True)
        for device in physical_devices:
            logging.info(f'\t{device}\n')

    # Folders creation
    os.mkdir(f'plots/{job_id}_{args.type_gen}_{args.type_disc}_{args.n_layers_gen}_{args.n_layers_disc}_{args.T_condition}_{args.loss}') # Model architecture plots, metrics plots
    os.mkdir(f'generated_samples/{job_id}_{args.type_gen}_{args.type_disc}_{args.n_layers_gen}_{args.n_layers_disc}_{args.T_condition}_{args.loss}') # Generated samples
    os.mkdir(f'models/{job_id}_{args.type_gen}_{args.type_disc}_{args.n_layers_gen}_{args.n_layers_disc}_{args.T_condition}_{args.loss}') # Models

    # Create the orderbook dataframe
    orderbook_df = create_orderbook_dataframe(N, previos_days=False)

    # Define the parameters of the GAN. Some of them are set via argparse
    T_condition = args.T_condition
    T_gen = args.T_gen
    window_size = T_condition + T_gen
    n_features_input = orderbook_df.shape[1]
    n_features_gen = 2*depth
    latent_dim = args.latent_dim
    n_epochs = 5000
    batch_size = args.batch_size

    # Define the parameters for the early stopping criterion
    best_gen_weights = None
    best_disc_weights = None
    best_wass_dist = float('inf')
    patience_counter = 0
    patience = 200

    num_pieces = 5
    if not os.path.exists(f'../data/input_train_{stock}_{window_size}_day{N}_orderbook.npy'):
        logging.info('\n[Input] ---------- PREPROCESSING ----------')

        data_input = orderbook_df.values
        # data_input = np.load(f'anomaly_data_{N}.npy')
        # logging.info(f'\nAre anomaly_data and normal_data the same?\n\t{np.all(data_input == data_input_a)}')
        # exit()
        # Divide input data into overlapping pieces
        sub_data, length = divide_into_overlapping_pieces(data_input, window_size, num_pieces)

        if sub_data[-1].shape[0] < window_size:
            raise ValueError(f'The last piece has shape {sub_data[-1].shape} and it is smaller than the window size {window_size}.')

        logging.info(f'Number of windows: {length}')

        # Create a memmap to store the scaled data.
        final_shape = (length-num_pieces*(window_size-1), window_size, n_features_input)
        fp = np.memmap("final_data.dat", dtype='float32', mode='w+', shape=final_shape)

        start_idx = 0
        logging.info(f'\nStart scaling the data...')
        for piece_idx, data in enumerate(sub_data):
            logging.info(f'\t{piece_idx+1}/{num_pieces}')
            windows = np.array(divide_into_windows(data, window_size))
            logging.info(f'\twindows shape: {windows.shape}')
            end_idx = start_idx + windows.shape[0]
            fp[start_idx:end_idx] = windows
            start_idx = end_idx
            del windows  # Explicit deletion
        logging.info('Done.')

        np.save(f'normal_data_{N}.npy', fp)

        logging.info('\nDividing each window into condition and input...')
        condition_train, input_train = fp[:, :T_condition, :], fp[:, T_condition:, :n_features_gen]
        logging.info('Done.')

        logging.info(f'input_train shape:\n\t{input_train.shape}')
        logging.info(f'condition_train shape:\n\t{condition_train.shape}')

        logging.info('\nSave the files...')
        np.save(f'../data/condition_train_{stock}_{window_size}_day{N}_orderbook.npy', condition_train)
        np.save(f'../data/input_train_{stock}_{window_size}_day{N}_orderbook.npy', input_train)
        logging.info('Done.')

        logging.info('\n[Input] ---------- DONE ----------')
    else:
        logging.info('Loading input_train, input_validation and input_test sets...')
        input_train = np.load(f'../data/input_train_{stock}_{window_size}_{N}days_orderbook.npy', mmap_mode='r')
        condition_train = np.load(f'../data/condition_train_{stock}_{window_size}_{N}days_orderbook.npy', mmap_mode='r')

        logging.info(f'input_train shape:\n\t{input_train.shape}')
        logging.info(f'condition_train shape:\n\t{condition_train.shape}')

    logging.info(f"\nHYPERPARAMETERS:\n"
                    f"\tstock: {stock}\n"
                    f"\tdepth: {depth}\n"
                    f"\tgenerator: {args.type_gen}\n"
                    f"\tdiscriminator: {args.type_disc}\n"
                    f"\tn_layers_gen: {args.n_layers_gen}\n"
                    f"\tn_layers_disc: {args.n_layers_disc}\n"
                    f"\tskip_connection: {args.skip_connection}\n"
                    f"\tlatent_dim per time: {latent_dim}\n"
                    f"\tn_features_input: {n_features_input}\n"
                    f"\tn_features_gen: {n_features_gen}\n"
                    f"\tfeatures: {orderbook_df.columns}\n"
                    f"\tn_epochs: {n_epochs}\n"
                    f"\tT_condition: {T_condition}\n"
                    f"\tT_gen: {T_gen}\n"
                    f"\tbatch_size: {batch_size} (num_batches: {input_train.shape[0]//batch_size})\n"
                    f"\tloss: {args.loss}\n"
                    f"\tpatience: {patience}\n"
                    f"\tjob_id: {job_id}\n"
                    f"\tLoaded model: {None if args.load==0 else args.load}\n")

    # Define the optimizers
    generator_optimizer = tf.keras.optimizers.Adam(learning_rate=0.00001)
    discriminator_optimizer = tf.keras.optimizers.Adam(learning_rate=0.00001)
    optimizer = [generator_optimizer, discriminator_optimizer]

    if args.load == 0:
        # Build the models
        generator_model = build_generator(args.n_layers_gen, args.type_gen, args.skip_connection, T_gen, T_condition, n_features_input, n_features_gen, latent_dim, True)
        discriminator_model = build_discriminator(args.n_layers_disc, args.type_disc, args.skip_connection, T_gen, T_condition, n_features_input, n_features_gen, True, args.loss)
        feature_extractor = build_feature_extractor(discriminator_model, [i for i in range(1, args.n_layers_disc)])
    else:
        prev_job_id = args.load
        # Load the models
        generator_model = tf.keras.models.load_model(f'models/{prev_job_id}.pbs01_{args.type_gen}_{args.type_disc}_{args.n_layers_gen}_{args.n_layers_disc}_{args.T_condition}_{args.loss}/generator_model.h5')
        discriminator_model = tf.keras.models.load_model(f'models/{prev_job_id}.pbs01_{args.type_gen}_{args.type_disc}_{args.n_layers_gen}_{args.n_layers_disc}_{args.T_condition}_{args.loss}/discriminator_model.h5')
        feature_extractor = build_feature_extractor(discriminator_model, [i for i in range(1, args.n_layers_disc)])

    logging.info('\n[Model] ---------- MODEL SUMMARIES ----------')
    generator_model.summary(print_fn=logging.info)
    logging.info('\n')
    discriminator_model.summary(print_fn=logging.info)
    logging.info('[Model] ---------- DONE ----------\n')

    # Define a dictionary to store the metrics
    metrics = {'discriminator_loss': [], 'gen_loss': [], 'real_disc_out': [], 'fake_disc_out': []}

    # Train the GAN.
    logging.info('\n[Training] ---------- START TRAINING ----------')
    dataset_train = tf.data.Dataset.from_tensor_slices((condition_train, input_train)).batch(batch_size)

    num_batches = len(dataset_train)
    logging.info(f'Number of batches:\n\t{num_batches}\n')

    # Initialize a list to store the mean over all the features of the wasserstein distances at each epoch
    wass_to_plot = []
    for epoch in range(n_epochs):
        j = 0
        W_batch = [] # W_batch will have num_batches elements
        noises = [[] for _ in range(num_batches)] # noises will have num_batches elements. Each elements is a list containing the noises used for each batch in that epoch
        for batch_condition, batch_real_samples in dataset_train:
            j += 1
            batch_size = batch_real_samples.shape[0]
            generator_model, discriminator_model, generated_samples, noise = train_step(batch_real_samples, batch_condition, generator_model, discriminator_model, feature_extractor, optimizer, args.loss, T_gen, T_condition, latent_dim, batch_size, num_batches, j, job_id, epoch, metrics, args)
            # Append the noise
            noises[j-1] = noise
            W_features = [] # W_features will have n_features_gen elements
            for feature in range(n_features_gen): # Iteration over the features
                W_samples = [] # W_samples will have batch_size elements
                for i in range(generated_samples.shape[0]): # Iteration over the samples
                    w = wasserstein_distance(batch_real_samples[i, :, feature], generated_samples[i, :, feature])
                    W_samples.append(w)
                W_features.append(np.mean(np.array(W_samples))) # averaged over the samples in a batch
            W_batch.append(np.mean(np.array(W_features))) # averaged over the features
        overall_W_mean = np.mean(np.array(W_batch)) # averaged over the batches
        wass_to_plot.append(overall_W_mean)
        logging.info(f'Wasserstein distance: {overall_W_mean}')

        if epoch % 150 == 0:
            logging.info('Creating a time series with the generated samples...')
            features = orderbook_df.columns[:n_features_gen]
            plot_samples(dataset_train, generator_model, noises, features, T_gen, n_features_gen, job_id, epoch, args)

            logging.info('Saving the models...')
            generator_model.save(f'models/{job_id}_{args.type_gen}_{args.type_disc}_{args.n_layers_gen}_{args.n_layers_disc}_{args.T_condition}_{args.loss}/generator_model.h5')
            discriminator_model.save(f'models/{job_id}_{args.type_gen}_{args.type_disc}_{args.n_layers_gen}_{args.n_layers_disc}_{args.T_condition}_{args.loss}/discriminator_model.h5')
            logging.info('Done')

        logging.info('Check Early Stopping Criteria...')
        if epoch > 2500:
            if overall_W_mean + 5e-4 < best_wass_dist:
                logging.info(f'Wasserstein distance improved from {best_wass_dist} to {overall_W_mean}')
                best_wass_dist = overall_W_mean
                best_gen_weights = generator_model.get_weights()
                best_disc_weights = discriminator_model.get_weights()
                patience_counter = 0
                np.save(f'generated_samples/{job_id}_{args.type_gen}_{args.type_disc}_{args.n_layers_gen}_{args.n_layers_disc}_{args.T_condition}_{args.loss}/noise_{epoch}.npy', noises)
            else:
                logging.info(f'Wasserstein distance did not improve from {best_wass_dist}')
                patience_counter += 1

            if patience_counter >= patience:
                best_epoch = epoch - patience
                logging.info(f"Early stopping on epoch {epoch}. Restoring best weights of epoch {best_epoch}...")
                generator_model.set_weights(best_gen_weights)  # restore best weights
                discriminator_model.set_weights(best_disc_weights)

                logging.info('Saving the models...')
                generator_model.save(f'models/{job_id}_{args.type_gen}_{args.type_disc}_{args.n_layers_gen}_{args.n_layers_disc}_{args.T_condition}_{args.loss}/generator_model.h5')
                discriminator_model.save(f'models/{job_id}_{args.type_gen}_{args.type_disc}_{args.n_layers_gen}_{args.n_layers_disc}_{args.T_condition}_{args.loss}/discriminator_model.h5')
                logging.info('Done')
            else:
                logging.info(f'Early stopping criterion not met. Patience counter:\n\t{patience_counter}')

        # Plot the wasserstein distance
        plt.figure(figsize=(10, 6))
        plt.plot(wass_to_plot)
        plt.xlabel('Epoch')
        plt.ylabel('Wasserstein distance')
        plt.title(f'Mean over the features of the Wasserstein distances')
        # add a vertical line at the best epoch
        plt.axvline(x=epoch-patience_counter, color='r', linestyle='--', alpha=0.8, label=f'Best epoch: {epoch-patience_counter}')
        plt.legend()
        plt.savefig(f'plots/{job_id}_{args.type_gen}_{args.type_disc}_{args.n_layers_gen}_{args.n_layers_disc}_{args.T_condition}_{args.loss}/0_wasserstein_distance.png')
        plt.close()
        if patience_counter >= patience:
            break

    logging.info('[Training] ---------- DONE ----------\n')

    logging.info('Plotting the first 2 principal components of the generated and real samples...')
    # Plot the first 2 principal components of the generated and real samples
    # Load the best generator
    generator_model = tf.keras.models.load_model(f'models/{job_id}_{args.type_gen}_{args.type_disc}_{args.n_layers_gen}_{args.n_layers_disc}_{args.T_condition}_{args.loss}/generator_model.h5')

    generated_samples = []
    real_samples = []
    k = 0
    for batch_condition, batch in dataset_train:
        gen_sample = generator_model([noises[k], batch_condition])
        for i in range(gen_sample.shape[0]):
            # All the appended samples will be of shape (T_gen, n_features_gen)
            generated_samples.append(gen_sample[i, -1, :])
            real_samples.append(batch[i, -1, :])
        k += 1
    plot_pca_with_marginals(generated_samples, real_samples, job_id, args)
    logging.info('Done.')

    logging.info('Computing the errors on the correlation matrix using bootstrap...')
    # At the end of the training, compute the errors on the correlation matrix using bootstrap.
    # In order to do so, I need the best generator and the noises used.
    correlation_matrix(dataset_train, generator_model, noises, T_gen, n_features_gen, job_id)
    logging.info('Done.')
    # Maybe it is not necessary, but I prefer to clear all the memory and exit the script
    gc.collect()
    tf.keras.backend.clear_session()
    sys.exit()


# LinearlyVariableInfill
"""
Linearly Variable Infill for 3D prints.

Author: Barnabas Nemeth
Version: 1.5

"""

from ..Script import Script
from UM.Logger import Logger
from UM.Application import Application
import re #To perform the search
from cura.Settings.ExtruderManager import ExtruderManager
from collections import namedtuple
from enum import Enum
from typing import List, Tuple
from UM.Message import Message
from UM.i18n import i18nCatalog
catalog = i18nCatalog("cura")

__version__ = '1.5'

##-----------------------------------------------------------------------------------------------------------------------------------------------------------------

Point2D = namedtuple('Point2D', 'x y')
Segment = namedtuple('Segment', 'point1 point2')

class Infill(Enum):
    """Enum for infill type."""

    LINEAR = 1  # Linear infill like rectilinear or triangles

class Section(Enum):
    """Enum for section type."""

    NOTHING = 0
    INNER_WALL = 1
    OUTER_WALL = 2
    INFILL = 3

def dist(segment: Segment, point: Point2D) -> float:
    """Calculate the distance from a point to a line with finite length.

    Args:
        segment (Segment): line used for distance calculation
        point (Point2D): point used for distance calculation

    Returns:
        float: distance between ``segment`` and ``point``
    """
    px = segment.point2.x - segment.point1.x
    py = segment.point2.y - segment.point1.y
    norm = px * px + py * py
    u = ((point.x - segment.point1.x) * px + (point.y - segment.point1.y) * py) / float(norm)
    if u > 1:
        u = 1
    elif u < 0:
        u = 0
    x = segment.point1.x + u * px
    y = segment.point1.y + u * py
    dx = x - point.x
    dy = y - point.y

    return (dx * dx + dy * dy) ** 0.5

def two_points_distance(point1: Point2D, point2: Point2D) -> float:
    """Calculate the euclidean distance between two points.

    Args:
        point1 (Point2D): first point
        point2 (Point2D): second point

    Returns:
        float: euclidean distance between the points
    """
    return ((point1.x - point2.x) ** 2 + (point1.y - point2.y) ** 2) ** 0.5

def min_distance_to_segment(segment: Segment, segments: List[Segment]) -> float:
    """Calculate the minimum distance from the midpoint of ``segment`` to the nearest segment in ``segments``.

    Args:
        segment (Segment): segment to use for midpoint calculation
        segments (List[Segment]): segments list

    Returns:
        float: the smallest distance from the midpoint of ``segment`` to the nearest segment in the list
    """
    middlePoint = Point2D((segment.point1.x + segment.point2.x) / 2, (segment.point1.y + segment.point2.y) / 2)

    return min(dist(s, middlePoint) for s in segments)

def getXY(currentLineINcode: str) -> Point2D:
    """Create a ``Point2D`` object from a gcode line.

    Args:
        currentLineINcode (str): gcode line

    Raises:
        SyntaxError: when the regular expressions cannot find the relevant coordinates in the gcode

    Returns:
        Point2D: the parsed coordinates
    """
    searchX = re.search(r"X(\d*\.?\d*)", currentLineINcode)
    searchY = re.search(r"Y(\d*\.?\d*)", currentLineINcode)
    if searchX and searchY:
        elementX = searchX.group(1)
        elementY = searchY.group(1)
    else:
        raise SyntaxError('Gcode file parsing error for line {currentLineINcode}')

    return Point2D(float(elementX), float(elementY))

def mapRange(a: Tuple[float, float], b: Tuple[float, float], s: float) -> float:
    """Calculate a multiplier for the extrusion value from the distance to the perimeter.

    Args:
        a (Tuple[float, float]): a tuple containing:
            - a1 (float): the minimum distance to the perimeter (always zero at the moment)
            - a2 (float): the maximum distance to the perimeter where the interpolation is performed
        b (Tuple[float, float]): a tuple containing:
            - b1 (float): the maximum flow as a fraction
            - b2 (float): the minimum flow as a fraction
        s (float): the euclidean distance from the middle of a segment to the nearest perimeter

    Returns:
        float: a multiplier for the modified extrusion value
    """
    (a1, a2), (b1, b2) = a, b

    return b1 + ((s - a1) * (b2 - b1) / (a2 - a1))

def gcode_template(x: float, y: float, extrusion: float) -> str:
    """Format a gcode string from the X, Y coordinates and extrusion value.

    Args:
        x (float): X coordinate
        y (float): Y coordinate
        extrusion (float): Extrusion value

    Returns:
        str: Gcode line
    """
    return "G1 X{} Y{} E{}".format(round(x, 3), round(y, 3), round(extrusion, 5))

def is_layer(line: str) -> bool:
    """Check if current line is the start of a layer section.

    Args:
        line (str): Gcode line

    Returns:
        bool: True if the line is the start of a layer section
    """
    return line.startswith(";LAYER:")

def is_innerwall(line: str) -> bool:
    """Check if current line is the start of an inner wall section.

    Args:
        line (str): Gcode line

    Returns:
        bool: True if the line is the start of an inner wall section
    """
    return line.startswith(";TYPE:WALL-INNER")

def is_outerwall(line: str) -> bool:
    """Check if current line is the start of an outer wall section.

    Args:
        line (str): Gcode line

    Returns:
        bool: True if the line is the start of an outer wall section
    """
    return line.startswith(";TYPE:WALL-OUTER")

def ez_nyomtatasi_vonal(line: str) -> bool:
    """Check if current line is a standard printing segment.

    Args:
        line (str): Gcode line

    Returns:
        bool: True if the line is a standard printing segment
    """
    return "G1" in line and " X" in line and "Y" in line and "E" in line

def is_infill(line: str) -> bool:
    """Check if current line is the start of an infill.

    Args:
        line (str): Gcode line

    Returns:
        bool: True if the line is the start of an infill section
    """
    return line.startswith(";TYPE:FILL")

def fill_type(Mode):
    """Definie the type of Infill pattern

       Linearly Variable Infill like lineas or triangles = 1

    Args:
        line (Mode): Infill Pattern

    Returns:
        Int: the Type of infill pattern
    """
    iMode=0
    if Mode == 'grid':
        iMode=1
    if Mode == 'lines':
        iMode=1
    if Mode == 'triangles':
        iMode=1
    if Mode == 'trihexagon':
        iMode=1
    if Mode == 'cubic':
        iMode=1
    if Mode == 'cubicsubdiv':
        iMode=0
    if Mode == 'tetrahedral':
        iMode=1
    if Mode == 'quarter_cubic':
        iMode=1
    if Mode == 'concentric':
        iMode=0
    if Mode == 'zigzag':
        iMode=0
    if Mode == 'cross':
        iMode=0
    if Mode == 'cross_3d':
        iMode=0
    if Mode == 'gyroid':
        iMode=0

    return iMode

class LinearlyVariableInfill(Script):
    def getSettingDataString(self):
        return """{
            "name": "Linearly Variable Infill",
            "key": "LinearlyVariableInfill",
            "metadata": {},
            "version": 2,
            "settings":
            {
                "variableSegmentLength":
                {
                    "label": "Valtoztatott szakasz hossza",
                    "description": "Distance of the gradient (max to min) in mm",
                    "unit": "mm",
                    "type": "float",
                    "default_value": 6.0,
                    "minimum_value": 1.0,
                    "minimum_value_warning": 2.0
                },
                "divisionNR":
                {
                    "label": "Szakasz felosztasanak szama",
                    "description": "Only applicable for Linearly Variable Infills; number of segments within the gradient(fullSegmentLength=variableSegmentLength / divisionNR); use sensible values to not overload",
                    "type": "int",
                    "default_value": 4,
                    "minimum_value": 1,
                    "minimum_value_warning": 2
                },

                "variableSpeed":
                {
                    "label": "Valtozo sebesseg",
                    "description": "Activate also Valtozo sebesseg linked to the gradual flow",
                    "type": "bool",
                    "default_value": false
                },
                "maxSpeedFactor":
                {
                    "label": "Max sebesseg szorzo",
                    "description": "Maximum over speed factor",
                    "unit": "%",
                    "type": "int",
                    "default_value": 200,
                    "minimum_value": 100,
                    "maximum_value": 400,
                    "minimum_value_warning": 110,
                    "maximum_value_warning": 370,
                    "enabled": "variableSpeed"

                },
                "minSpeedFactor":
                {
                    "label": "Min sebesseg szorzo",
                    "description": "Minimum over speed factor",
                    "unit": "%",
                    "type": "int",
                    "default_value": 60,
                    "minimum_value": 10,
                    "maximum_value": 100,
                    "minimum_value_warning": 40,
                    "maximum_value_warning": 90,
                    "enabled": "variableSpeed"

                },
                "extruderNR":
                {
                    "label": "Extruder sorszam",
                    "description": "Define Extruder szam in case of multi extruders",
                    "unit": "",
                    "type": "int",
                    "default_value": 1

                }
            }
        }"""

## -----------------------------------------------------------------------------
#
#  Main Prog
#
## -----------------------------------------------------------------------------

    def execute(self, data):
        Logger.log('w', 'Plugin is starting '  )
        print('naygvera')
        division_nr = float(self.getSettingValueByKey("divisionNR"))
        variable_segment_lengh = float(self.getSettingValueByKey("variableSegmentLength"))
        extruder_nr  = self.getSettingValueByKey("extruderNR")
        extruder_nr = extruder_nr -1
        variable_speed= bool(self.getSettingValueByKey("variableSpeed"))
        max_speed_factor = float(self.getSettingValueByKey("maxSpeedFactor"))
        max_speed_factor = max_speed_factor /100
        min_speed_factor = float(self.getSettingValueByKey("minSpeedFactor"))
        min_speed_factor = min_speed_factor /100

        #   machine_extruder_count
     #   extruder_count=Application.getInstance().getGlobalContainerStack().getProperty("machine_extruder_count", "value")
      #  extruder_count = extruder_count-1
       # if extruder_nr>extruder_count :
        #    extruder_nr=extruder_count

        # Deprecation function
        extrud = list(Application.getInstance().getGlobalContainerStack().extruders.values())
        #extrud = Application.getInstance().getGlobalContainerStack().extruderList
        Message('Extrud:{}'.format(extrud), title = catalog.i18nc("@info:title", "Post Processing")).show()
        infillpattern = extrud[extruder_nr].getProperty("infill_pattern", "value")
        connectinfill = extrud[extruder_nr].getProperty("zig_zaggify_infill", "value")

        """Parse Gcode and modify infill portions with an extrusion width gradient."""
        currentSection = Section.NOTHING
        lastPosition = Point2D(-10000, -10000)
        littleSegmentLength = variable_segment_lengh / division_nr

        infill_type=fill_type(infillpattern)
        if infill_type == 0:
            #
            Logger.log('d', 'Infill Pattern not supported : ' + infillpattern)
            Message('Infill Pattern not supported : ' + infillpattern , title = catalog.i18nc("@info:title", "Post Processing")).show()

            return None

        if connectinfill == True:
            #
            Logger.log('d', 'Connect Infill Lines no supported')
            Message('Gcode must be generate without Connect Infill Lines mode activated' , title = catalog.i18nc("@info:title", "Post Processing")).show()
            return None

        Logger.log('d',  "GradientFill Param : " + str(littleSegmentLength) + "/" + str(division_nr)+ "/" + str(variable_segment_lengh) ) #str(max_flow) + "/" + str(min_flow) + "/" +
        Logger.log('d',  "Pattern Param : " + infillpattern + "/" + str(infill_type) )

        for layer in data:
            layer_index = data.index(layer)
            lines = layer.split("\n")
            for currentLineINcode in lines:
                new_Line=""
                stringFeed = ""
                line_index = lines.index(currentLineINcode)

                if is_layer(currentLineINcode):
                    perimeterSegments = []

                if is_innerwall(currentLineINcode):
                    currentSection = Section.INNER_WALL
                    # Logger.log('d', 'is_innerwall'  )

                if is_outerwall(currentLineINcode):
                    currentSection = Section.OUTER_WALL
                    # Logger.log('d', 'is_outerwall' )

                if currentSection == Section.INNER_WALL:
                    if ez_nyomtatasi_vonal(currentLineINcode):
                        Logger.log('d', 'Ez sor rossz ' + currentLineINcode)
                        perimeterSegments.append(Segment(getXY(currentLineINcode), lastPosition))

                if is_infill(currentLineINcode):
                    # Log Size of perimeterSegments for debuging
                    Logger.log('d', 'PerimeterSegments seg : {}'.format(len(perimeterSegments)))
                    currentSection = Section.INFILL
                    # ! Important
                    continue

                if currentSection == Section.INFILL:
                    if "F" in currentLineINcode and "G1" in currentLineINcode:
                        searchSpeed = re.search(r"F(\d*\.?\d*)", currentLineINcode)

                        if searchSpeed:
                            current_speed=float(searchSpeed.group(1))
                            new_Line="G1 F{}\n".format(current_speed)
                        else:
                            Logger.log('d', 'Gcode file parsing error for line : ' + currentLineINcode )

                    if "E" in currentLineINcode and "G1" in currentLineINcode and "X" in currentLineINcode and "Y" in currentLineINcode:
                        currentPosition = getXY(currentLineINcode)
                        splitLine = currentLineINcode.split(" ")

                        # ha lineraris
                        if infill_type == 1:
                            for element in splitLine:
                                if "E" in element:
                                    E_inCode = float(element[1:])

                            fullSegmentLength = two_points_distance(lastPosition, currentPosition)
                            segmentSteps = fullSegmentLength / littleSegmentLength
                            extrudeLengthPERsegment = (0.006584 * fullSegmentLength) / segmentSteps
                            E_inCode_last = E_inCode - (extrudeLengthPERsegment * segmentSteps)
                            littlesegmentDirectionandLength = Point2D((currentPosition.x - lastPosition.x) / fullSegmentLength * littleSegmentLength,(currentPosition.y - lastPosition.y) / fullSegmentLength * littleSegmentLength)
                            speed_deficit = ((current_speed * max_speed_factor + current_speed * min_speed_factor) / division_nr)
                            step_number = 0
                            last_step_number = 0

                            if segmentSteps >= 2:
                                # new_Line=new_Line+"; LinearlyVariableInfill segmentSteps >= 2\n"
                                for step in range(int(segmentSteps)):
                                    segmentEnd = Point2D(lastPosition.x + littlesegmentDirectionandLength.x, lastPosition.y + littlesegmentDirectionandLength.y)
                                    extrudeLength=E_inCode_last+extrudeLengthPERsegment
                                    if perimeterSegments==[] :
                                        Logger.log('d', 'Itt a hiba ' + currentLineINcode)
                                    shortestDistance = min_distance_to_segment(Segment(lastPosition, segmentEnd), perimeterSegments)
                                    if shortestDistance < variable_segment_lengh:
                                        segmentSpeed = current_speed

                                        if variable_speed:
                                            if variable_speed:

                                             if step_number < division_nr:
                                                 segmentSpeed = current_speed * min_speed_factor + (speed_deficit * step_number)

                                             if step_number >= division_nr:
                                                 segmentSpeed = current_speed * max_speed_factor

                                             if step_number >= segmentSteps - division_nr:
                                                 segmentSpeed = current_speed * max_speed_factor - (speed_deficit * last_step_number)
                                                 last_step_number=last_step_number + 1
                                            stringFeed = " F{}".format(int(segmentSpeed))

                                    else:
                                         segmentSpeed = current_speed * min_speed_factor

                                         if variable_speed:
                                            if step_number < division_nr:
                                                 segmentSpeed = current_speed * min_speed_factor + (speed_deficit * step_number)

                                            if step_number >= division_nr:
                                                 segmentSpeed = current_speed * max_speed_factor

                                            if step_number >= segmentSteps - division_nr:
                                                 segmentSpeed = current_speed * max_speed_factor - (speed_deficit * last_step_number)
                                                 last_step_number=last_step_number + 1

                                            stringFeed = " F{}".format(int(segmentSpeed))

                                    new_Line=new_Line + gcode_template(segmentEnd.x, segmentEnd.y, extrudeLength) + stringFeed + "\n" #szakaszExtrudalas
                                    lastPosition  = segmentEnd
                                    E_inCode_last = extrudeLength
                                    step_number = step_number + 1

                                segmentSpeed = current_speed * min_speed_factor
                                lastSpeed = " F{}".format(int(segmentSpeed))
                                new_Line=new_Line + gcode_template(currentPosition.x, currentPosition.y, E_inCode, ) + lastSpeed + "\n" #Original line for finish
                                lines[line_index] = new_Line

                            else :
                                outPutLine = ""
                                # outPutLine = "; LinearlyVariableInfill segmentSteps < 2\n"

                                for element in splitLine:
                                    if "E" in element:
                                        outPutLine = outPutLine + "E" + str(round(E_inCode, 5))
                                    else:
                                        outPutLine = outPutLine + element + " "
                                outPutLine = outPutLine # + "\n"
                                lines[line_index] = outPutLine

                            # writtenToFile = 1

                    #
                    # comment like ;MESH:NONMESH
                    #
                    if ";" in currentLineINcode:
                        currentSection = Section.NOTHING
                        lines[line_index] = currentLineINcode # other Comment
                #
                # line with move
                #
                if "X" in currentLineINcode and "Y" in currentLineINcode and ("G1" in currentLineINcode or "G0" in currentLineINcode):
                    lastPosition  = getXY(currentLineINcode)

            final_lines = "\n".join(lines)
            data[layer_index] = final_lines
        return data


import pytest

from pytest_persistence import plugin

plg = plugin.Plugin()

@pytest.mark.parametrize("scope", ["session", "package", "module", "class", "function"])
@pytest.mark.parametrize("result", ["result", 42])
def test_store_fixture(result, scope):
    fixture_id = ('fixture1', scope, 'tests/test_mock.py')
    plg.store_fixture(result, fixture_id, 'tests/test_mock.py', None)
    if scope == "session":
        assert plg.output[scope] == {"('fixture1', 'session', 'tests/test_mock.py', None)": result}
    else:
        assert plg.output[scope]["tests/test_mock.py"] == {
            f"('fixture1', '{scope}', 'tests/test_mock.py', None)": result}

@pytest.fixture(params=[(x, y)
                        for x in ["session", "package", "module", "class", "function"]
                        for y in ["result", 42]])
def store_fixtures(request):
    scope = request.param[0]
    result = request.param[1]
    fixture_id = ('fixture1', scope, 'tests/test_mock.py')
    plg.store_fixture(result, fixture_id, 'tests/test_mock.py', None)
    plg.input = plg.output
    return scope, result

def test_load_fixture(store_fixtures):
    scope = store_fixtures[0]
    result = store_fixtures[1]
    fixture_id = ('fixture1', scope, 'tests/test_mock.py')
    fixture_result = plg.load_fixture(fixture_id, 'tests/test_mock.py')
    assert fixture_result == result


"""
To render html web pages
"""
import random
from django.http import HttpResponse

from django.template.loader import render_to_string

from articles.models import Article

def home_view(request, id=None, *args, **kwargs):
    """
    Take in a request (Django send request)
    return HTML as a response
    (We pick to return the response)
    """
    name = "Artem" # hard coded
    number = random.randint(1, 2) # pseudo random

    article_obj = Article.objects.get(id=number)
    article_queryset = Article.objects.all()

    context = {
        "object_list": article_queryset,
        "title": article_obj.title,
        "content": article_obj.content,
        "id": article_obj.id
    }

    # Django templates
    HTML_STRING =  render_to_string("home-view.html", context=context)
    return HttpResponse(HTML_STRING)


#!/usr/bin/python

import curses
import sys
import RPi.GPIO as GPIO

def main(stdscr):
    # do not wait for input when calling getch
    stdscr.nodelay(1)

    initGPIO()

    while True:
        # get keyboard input, returns -1 if none available
        c = stdscr.getch()
        if c != -1:
            # print numeric value
            stdscr.addstr(str(c))
            stdscr.refresh()
            # return curser to start position
            stdscr.move(0, 0)
        if c == 97:
            GPIO.cleanup()
            sys.exit('Bye bye')
        elif c == 117:
            lightToggle()
        elif c == 114:
            light()

def initGPIO():
    GPIO.setmode(GPIO.BCM)
    GPIO.setup(8, GPIO.OUT)

def lightToggle():
    GPIO.output(8, not GPIO.input(8))

def light():
    GPIO.output(8, GPIO.HIGH)

def dark():
    GPIO.output(8, GPIO.LOW)

if __name__ == '__main__':
    curses.wrapper(main)


"""Celery를 사용하는 예제"""
import random
import time
from os import path
from urllib import parse
import requests
from celery import Celery
from pydub import AudioSegment
from my_logging import get_my_logger

logger = get_my_logger(__name__)

# 크롤링 요청 간격 리스트 정의
RANDOM_SLEEP_TIMES = [x * 0.1 for x in range(10, 40, 5)]

# 아티스트 이름
ARTIST_NAME = "Maurice RAVEL "

# 앨범 타이틀
ALBUM_NAME = "The Piano Music of Maurice Ravel from archive.org"

# 크롤링 대상 URL 리스트
MUSIC_URLS = [
    'https://archive.org/download/ThePianoMusicOfMauriceRavel/01PavanePourUneInfanteDfuntePourPianoMr19.mp3',
    'https://archive.org/download/ThePianoMusicOfMauriceRavel/02JeuxDeauPourPianoMr30.mp3',
    'https://archive.org/download/ThePianoMusicOfMauriceRavel/03SonatinePourPianoMr40-Modr.mp3',
    'https://archive.org/download/ThePianoMusicOfMauriceRavel/04MouvementDeMenuet.mp3',
    'https://archive.org/download/ThePianoMusicOfMauriceRavel/05Anim.mp3',
]

# Reids의 0번째 DB를 사용하는 예
app = Celery('crawler_with_celery_sample', broker='redis://localhost:6379/0')
app.conf.update(
    # Redis에 태스크 또는 실행 결과를 저장할 때의 형식을 JSON으로 지정
    task_serializer='json',
    accept_content=['json'],
    result_serializer='json',
    timezone='Asia/Seoul',
    enable_utc=True,  # Celery 태스크 내부의 시간을 UTC로 다룸
    # 1개의 워커는 동시에 1개의 프로세스만 실행하게 함
    worker_max_tasks_per_child=1,
    # Redis에 저장되는 태스크의 실행 결과는 60초가 지나면 제거(파기)함
    result_expires=60,
    # 워커가 표준 출력으로 출력한 내용을 명령어 실행 터미널에 출력하지 않음
    worker_redirect_stdouts=False,
    # 태스크 실행 시간이 180초를 넘으면 자동으로 종료함
    task_soft_time_limit=180,

    # 어떤 태스크를 어떤 워커로 라우팅할지 설정함
    task_routes={
        'crawler_with_celery_sample.download': {
            'queue': 'download',
            'routing_key': 'download',
        },
        'crawler_with_celery_sample.cut_mp3': {
            'queue': 'media',
            'routing_key': 'media',
        },
    },
)

# 재시도는 최대 2회까지, 재시도할 때는 10초 간격을 둠
@app.task(bind=True, max_retries=2, default_retry_delay=10)
def download(self, url, timeout=180):
    """파일 내려받기"""
    try:
        # mp3 파일 이름을 URL을 기반으로 추출
        parsed_url = parse.urlparse(url)
        file_name = path.basename(parsed_url.path)

        # 요청 간격을 랜덤하게 선택
        sleep_time = random.choice(RANDOM_SLEEP_TIMES)

        # 내려받기 시작을 로그에 출력
        logger.info("[download start] sleep: {time} {file_name}".format(time=sleep_time, file_name=file_name))

        # 요청 대기
        time.sleep(sleep_time)

        # 음악 파일 내려받기
        r = requests.get(url, timeout=timeout)
        with open(file_name, 'wb') as fw:
            fw.write(r.content)

        # 내려받기 종료를 로그에 출력
        logger.info("[download finished] {file_name}".format(file_name=file_name))
        cut_mp3.delay(file_name)  # cut_mp3 함수 실행을 태스크로 큐에 넣음

    except requests.exceptions.RequestException as e:
        # 예외가 발생하면 로그를 출력하고 재시도
        logger.error("[download error - retry] file: {file_name}, e: {e}".format( file_name=file_name, e=e))
        raise self.retry(exc=e, url=url)

@app.task
def cut_mp3(file_name):
    """앞의 2초를 추출해서 저장하기"""
    logger.info("[cut_mp3 start] {file_name}".format(file_name=file_name))
    # 내려받은 파일을 pydub 데이터 형식으로 변환해서 읽어 들임
    music = AudioSegment.from_mp3(file_name)

    # mp3 파일의 앞 2초만 잘라내기
    head_time = 2 * 1000  # milliseconds
    head_part = music[:head_time]  # 잘라냄
    root_name, ext = path.splitext(file_name)  # 파일 이름을 확장자와 이외의 부분으로 분할

    # 저장
    # 원래 파일과 구별할 수 있게 확장자 이름 앞에 _head를 붙임
    file_handler = head_part.export(
        root_name + "_head" + ext,
        format="mp3",
        tags={
            'title': root_name,
            'artist': ARTIST_NAME,
            'album': ALBUM_NAME,
        }
    )
    # 주의: 파일 핸들러 닫기를 잊으면 안 됩니다.
    file_handler.close()
    logger.info("[cut_mp3 finished] {file_name}".format(file_name=file_name))

if __name__ == '__main__':
    logger.info("[main start]")
    # 크롤링 대상 URL 별로 download() 함수를 태스크로 큐에 넣음
    # 큐에 들어간 태스크는 워커에 의해서 자동 실행됨
    for music_url in MUSIC_URLS:
        download.delay(music_url)
    logger.info("[main finished]")


# !/usr/bin/python
import os
import sys

# Logging configuration
import logging

class logger(logging.Logger):

    def __init__(self):
        """Initializer."""
        super().__init__()
        logging.basicConfig(filename="errlog.log",
                    filemode="a",
                    format="(%(asctime)s)  | %(name)s | %(levelname)s:%(message)s",
                    datefmt="%d  %B  %Y , %H:%M:%S",
                    level=os.environ.get("LOGLEVEL", "INFO"))