from PIL import Image
import base64
import zlib
import StringIO
import io

# Run:		python compressionTest.py
# Creates 6 files for file size comparison:
#	1. The byte array obtained via io
#	2. The byte array as a base64 encoded string
#	3. The compressed byte array
#	4. The compressed base64 string byte array
#	5. The decompressed byte array
#	6. The decompressed base64 string byte array

try:
	# THIS IS HOW WE GET THE BYTE ARRAY WE SHOULD SAVE... 100-300 KB
	picture = Image.open("./patagonia/21911159_1940465642891993_2672871879034798080_n.jpg")

	imgByteArr = io.BytesIO()
	picture.save(imgByteArr, format='jpeg')
	imgByteArr = imgByteArr.getvalue()
	# COPY ABOVE

	#TESTING OTHER METHODS + COMPRESSION
	#Compression does not decrease file size
	picBase64 = base64.encodestring(imgByteArr)

	picBytesCompressed = zlib.compress(imgByteArr)
	picBase64Compressed = zlib.compress(picBase64)

	picBytesDecompressed = zlib.decompress(picBytesCompressed)
	picBase64Decompressed = zlib.decompress(picBase64Compressed)

	with open('file1.jpg', 'wb') as file:
		file.write(imgByteArr)
	with open('file2.jpg', 'wb') as file:
		file.write(picBase64)

	with open('file1.jpg.gz', 'wb') as file:
		file.write(picBytesCompressed)
	with open('file2.jpg.gz', 'wb') as file:
		file.write(picBase64Compressed)

	with open('file1Decom.jpg', 'wb') as file:
		file.write(picBytesDecompressed)
	with open('file2Decom.jpg', 'wb') as file:
		file.write(picBase64Decompressed)

	'''stream = StringIO.StringIO(picBytes)
	stream.write(picBytes)
	with open('file1pic.jpg', 'wb') as file:
		file.write(stream.getvalue())'''

except IOError:
	print("error")


from sostrades_core.sos_processes.base_process_builder import BaseProcessBuilder

class ProcessBuilder(BaseProcessBuilder):

    # ontology information
    _ontology_data = {
        'label': 'Test Multi Instance With Sample Generator (DriverEvaluator)',
        'description': '',
        'category': '',
        'version': '',
    }

    def get_builders(self):
        # simple 2-disc process NOT USING nested scatters coupled with a SampleGenerator

        # putting the ns_sampling in the same value as the driver will trigger
        # the coupling like in mono instance case
        self.ee.ns_manager.add_ns(
            'ns_sampling', f'{self.ee.study_name}.multi_scenarios')

        # multi scenario driver builder
        multi_scenarios = self.ee.factory.get_builder_from_process(repo='sostrades_core.sos_processes.test',
                                                                   mod_id='test_multi_instance_basic')
        # sample generator builder
        mod_cp = 'sostrades_core.execution_engine.disciplines_wrappers.sample_generator_wrapper.SampleGeneratorWrapper'
        cp_builder = self.ee.factory.get_builder_from_module('Sample_Generator', mod_cp)

        return multi_scenarios + [cp_builder]


import argparse
import logging
import os
import sys

from datasets.email_dataset import EmailDataset
from train_builder import TrainBuilder

class ExperimentEmail:

    def __init__(self):
        pass

    def run(self, train_file, val_file, out_dir, batch_size=32, epochs=10, **kwargs):
        # Set up dataset
        train_dataset = EmailDataset(train_file)
        val_dataset = EmailDataset(val_file)

        # Get trainpipeline
        builder = TrainBuilder(epochs=epochs, batch_size=batch_size, kwargs=kwargs)
        train_pipeline = builder.get_pipeline(train_dataset)

        # Start training
        train_pipeline.run(train_dataset, val_dataset, out_dir)

def list_csv_files(path, limit):
    files = [os.path.join(path, f) for f in
             filter(lambda x: os.path.isfile(os.path.join(path, x)) and x.endswith(".csv"), os.listdir(path))]
    assert len(files) == limit, "Expected exactly {} .csv file in {}, but found {}".format(limit, path, len(files))

    return files

if __name__ == '__main__':
    parser = argparse.ArgumentParser()

    parser.add_argument("--traindir",
                        help="The input train  data", default=os.environ.get("SM_CHANNEL_TRAIN", None))
    parser.add_argument("--valdir",
                        help="The input val data", default=os.environ.get("SM_CHANNEL_VAL", None))

    parser.add_argument("--outdir", help="The output dir", default=os.environ.get("SM_MODEL_DIR", None))

    parser.add_argument("--batchsize", help="The batch size", type=int, default=32)

    parser.add_argument("--epochs", help="The number of epochs", type=int, default=10)

    parser.add_argument("--log-level", help="Log level", default="INFO", choices={"INFO", "WARN", "DEBUG", "ERROR"})

    args = parser.parse_args()

    print(args.__dict__)

    # Set up logging
    logging.basicConfig(level=logging.getLevelName(args.log_level), handlers=[logging.StreamHandler(sys.stdout)],
                        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

    train_csv_file = list_csv_files(args.traindir, limit=1)[0]
    val_csv_file = list_csv_files(args.valdir, limit=1)[0]

    ExperimentEmail().run(train_csv_file,
                          val_csv_file,
                          args.outdir,
                          args.batchsize,
                          args.epochs)


import matplotlib.pyplot as plt
import pandas as pd
import argparse
import json

from importlib import reload

def load_file(input_file):
    loaded = []
    with open(input_file) as jfile:
        for jline in jfile:
            load_dict = {}
            data = json.loads(jline)
            loaded.append({'x' : data['x'], 'y' : data['y']})
    return pd.DataFrame(loaded)

def plot_histogram(df, out_file="hist.png"):
    plt.hist(x=df.sort_values(by=['x'])['x'], bins=df['x'].size, weights=df.sort_values(by=['x'])['y'])
    plt.savefig(out_file)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--input_file', type=str, help='input file')
    args = parser.parse_args()
    df = load_file(args.input_file)
    plot_histogram(df)


import threading
import time

def calculate_square(numbers):
    for number in numbers:
        time.sleep(1)
        print("Square:", number * number)

def calculate_cube(numbers):
    for number in numbers:
        time.sleep(1)
        print("Cube:", number * number * number)

numbers = list(range(1, 6))

initial_time = time.time()
thread1 = threading.Thread(target=calculate_square, args=(numbers, ))
thread2 = threading.Thread(target=calculate_cube, args=(numbers, ))

thread1.start()
thread2.start()

thread1.join()
thread2.join()

final_time = time.time()
print("Time Taken : ", final_time - initial_time)


""" A testing and experimentation enviornment.
"""
import os

from src.neat_custom.config import Config
from neat import ctrnn, nn

from src.cellGenome import CellGenome
from src.hexmap import Map
from src.hexSimulation import HexSimulation
from src.views import View

# from experiments.tree import Simulation, genome_config
# from experiments.shapes import genome_config, R as Simulation
from experiments.table import Simulation, genome_config

from src.modules.morphogen import MorphogenModule
import time
# genome_config['modules'] = [
#     (MorphogenModule, {'start_genes': 1, 'min_genes': 0})
    # (Signal3Module, {'prob_add': .0, 'prob_remove': .0,
    #                  'min_genes': 0, 'start_genes': 3, 'max_genes': 6 }),

    # (neighbors_continuous.NeighborModule, {}),
    # (divide_theta.DivideThetaModule, {}),

    # (neighbors_distinct.NeighborModule, {}),
    # (divide_distinct.DivideDistinctModule, {})
# ]

LOCAL_DIR = os.path.dirname(__file__)
CONFIG = Config(os.path.join(LOCAL_DIR, 'experiments/shape_config.txt'))
CONFIG.genome_config = genome_config
DUMMY_GENOME = CellGenome.create_unconnected(1, CONFIG)
network = nn.create_recurrent_phenotype(DUMMY_GENOME)

class Sandbox(Simulation):
    """ Extend the simualtion to inject arbitrary cell behavior. """
    def __init__(self):
        super(Sandbox, self).__init__(DUMMY_GENOME)

        for row in range(1, self.bounds[0]):
            self.create_cell((row, 0))

        for row in range(self.bounds[0]-1):
            self.create_cell((row, 1))

        for col in range(1, self.bounds[1]):
            self.create_cell((5, col))
        # for row in range(0, 4):
        #     for col in range(2, self.bounds[1]-2):
        #         self.destroy_cell(self.hmap[row][col])
        #
        #     self.create_cell((5, col))
        # self._calculate_light()
        # self.filter_unconnected()

        # self.create_cell((0, 0))
        # self.create_cell((0, 1))
        # self.create_cell((4, 0))
        # self.create_cell((4, 1))
        # self.create_cell((5, 1))
        # self.create_cell((5, 2))
        # self.create_cell((4, 2))
        # self.create_cell((3, 1))

        # self.destroy_cell(self.hmap[4][1])
        # self.create_cell((4,1))

        # self.create_cell((1, 6))

    def create_beam(self):
        for i in [self.bounds[0]-1, 0]:
            for j in range(self.bounds[1]):
                if not self.hmap[i][j]:
                    self.create_cell((i, j))

        for i in range(self.bounds[0]):
            for j in [4, 8, self.bounds[1]-1]:
                if not self.hmap[i][j]:
                    self.create_cell((i, j))

    def step(self):
        # if self.step_count > 0 and self.step_count < self.bounds[1]:
        #     self.create_cell((5, self.step_count))
        #     self.destroy_cell(self.hmap[1][2])
        super(Sandbox, self).step()
        time.sleep(1)
        # for cell in [c for c in self.cells]:
        #     for i in range(6):
        #         self.divide_cell(cell, i)
        # print(self.genome)

    # def create_inputs(self, cell):
    #     # return []
    #     inputs = super(Sandbox, self).create_inputs(cell)
    #     print inputs
    #     return inputs

    # def handle_outputs(self, cell, outputs):
        # print(cell.userData['coords'], outputs)
        # pass

    # def get_outputs(self):
    #     return []

    # def fitness(self):
    #     return 0

simulation = Sandbox()
simulation.max_steps = 2
simulation.verbose = True
view = View(600, 800, simulation)
simulation.run(view)
simulation.step()
view.hold()


#!/usr/bin/env python
# coding: utf-8

# In[1062]:

import torch
import torch.nn as nn
import torch.nn.functional as F
import pandas as pd
import numpy as np
import cv2
# Preprocessing

from sklearn.model_selection import train_test_split

from sklearn.preprocessing import LabelEncoder, StandardScaler
from torch.utils.data import Dataset, TensorDataset
import torchvision.models as models
import torchvision.transforms as transforms

# In[1063]:

class CustomTensorDataset(Dataset):
    """TensorDataset with support of transforms.
    """
    def __init__(self, tensors, transform=None):
        #assert all(tensors[0].size(0) == tensor.size(0) for tensor in tensors)
        self.tensors = tensors
        self.transform = transform

    def __getitem__(self, index):
        x = self.tensors[0][index]

        if self.transform:
            x = self.transform(x)

        y = self.tensors[1][index]

        return x, y

    def __len__(self):
        return self.tensors[0].size(0)

# In[1064]:

X = np.load('melspec_train.npy')
X = np.expand_dims(X, axis=1)
X = np.resize(X, (220, 1, 128, 216))
#X = np.resize(X, (220, 3, 128, 216))
print(X.shape)
y = np.load('y_train.npy')
scaler = StandardScaler()
X = scaler.fit_transform(X.reshape(-1, X.shape[1]*X.shape[2]*X.shape[3])).reshape(X.shape)
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2)
X_train, y_train = X, y
print(len(y_train))
print(len(y_val))
X_train.shape
X_train = torch.from_numpy(np.array(X_train)).float()
y_train = torch.from_numpy(np.array(y_train)).float()
X_val = torch.from_numpy(np.array(X_val)).float()
y_val = torch.from_numpy(np.array(y_val)).float()
X = CustomTensorDataset(tensors=(X_train, y_train))
Valid = CustomTensorDataset(tensors=(X_val, y_val))

# In[1065]:

X_test = np.load('melspec_test.npy')
X_test = np.expand_dims(X_test, axis=1)
X_test = np.resize(X_test, (15, 1, 128, 216))
#X_test = np.resize(X_test, (15, 3, 128, 216))
scaler = StandardScaler()

X_test = scaler.fit_transform(X_test.reshape(-1, X_test.shape[1]*X_test.shape[2]*X_test.shape[3])).reshape(X_test.shape)
X_test.shape
X_test = torch.from_numpy(np.array(X_test)).float()
y_test = torch.FloatTensor(X_test.shape[0])
Test = CustomTensorDataset(tensors=(X_test, y_test))

# In[1066]:

# LENet
# Model structure
cfg = [8, 16, 32, 32, 32]
#cfg = [4, 4]
class LENet(nn.Module):
    def __init__(self):
        super(LENet, self).__init__()
        self.features = self._make_layers(cfg)
        self.dropout = nn.Dropout(0.3)
        self.fc1 = nn.Linear(256, 64)
        self.fc2 = nn.Linear(64, 1)

    def forward(self, x):
        x = self.features(x)
        x = x.view(x.size(0), -1)
        #print(x.shape)
        x = self.dropout(x)
        x = self.fc1(x)
        x = self.dropout(x)
        x = self.fc2(x)

        return x
    def _make_layers(self, cfg):
        layers = []
        in_channels = 1
        for x in cfg:
            layers += [nn.Conv2d(in_channels, x, kernel_size=3),
                        nn.BatchNorm2d(x),
                        nn.ReLU(inplace=True),
                        nn.MaxPool2d(kernel_size=2, stride=2)]
            in_channels = x
        return nn.Sequential(*layers)

# In[1067]:

trainLoader = torch.utils.data.DataLoader(X, batch_size=4, shuffle=True, num_workers=2)
valLoader = torch.utils.data.DataLoader(Valid, batch_size=1, shuffle=False, num_workers=2)
testLoader = torch.utils.data.DataLoader(Test, batch_size=1, shuffle=False, num_workers=2)

# In[1068]:

Conv_model = LENet()
"""
Conv_model = models.resnet50(pretrained=True)
for param in Conv_model.parameters():
    param.requires_grad = False

new_fc = nn.Sequential(*list(Conv_model.fc.children())[:-1] + [nn.Linear(2048, 1)])
Conv_model.fc = new_fc
"""
Conv_model = Conv_model.cuda()
# loss
criterion = nn.MSELoss(reduction='sum')
# optimizer
learning_rate = 1e-3
weight_decay = 0
optimizer = torch.optim.Adam(Conv_model.parameters(), lr = learning_rate, weight_decay=weight_decay)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=200)

# In[1069]:

# GPU
device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
print('GPU state:', device)

# In[1070]:

def val(valLoader, model):
    with torch.no_grad():
        model.eval()
        total_loss = 0
        for data in valLoader:
            inputs, label = data
            inputs, label = inputs.to(device), label.to(device)
            MLP_model = model.cuda()
            outputs = model(inputs)
            outputs = outputs[0][0]
            #print('label:', label)
            total_loss += np.abs(np.sum((label - outputs).cpu().numpy()))
        return total_loss

# In[1071]:

def train(model):
    validation_error = []
    patience = 100
    last_loss = 1e9
    trigger_times = 0
    total_epoch = 1000
    for epoch in range(total_epoch):
        running_loss = 0.0
        for times, data in enumerate(trainLoader, 0):
            inputs, labels = data
            inputs, labels = inputs.to(device), labels.to(device)

            #print(inputs.shape)
            # Zero the parameter gradients
            optimizer.zero_grad()

            # forward + backward + optimize
            outputs = model(inputs)
            outputs = outputs.reshape(-1)
            loss = criterion(outputs, labels)
            loss.backward()
            optimizer.step()

            # print statistics
            running_loss += loss.item()
            if times+1 == len(trainLoader):
                print('[%d/%d, %d/%d] loss: %.3f' % (epoch+1, total_epoch, times+1, len(trainLoader), running_loss))
        validation_error.append(val(valLoader, model))
        print('validation_error:', validation_error[-1])
        if validation_error[-1] > last_loss:
            trigger_times += 1
            print('trigger times:', trigger_times)
        else:
            trigger_times = 0
            print('trigger times: 0')
        if trigger_times >= patience:
            print('Early stopping!\nStart to test process.')
            break
        last_loss = validation_error[-1]
    print('Finished Training')

# In[1072]:

train(Conv_model)

# In[1073]:

PATH = './Conv_Net_melspec.pth'
torch.save(Conv_model.state_dict(), PATH)

# In[1074]:

PATH = './Conv_Net_melspec.pth'
net = LENet()
"""
net = models.resnet50(pretrained=True)
for param in net.parameters():
    param.requires_grad = False

new_fc = nn.Sequential(*list(net.fc.children())[:-1] + [nn.Linear(2048, 1)])
net.fc = new_fc
"""
net.load_state_dict(torch.load(PATH))
net = net.cuda()

# In[1075]:

# inference
predicted_res = []
with torch.no_grad():
    net.eval()
    for data in testLoader:
        input, label = data
        input = input.to(device)
        output = net(input)
        output = output.cpu().numpy()
        predicted_res.append(output[0][0])
submission_df = pd.read_csv('sample_submission.csv')
submission_df['score'] = predicted_res

#submission_df = submission_df.drop(['score'], axis=1)
#submission_df = submission_df.insert(1, "score", predicted_res)
submission_df.to_csv('submission/CNN_submission_melspec.csv', index=False)

# In[ ]:

get_ipython().system("jupyter nbconvert --to script 'music_regression_cnn.ipynb'")


from fastapi import APIRouter, Body, Depends, Request

from app.middlewares import JWTBearer, Role
from app.repositories import supabase
from app.schemas import PoseSet, PoseSetInsert, PoseSetWithPoses

router = APIRouter(
    prefix="/pose_sets", tags=["Pose sets"], dependencies=[Depends(JWTBearer())])

@router.get("/")
def get_pose_sets(request: Request):
    res = supabase.table("pose_sets").select(
        "*,poses(*,cover_image:images!poses_image_fkey(*),skeleton_image:images!poses_skeleton_image_fkey(*))").execute()

    pose_sets_with_poses = [PoseSetWithPoses(**pose_set)
                            for pose_set in res.data]

    return {"data": pose_sets_with_poses, "count": len(pose_sets_with_poses)}

@router.post("/new", dependencies=[Depends(JWTBearer(Role.ADMIN))])
def create_pose_set(new_pose_set: PoseSetInsert = Body(...)):
    pose_res = supabase.table("pose_sets").insert(
        json={"name": new_pose_set.name}).execute()

    pose_set = PoseSet(**pose_res.data[0])

    pose_set_poses_res = supabase.table("pose_sets_poses").insert(json=[
        {"set_id": pose_set.id, "pose_id": pose} for pose in new_pose_set.poses
    ]).execute()

    pose_set_with_poses_res = supabase.table("pose_sets").select(
        "*,poses(*,cover_image:images!poses_image_fkey(*),skeleton_image:images!poses_skeleton_image_fkey(*))").eq("id", pose_set.id).single().execute()

    print(pose_set_with_poses_res.data)

    pose_set_with_poses = PoseSetWithPoses(**pose_set_with_poses_res.data)

    return {"data": pose_set_with_poses, "count": 1}


from typing import List

from EnumerationIterator import Enumeration
from IteratorEnumeration import Iterator

class EI:
    @staticmethod
    def main(args: List[str]) -> None:
        v = args
        enumeration: Enumeration = Enumeration(v)
        while enumeration.hasMoreElements():
            print(enumeration.nextElement())
        iterator: Iterator = Iterator(v)
        while iterator.hasNext():
            print(iterator.next())

if __name__ == "__main__":
    EI.main(['a', 'b', 'c'])


import eam_core.util as util
import unittest
import os
from deepdiff import DeepDiff

import eam_core.yaml_runner
from eam_core.yaml_runner import load_configuration
from eam_core.YamlLoader import YamlLoader
from types import SimpleNamespace
import yaml
from ruamel import yaml

def compare_objects(first_object, second_object, ignore_order=False, report_repetition=False, significant_digits=None,
                    verbose_level=2):
    """
    Return difference between two objects.
    :param first_object: first object to compare
    :param second_object: second object to compare
    :param ignore_order: ignore difference in order
    :param report_repetition: report when is repetition
    :param significant_digits: use to properly compare numbers(float arithmetic error)
    :param verbose_level: higher verbose level shows you more details - default 0.
    :return: difference between two objects
    """

    diff = DeepDiff(first_object, second_object, ignore_order=ignore_order,
                    report_repetition=report_repetition, significant_digits=significant_digits,
                    verbose_level=verbose_level)
    return diff

def get_ids(file, flag=True):
    dict = {"IDs": flag, "analysis_config": "dev", "comment": None, "docker": True, "documentation": False,
            "filetype": "pdf", "local": True, "sensitivity": False, "verbose": False,
            "yamlfile": "tests/models/" + file + ".yml"}
    args = SimpleNamespace(**dict)
    model_run_base_directory, simulation_run_description, yaml_struct = load_configuration(args)
    scenario = "default"
    model_output_directory = model_run_base_directory + f"/{scenario}"
    if not os.path.exists(model_output_directory):
        os.makedirs(model_output_directory)
    create_model_func, sim_control, yaml_struct = eam_core.yaml_runner.prepare_simulation(model_output_directory,
                                                                                simulation_run_description, yaml_struct,
                                                                                scenario, filename=args.yamlfile,
                                                                                IDs=args.IDs)
    loader = YamlLoader(version=yaml_struct.get('variant', 1))
    loader.parse_yaml_structure(yaml_struct, sim_control)
    old = open("tests/models/" + file + "_copy.yml")
    old_struct = yaml.load(old, Loader=yaml.RoundTripLoader)
    new = open("tests/models/" + file + ".yml")
    new_struct = yaml.load(new, Loader=yaml.RoundTripLoader)
    diff = compare_objects(old_struct, new_struct)
    old.close()
    new.close()
    with open("tests/models/" + file + ".yml", 'w') as file:
        yaml.dump(old_struct, file, default_flow_style=False, Dumper=yaml.RoundTripDumper,
                  width=4096)
    return diff, loader.id_map

class TestProcessIDs(unittest.TestCase):

    def test_existing_ids(self):
        diff, id_map = get_ids("existing_ids")
        assert (diff == {})
        assert id_map["process"] == {"CDN": 0, "Internet Network": 1, "Laptop": 2}

    def test_some_existing_ids(self):
        diff, id_map = get_ids("some_existing_ids")
        assert (diff == {'dictionary_item_added': {"root['Processes'][1]['id']": 4, "root['Processes'][2]['id']": 5}})
        assert id_map["process"] == {"CDN": 3, "Internet Network": 4, "Laptop": 5}

    def test_no_existing_ids(self):
        diff, id_map = get_ids("no_existing_ids")
        print(diff)
        assert diff == {'dictionary_item_added': {"root['Processes'][0]['id']": 0, "root['Processes'][1]['id']": 1,
                                                  "root['Processes'][2]['id']": 2}}
        assert id_map["process"] == {"CDN": 0, "Internet Network": 1, "Laptop": 2}

    def test_duplicate_ids(self):
        with self.assertRaises(Exception) as context:
            get_ids("duplicate_ids")
        self.assertTrue('Duplicate ID for process ' in str(context.exception))

    def test_no_id_flag(self):
        diff, id_map = get_ids("no_existing_ids", flag=False)
        assert diff == {}
        assert id_map["process"] == {}


import weechat

SCRIPT_NAME = "closeall"
SCRIPT_AUTHOR = "florianb"
SCRIPT_VERSION = "0.1"
SCRIPT_LICENSE = "GPL3"
SCRIPT_DESC = "Closes all query/plugin buffers"

settings = {
        "channel_prefix": "#,&",
}

description = {
        "channel_prefix"           : "Set to Channel prefixes you want not to close on /close query [default: #,&]",
}

def closeall_plugin(pluginname):
    """ Closes all Buffers of a Plugin Type """

    infolist = weechat.infolist_get("buffer", "", "")

    while weechat.infolist_next(infolist):
        if weechat.infolist_string(infolist, "plugin_name") == pluginname:
            if not weechat.infolist_string(infolist, "name") == "weechat":
                weechat.command('', '/buffer close %s.%s' %
                               (pluginname,
                                weechat.infolist_string(infolist, "name")))
    weechat.infolist_free(infolist)
    return weechat.WEECHAT_RC_OK

def closeall_query():
    """ Closes all IRC Buffers which do not start with channel_prefix """

    infolist = weechat.infolist_get("buffer", "", "")
    while weechat.infolist_next(infolist):
        if weechat.infolist_string(infolist, "plugin_name") == "irc":
            pref = tuple(weechat.config_get_plugin("channel_prefix").split(","))
            channel = weechat.infolist_string(infolist, "name").split(".", 1)
            if not channel[1].startswith(pref) and channel[0] != "server":
                weechat.command('', '/buffer close irc.%s.%s' % (channel[0], channel[1]))
    weechat.infolist_free(infolist)

    return weechat.WEECHAT_RC_OK

def closeall_command_cb(data, buffer, args):
    """ Callback for /closeall """
    argv = args.strip().split(" ", 1)
    if argv[0] == "query":
        closeall_query()
    if argv[0] == "plugin":
        closeall_plugin(argv[1])
    return weechat.WEECHAT_RC_OK

if weechat.register(SCRIPT_NAME, SCRIPT_AUTHOR, SCRIPT_VERSION, SCRIPT_LICENSE,
                    SCRIPT_DESC, "", ""):

    # Set config options if not set
    for option, default_value in settings.iteritems():
        if not weechat.config_is_set_plugin(option):
            weechat.config_set_plugin(option, default_value)
    # Set description to options if weechat >= 0.3.5
    version = weechat.info_get("version_number", "") or 0
    if int(version) >= 0x00030500:
        for desc, desc_value in description.iteritems():
            weechat.config_set_desc_plugin(desc, desc_value)

    weechat.hook_command('closeall', SCRIPT_DESC,
                         'query || plugin <plugin name>',
                         '               query: closes all IRC query buffer\n'
                         'plugin <plugin name>: closes all buffers of given plugin\n',
                         '', 'closeall_command_cb', '')


"""
DJANGO DB TOOLS MODULE
"""
from django.db import connection

def executeRawSqlQuery(query):
    """
    executes a SQL query and returns the result
    """
    cursor = connection.cursor()
    try:
        cursor.execute(query)
        return cursor.fetchall()
    finally:
        cursor.close()


"""
download HGSVC2_phased_assembly for full-resolution typing assessment

wangshuai July 11, 2022
"""

def get_link():
    f = open(file)
    out = open(command_file, 'w')
    for line in f:
        array = line.split()
        link = prefix_link + array[0]
        command = "wget %s"%(link)
        print (command, file = out)

    out.close()
    f.close()

prefix_link = "https://ftp.1000genomes.ebi.ac.uk/vol1/ftp/data_collections/HGSVC2/release/v1.0/assemblies/"
file = "HGSVC2_phased_assembly_download_link.txt"
command_file = "/mnt/d/HLAPro_backup/haplotype/my_HLA/download.sh"
get_link()


import clr
clr.AddReference('RevitAPI')
from Autodesk.Revit.DB import *

clr.AddReference("RevitNodes")
import Revit
clr.ImportExtensions(Revit.Elements)

items = UnwrapElement(IN[0])
functionlist = list()

for item in items:
	try:
		functionlist.append(item.get_Parameter(BuiltInParameter.FUNCTION_PARAM).AsValueString())
	except:
		functionlist.append(None)
OUT = functionlist


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Nov 17 16:09:33 2018

@author: zili & haolinl
"""

import numpy as np
import torch
import torchvision
import torch.nn as nn
import torchvision.transforms as transforms
import scipy.io
import time

class Net1(nn.Module):
    def __init__(self):
        super(Net1, self).__init__()

        self.conv1 = nn.Sequential(
            nn.Conv2d(1,10,13),
#            nn.Dropout2d(0.5),
            nn.BatchNorm2d(10),
            nn.MaxPool2d(2,2),
            nn.ReLU()
        )
        self.conv2 = nn.Sequential(
            nn.Conv2d(10,20,7),
#            nn.Dropout2d(0.5),
            nn.BatchNorm2d(20),
            nn.MaxPool2d(2,2),
            nn.ReLU()
        )
        self.conv3 = nn.Sequential(
            nn.Conv2d(20,40,7),
            nn.BatchNorm2d(40),
#            nn.Dropout2d(0.5),
            nn.MaxPool2d(2,2),
            nn.ReLU()
        )
        self.conv4 = nn.Sequential(
            nn.Conv2d(40,80,5),
            nn.BatchNorm2d(80),
#            nn.Dropout2d(0.5),
            nn.MaxPool2d(2,2),
            nn.ReLU()
        )
        self.fc1 = nn.Sequential(
            nn.Linear(720,400),
            nn.ReLU(),
            nn.Dropout(0.5)
        )
        self.fc2 = nn.Linear(400,1)

    def forward(self, x):
        f1 = self.conv1(x)
        f2 = self.conv2(f1)
        f3 = self.conv3(f2)
        f4 = self.conv4(f3)
        f4_flat = f4.view(f4.size(0), -1)
        f5 = self.fc1(f4_flat)
        output = self.fc2(f5)
        return output

if __name__ == "__main__":
    train_data_path = 'all_data_1000.mat'
    neural_net_savePath = 'CNN.pkl' # Path to save CNN's parameters

    data = scipy.io.loadmat(train_data_path)
    data_x, data_y, rNorm = data['data_x'], data['data_y'], data['Normalization_Factor']
    num_data = np.shape(data_x)[0]
    train_x, train_y = data_x[:int(0.9*num_data)], data_y[:int(0.9*num_data)]
    valid_x,valid_y = data_x[int(0.9*num_data):], data_y[int(0.9*num_data):]
    #test_x, test_y = data_x[int(0.9*num_data):], data_y[int(0.9*num_data):]

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    #set up parameters
    batch_size = 32
    learning_rate = 0.00005
    num_epochs = 200

    train_x = torch.from_numpy(train_x).float()
    train_y = torch.from_numpy(train_y).float()
    valid_x = torch.from_numpy(valid_x).float()
    valid_y = torch.from_numpy(valid_y).float()
    #test_x = torch.from_numpy(test_x).float()
    #test_y = torch.from_numpy(test_y).float()

    train_dataset = torch.utils.data.TensorDataset(train_x, train_y)
    valid_dataset = torch.utils.data.TensorDataset(valid_x, valid_y)
    #test_dataset = torch.utils.data.TensorDataset(test_x, test_y)

    train_dataloader = torch.utils.data.DataLoader(
        dataset=train_dataset,
        batch_size=batch_size,
        shuffle=True
    )
    valid_dataloader = torch.utils.data.DataLoader(
        dataset=valid_dataset,
        batch_size=batch_size,
        shuffle=True
    )
    #test_dataloader = torch.utils.data.DataLoader(
    #    dataset=test_dataset
    #)
    neural_net = Net1().to(device)

    criterion = nn.MSELoss()
    optimizer = torch.optim.Adam(neural_net.parameters(),learning_rate)
    lossList = []
    accList = []
    valid_lossList = []
    valid_accList = []

    for epoch in range(num_epochs):
        loss_sum_train = 0
        loss_sum_valid = 0
        acc_sum = 0

        for iteration, (images, labels) in enumerate(train_dataloader):
            x_batch = torch.autograd.Variable(images)
            x_batch = x_batch.reshape(-1,1,128,128)
            y_batch = torch.autograd.Variable(labels)
            x_batch = x_batch.to(device)
            y_batch = y_batch.to(device)

            output = neural_net(x_batch)
            loss = criterion(output,y_batch)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            acc = abs((output.cpu() - labels)/labels).mean()
            acc_sum += acc
            loss_sum_train += loss.cpu().data.numpy()
        loss_sum_train = batch_size * loss_sum_train /len(train_dataloader.dataset)
        lossList.append(loss_sum_train*rNorm[0][0])

        for iteration, (images, labels) in enumerate(valid_dataloader):
            x_batch = torch.autograd.Variable(images)
            x_batch = x_batch.reshape(-1,1,128,128)
            y_batch = torch.autograd.Variable(labels)
            x_batch = x_batch.to(device)
            y_batch = y_batch.to(device)

            output_valid = neural_net(x_batch)
            loss = criterion(output_valid,y_batch)
            loss_sum_valid += loss.cpu().data.numpy()

        loss_sum_valid = batch_size * loss_sum_valid/len(valid_dataloader.dataset)
        valid_lossList.append(loss_sum_valid*rNorm[0][0])
        print('Epoch: ', epoch, '| train loss: %.6f | valid loss: %.6f  ' % (loss_sum_train*rNorm[0][0], loss_sum_valid*rNorm[0][0]))

        if ((epoch+1)%20 == 0):
            CNN_savePath_temp = 'CNN_' + str(int((epoch+1)/20)) + '.pkl'
            torch.save(neural_net.state_dict(), CNN_savePath_temp) # Save model every 20 epochs

    np.save('valid_loss.npy', valid_lossList)
    np.save('train_loss.npy', lossList)
    torch.save(neural_net.state_dict(), neural_net_savePath) # Save the final CNN model parameters

#__name__ = "__main__"


from __future__ import (absolute_import, division,
                        print_function, unicode_literals)
from builtins import *

import numpy as np
import skimage.draw

def overlap_measure(A, B, resolution=250):
    # implemented according to ginneken 2002 equation (10), performance = TP/(TP+FP+FN)
    # performance = (intersection of shape 1 and 2) / (union of shape 1 and 2)
    # resolution determines the grid resolution (resolution * resolution) that is used to calculate overlap
    A = np.asanyarray(A)
    B = np.asanyarray(B)

    A_x_min, A_y_min = A.min(axis=0)
    A_x_max, A_y_max = A.max(axis=0)

    B_x_min, B_y_min = B.min(axis=0)
    B_x_max, B_y_max = B.max(axis=0)

    AB_min = np.array([np.min([A_x_min, B_x_min]), np.min([A_y_min, B_y_min])])
    AB_max = np.array([np.max([A_x_max, B_x_max]), np.max([A_y_max, B_y_max])])

    AB_size = AB_max - AB_min

    A_unit = (A - AB_min) / AB_size
    B_unit = (B - AB_min) / AB_size

    A_unit *= resolution
    B_unit *= resolution

    A_polygon = np.zeros((resolution, resolution))
    B_polygon = np.zeros((resolution, resolution))

    A_rr, A_cc = skimage.draw.polygon(A_unit[:, 1], A_unit[:, 0])
    B_rr, B_cc = skimage.draw.polygon(B_unit[:, 1], B_unit[:, 0])

    A_polygon[A_rr, A_cc] = 1
    B_polygon[B_rr, B_cc] = 1

    AB_union = np.logical_or(A_polygon, B_polygon)
    AB_intersection = np.logical_and(A_polygon, B_polygon)

    return np.count_nonzero(AB_intersection) / np.count_nonzero(AB_union)


from flask_wtf import FlaskForm
from wtforms import StringField, BooleanField, SelectField
from wtforms.validators import DataRequired
from wtforms.widgets import CheckboxInput
from flask_sqlalchemy import SQLAlchemy
from sqlalchemy.orm import aliased
from settings import admins, allreports, site_url, saml_sp, cookie_issuing_file
from datetime import datetime

db = SQLAlchemy()  # Create a database object

##################
# Object Classes #
##################

# Institution class
class Institution(db.Model):
    code = db.Column(db.String(255), primary_key=True)
    name = db.Column(db.String(255), nullable=False)
    fulfillment_code = db.Column(db.String(255), nullable=False, unique=True)
    partner_code = db.Column(db.String(255), nullable=False, unique=True)
    key = db.Column(db.String(255), nullable=True)
    exceptions = db.Column(db.String(255), nullable=True)
    ext_requests_in_transit = db.Column(db.String(255), nullable=True)
    in_transit_data = db.Column(db.String(255), nullable=True)

    # Constructor
    def __repr__(self):
        return '<Institution %r>' % self.code

    # Get institution's last update datetime
    def get_last_update(self):
        updated = db.session.execute(
            db.select(
                InstUpdate
            ).filter(
                InstUpdate.instcode == self.code
            ).order_by(
                InstUpdate.last_update.desc()
            )
        ).first()

        return updated

    # Get all current RequestExceptions statuses for the institution
    def get_statuses(self):
        statuses = db.session.execute(
            db.select(
                RequestException.borreqstat
            ).filter(
                RequestException.instcode == self.code
            ).group_by(
                RequestException.borreqstat
            ).order_by(
                RequestException.borreqstat
            )
        ).mappings().all()
        return statuses

    # Get all current RequestExceptions for the institution by status
    def get_exceptions_by_status(self, status):
        ib = aliased(Institution)
        il = aliased(Institution)
        requests = db.session.execute(
            db.select(
                RequestException.borreqstat, RequestException.internalid, RequestException.borcreate,
                RequestException.title, RequestException.author, RequestException.networknum,
                RequestException.partnerstat, RequestException.reqsend, RequestException.days,
                RequestException.requestor, RequestException.partnername, RequestException.partnercode,
                ExternalRequestInTransit.request_id, TransitStart.transit_date
            ).join(
                ib, RequestException.instcode == ib.code
            ).outerjoin(
                il, RequestException.partnercode == il.partner_code
            ).outerjoin(
                ExternalRequestInTransit, (ExternalRequestInTransit.title == RequestException.title) &
                                          (ExternalRequestInTransit.external_id == ib.fulfillment_code) &
                                          (ExternalRequestInTransit.instcode == il.code)
            ).outerjoin(
                TransitStart, ExternalRequestInTransit.request_id == TransitStart.request_id
            ).filter(
                RequestException.instcode == self.code,
                RequestException.borreqstat == status
            ).order_by(
                RequestException.borreqstat, RequestException.internalid.desc(), RequestException.borcreate.desc(),
                RequestException.reqsend.desc()
            )
        ).mappings().all()
        return requests

    # Get all current RequestExceptions for all institutions
    def get_all_requests(self):
        ib = aliased(Institution)
        il = aliased(Institution)
        requests = db.session.execute(
            db.select(
                RequestException.borreqstat.label('Borrowing Request Status'),
                RequestException.internalid.label('Internal ID'),
                RequestException.borcreate.label('Borrowing Request Date'),
                RequestException.title.label('Title'),
                RequestException.author.label('Author'),
                RequestException.networknum.label('Network Number'),
                RequestException.partnerstat.label('Partner Active Status'),
                RequestException.reqsend.label('Request Sending Date'),
                RequestException.days.label('Days Since Request'),
                RequestException.requestor.label('Requestor'),
                RequestException.partnername.label('Partner Name'),
                RequestException.partnercode.label('Partner Code'),
                ExternalRequestInTransit.request_id.label('In Transit?'),
                TransitStart.transit_date.label('In Transit Start')
            ).join(
                ib, RequestException.instcode == ib.code
            ).outerjoin(
                il, RequestException.partnercode == il.partner_code
            ).outerjoin(
                ExternalRequestInTransit, (ExternalRequestInTransit.title == RequestException.title) &
                                          (ExternalRequestInTransit.external_id == ib.fulfillment_code) &
                                          (ExternalRequestInTransit.instcode == il.code)
            ).outerjoin(
                TransitStart, ExternalRequestInTransit.request_id == TransitStart.request_id
            ).filter(
                RequestException.instcode == self.code,
            ).order_by(
                RequestException.borreqstat, RequestException.internalid.desc(), RequestException.borcreate.desc(),
                RequestException.reqsend.desc()
            )
        ).mappings().all()

        requests_dict = [dict(row) for row in requests]

        columns = ['Borrowing Request Status', 'Internal ID', 'Borrowing Request Date', 'Title', 'Author',
                   'Network Number',
                   'Requestor', 'Partner Active Status', 'Request Sending Date', 'Days Since Request', 'Partner Name',
                   'Partner Code', 'In Transit?', 'In Transit Start']

        parsed_requests = []  # List to hold parsed requests

        for request in requests_dict:  # Parse requests
            if request['In Transit?'] is None:  # If the request is not in transit
                request['In Transit?'] = 'N'  # Set the value to 'N'
            else:  # If the request is in transit
                request['In Transit?'] = 'Y'  # Set the value to 'Y'
            ordered_request = {k: request[k] for k in columns}  # Create a dictionary of the request
            parsed_requests.append(ordered_request)  # Add the parsed request to the list

        return parsed_requests  # Return the list of parsed requests

    # Get all current RequestExceptions for all institutions filtered by status
    def get_all_requests_filtered(self, userstatuses):
        ib = aliased(Institution)
        il = aliased(Institution)
        requests = db.session.execute(
            db.select(
                RequestException.borreqstat.label('Borrowing Request Status'),
                RequestException.internalid.label('Internal ID'),
                RequestException.borcreate.label('Borrowing Request Date'),
                RequestException.title.label('Title'),
                RequestException.author.label('Author'),
                RequestException.networknum.label('Network Number'),
                RequestException.partnerstat.label('Partner Active Status'),
                RequestException.reqsend.label('Request Sending Date'),
                RequestException.days.label('Days Since Request'),
                RequestException.requestor.label('Requestor'),
                RequestException.partnername.label('Partner Name'),
                RequestException.partnercode.label('Partner Code'),
                ExternalRequestInTransit.request_id.label('In Transit?'),
                TransitStart.transit_date.label('In Transit Start')
            ).join(
                ib, RequestException.instcode == ib.code
            ).outerjoin(
                il, RequestException.partnercode == il.partner_code
            ).outerjoin(
                ExternalRequestInTransit, (ExternalRequestInTransit.title == RequestException.title) &
                                          (ExternalRequestInTransit.external_id == ib.fulfillment_code) &
                                          (ExternalRequestInTransit.instcode == il.code)
            ).outerjoin(
                TransitStart, ExternalRequestInTransit.request_id == TransitStart.request_id
            ).filter(
                RequestException.instcode == self.code,
            ).filter(
                RequestException.borreqstat.any(userstatuses)
            ).order_by(
                RequestException.borreqstat, RequestException.internalid.desc(), RequestException.borcreate.desc(),
                RequestException.reqsend.desc()
            )
        ).mappings().all()

        requests_dict = [dict(row) for row in requests]

        columns = ['Borrowing Request Status', 'Internal ID', 'Borrowing Request Date', 'Title', 'Author',
                   'Network Number',
                   'Requestor', 'Partner Active Status', 'Request Sending Date', 'Days Since Request', 'Partner Name',
                   'Partner Code', 'In Transit?', 'In Transit Start']

        parsed_requests = []  # List to hold parsed requests

        for request in requests_dict:  # Parse requests
            if request['In Transit?'] is None:  # If the request is not in transit
                request['In Transit?'] = 'N'  # Set the value to 'N'
            else:  # If the request is in transit
                request['In Transit?'] = 'Y'  # Set the value to 'Y'
            ordered_request = {k: request[k] for k in columns}  # Create a dictionary of the request
            parsed_requests.append(ordered_request)  # Add the parsed request to the list

        return parsed_requests  # Return the list of parsed requests

# Institution form class
class InstitutionForm(FlaskForm):
    code = StringField('Code', validators=[DataRequired()])
    name = StringField('Name', validators=[DataRequired()])
    fulfillment_code = StringField('Fulfillment Code', validators=[DataRequired()])
    partner_code = StringField('Partner Code', validators=[DataRequired()])
    key = StringField('API Key')
    exceptions = StringField('Exceptions Path')
    ext_requests_in_transit = StringField('External Requests in Transit Path')
    in_transit_data = StringField('In Transit Data Path')

# Request Exception class
class RequestException(db.Model):
    exception_id = db.Column(db.BigInteger, primary_key=True)
    instcode = db.Column(db.ForeignKey(Institution.code))
    borreqstat = db.Column(db.String(255), nullable=True)
    internalid = db.Column(db.String(255), nullable=True)
    borcreate = db.Column(db.Date, nullable=True)
    title = db.Column(db.String(510), nullable=True)
    author = db.Column(db.String(255), nullable=True)
    networknum = db.Column(db.String(255), nullable=True)
    isbn = db.Column(db.String(255), nullable=True)
    issn = db.Column(db.String(255), nullable=True)
    requestor = db.Column(db.String(255), nullable=True)
    partnerstat = db.Column(db.String(255), nullable=True)
    reqsend = db.Column(db.DateTime, nullable=True)
    days = db.Column(db.Integer, nullable=True)
    partnername = db.Column(db.String(255), nullable=True)
    partnercode = db.Column(db.String(255), nullable=True)

    # Constructor
    def __repr__(self):
        return '<RequestException %r>' % self.exception_id

# External Request in Transit class
class ExternalRequestInTransit(db.Model):
    request_id = db.Column(db.BigInteger, primary_key=True)
    instcode = db.Column(db.ForeignKey(Institution.code))
    external_id = db.Column(db.String(255), nullable=False)
    requestor = db.Column(db.String(255), nullable=True)
    title = db.Column(db.String(510), nullable=True)
    author = db.Column(db.String(255), nullable=True)
    barcode = db.Column(db.String(255), nullable=True)
    physical_item_id = db.Column(db.String(255), nullable=True)
    isbn = db.Column(db.String(255), nullable=True)
    issn = db.Column(db.String(255), nullable=True)

    # Constructor
    def __repr__(self):
        return '<ExternalRequestInTransit %r>' % self.request_id

# Transit Start class
class TransitStart(db.Model):
    event_id = db.Column(db.BigInteger, primary_key=True)
    instcode = db.Column(db.ForeignKey(Institution.code))
    request_id = db.Column(db.String(255), nullable=False)
    transit_date = db.Column(db.DateTime, nullable=False)

    # Constructor
    def __repr__(self):
        return '<InTransitData %r>' % self.event_id

# Institution Update class
class InstUpdate(db.Model):
    id = db.Column(db.BigInteger, primary_key=True)
    instcode = db.Column(db.ForeignKey(Institution.code))
    last_update = db.Column(db.DateTime, nullable=False)

    # Constructor
    def __repr__(self):
        return '<InstUpdate %r>' % self.id

# User class
class User(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    username = db.Column(db.String(255), nullable=False)
    displayname = db.Column(db.String(255), nullable=True)
    instcode = db.Column(db.ForeignKey(Institution.code))
    emailaddress = db.Column(db.String(255), nullable=True)
    admin = db.Column(db.Boolean, nullable=False)
    allreports = db.Column(db.Boolean, nullable=False)
    last_login = db.Column(db.DateTime, nullable=True)

    # Constructor
    def __repr__(self):
        return '<User %r>' % self.username

# User Day class
class UserDay(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    user = db.Column(db.ForeignKey(User.id), nullable=False)
    day = db.Column(db.Integer, nullable=False)
    __table_args__ = (
        db.UniqueConstraint('user', 'day', name='_user_day_uc'),
    )

    # Constructor
    def __repr__(self):
        return '<UserDay %r>' % self.id

class StatusUser(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    status = db.Column(db.String(255), nullable=False)
    user = db.Column(db.ForeignKey(User.id), nullable=False)
    __table_args__ = (
        db.UniqueConstraint('status', 'user', name='_status_user_uc'),
    )

    # Constructor
    def __repr__(self):
        return '<StatusUser %r>' % self.id

# User Settings form class
class UserSettingsForm(FlaskForm):
    sunday = BooleanField('Sunday', widget=CheckboxInput())
    monday = BooleanField('Monday', widget=CheckboxInput())
    tuesday = BooleanField('Tuesday', widget=CheckboxInput())
    wednesday = BooleanField('Wednesday', widget=CheckboxInput())
    thursday = BooleanField('Thursday', widget=CheckboxInput())
    friday = BooleanField('Friday', widget=CheckboxInput())
    saturday = BooleanField('Saturday', widget=CheckboxInput())
    AUTOMATIC_RENEW = BooleanField('Automatic renew', widget=CheckboxInput())
    AUTO_WILL_SUPPLY = BooleanField('Automatic will supply', widget=CheckboxInput())
    BAD_CITATION = BooleanField('Bad citation', widget=CheckboxInput())
    CANCEL_NOT_ACCEPTED = BooleanField('Cancel request not accepted', widget=CheckboxInput())
    CANCELLED = BooleanField('Cancelled by partner', widget=CheckboxInput())
    CLAIMED = BooleanField('Claimed', widget=CheckboxInput())
    CONDITIONAL = BooleanField('Conditional', widget=CheckboxInput())
    REQUEST_CREATED_BOR = BooleanField('Created borrowing request', widget=CheckboxInput())
    DAMAGED_COMMUNICATED = BooleanField('Damaged communicated', widget=CheckboxInput())
    DECLARED_LOST = BooleanField('Declared lost by partner', widget=CheckboxInput())
    RECEIVED_DIGITALLY = BooleanField('Digitally received by library', widget=CheckboxInput())
    EXPIRED = BooleanField('Expired', widget=CheckboxInput())
    EXTERNALLY_OBTAINED = BooleanField('Externally Obtained', widget=CheckboxInput())
    LENDER_CHECK_IN = BooleanField('Lender check in', widget=CheckboxInput())
    LOCAL_HOLDING = BooleanField('Local holding', widget=CheckboxInput())
    LOCATE_FAILED = BooleanField('Locate failed', widget=CheckboxInput())
    LOCATE_IN_PROCESS = BooleanField('Locate in process', widget=CheckboxInput())
    LOST_AND_FEE_PAID = BooleanField('Lost and fee paid', widget=CheckboxInput())
    LOST_COMMUNICATED = BooleanField('Lost communicated', widget=CheckboxInput())
    MANUAL_RENEW = BooleanField('Manual renew', widget=CheckboxInput())
    MEDIATED_RENEWAL = BooleanField('Mediated Patron Renewal', widget=CheckboxInput())
    OVERDUE_ITEM = BooleanField('Overdue request', widget=CheckboxInput())
    PENDING_APPROVAL = BooleanField('Pending Approval', widget=CheckboxInput())
    REACTIVATED = BooleanField('Reactivated', widget=CheckboxInput())
    READY_TO_SEND = BooleanField('Ready to be sent', widget=CheckboxInput())
    RECALLED_BOR = BooleanField('Recalled by partner', widget=CheckboxInput())
    REJECT = BooleanField('Reject', widget=CheckboxInput())
    RENEW_NOT_ACCEPTED = BooleanField('Renew request not accepted', widget=CheckboxInput())
    RENEW_REQUESTED = BooleanField('Renew requested', widget=CheckboxInput())
    RENEWED = BooleanField('Renewed by partner', widget=CheckboxInput())
    DAMAGED = BooleanField('Reported damaged item to partner', widget=CheckboxInput())
    REPORT_LOST = BooleanField('Reported lost item to partner', widget=CheckboxInput())
    REQUEST_ACCEPTED = BooleanField('Request accepted', widget=CheckboxInput())
    REQUEST_SENT = BooleanField('Request sent to partner', widget=CheckboxInput())
    RETURNED_TO_PARTNER = BooleanField('Returned item to partner', widget=CheckboxInput())
    SHIPPED_DIGITALLY = BooleanField('Shipped Digitally', widget=CheckboxInput())
    SHIPPED_PHYSICALLY = BooleanField('Shipped Physically', widget=CheckboxInput())
    CANCEL_REPLY = BooleanField('Waiting for cancel response', widget=CheckboxInput())
    RECEIVE_DIGITALLY_REPLY = BooleanField('Waiting for receive digitally', widget=CheckboxInput())
    WILL_SUPPLY = BooleanField('Will supply', widget=CheckboxInput())

# Login form class
class LoginForm(FlaskForm):
    institution = SelectField('Login At', choices=[
        ('au', 'American University'),
        ('amulaw', 'AU Washington College of Law'),
        ('cu', 'Catholic University of America'),
        ('ga', 'Gallaudet University'),
        ('gm', 'George Mason University'),
        ('gw', 'George Washington University'),
        ('gwalaw', 'GW Jacob Burns Law Library'),
        ('hi', 'GW Himmelfarb Health Sciences Library'),
        ('gt', 'Georgetown University'),
        ('gt-law', 'Georgetown University Law'),
        ('hu', 'Howard University'),
        ('mu', 'Marymount University'),
        ('dc', 'University of the District of Columbia'),
        ('wr', 'WRLC SCF'),
    ])

####################
# Helper Functions #
####################

# Add institution form submit
def add_institution_form_submit(form):
    institution = Institution(  # Build institution object from form data
        code=form.code.data, name=form.name.data, fulfillment_code=form.fulfillment_code.data,
        partner_code=form.partner_code.data, key=form.key.data, exceptions=form.exceptions.data,
        ext_requests_in_transit=form.ext_requests_in_transit.data, in_transit_data=form.in_transit_data.data
    )
    db.session.add(institution)  # Add institution to database
    db.session.commit()  # Commit changes to database

# Get all institutions
def get_all_institutions():
    institutions = db.session.execute(db.select(Institution).order_by(Institution.name)).scalars()
    return institutions

# Construct login URL
def construct_login_url(institution):
    login_url = saml_sp
    login_url += cookie_issuing_file
    login_url += '?institution='
    login_url += institution
    login_url += '&url='
    login_url += site_url
    login_url += '/login/n'
    return login_url

# Log the user in
def user_login(session, user_data):

    # Set the session variables
    session['username'] = user_data['primary_id']  # Set the username
    session['user_home'] = user_data['inst']  # Set the user's home institution
    session['display_name'] = user_data['full_name']  # Set the user's display name
    session['authorizations'] = user_data['authorizations']  # Set the user's authorizations
    session['email'] = user_data['email']  # Set the user's email

    user = check_user(session['username'])  # Check if the user exists in the database

    # If the user is in the database...
    if user is not None:
        set_email(user, session)  # ...set the user's email address
        set_user_admin(user, session)  # ...set the user's admin status
        set_user_allreports(user, session)  # ...set the user's allreports status
        if 'exceptions' in session['authorizations']:
            set_last_login(user)  # ...set the last login time for the user

    # If the user isn't in the database...
    else:
        admincheck = admincheck_user(session)  # ...check if the user is an admin
        allreportscheck = allreportscheck_user(session)  # ...check if the user is an admin
        add_user(session, admincheck, allreportscheck)  # ...add the user to the database
        set_user_admin(user, session)  # ...set the user's admin status
        set_user_allreports(user, session)  # ...set the user's allreports status

# Check if the user exists in the database
def check_user(username):
    user = db.session.execute(db.select(User).filter(User.username == username)).scalar_one_or_none()
    return user

# Set the user's email address
def set_email(user, session):
    user.emailaddress = session['email']
    db.session.commit()

# Set the last login time for the user
def set_last_login(user):
    user.last_login = datetime.now()  # Set the last login time to the current time
    db.session.commit()  # Commit the changes

# Set the user's admin status based on the database
def set_user_admin(user, session):
    if user.admin is True:  # Check if the user is an admin
        if 'exceptions' not in session['authorizations']:
            session['authorizations'].append('exceptions')  # If they are, add the admin authorization to the session
        session['authorizations'].append('admin')  # If they are, add the admin authorization to the session

# Check if the username is in the admin list
def admincheck_user(session):
    if session['username'] in admins:  # Check if the user is an admin
        admincheck = True  # If they are, set admincheck to True
    else:
        admincheck = False  # If they are not, set admincheck to False

    return admincheck

# Set the user's allreport status based on the database
def set_user_allreports(user, session):
    if user.allreports is True:  # Check if the user is an admin
        if 'exceptions' not in session['authorizations']:
            session['authorizations'].append('exceptions')  # If they are, add the admin authorization to the session
        session['authorizations'].append('allreports')  # If they are, add the admin authorization to the session

# Check if the username is in the allreports list
def allreportscheck_user(session):
    if session['username'] in allreports:  # Check if the user is an admin
        allreportscheck = True  # If they are, set admincheck to True
    else:
        allreportscheck = False  # If they are not, set admincheck to False

    return allreportscheck

# Add the user to the database
def add_user(session, admincheck, allreportscheck):

    # Create the user object
    user = User(
        username=session['username'],
        displayname=session['display_name'],
        instcode=session['user_home'],
        emailaddress=session['email'],
        admin=admincheck,
        allreports=allreportscheck,
        last_login=datetime.now()
    )
    db.session.add(user)  # Add the user to the database
    db.session.commit()  # Commit the changes

# Get all last updated times for institutions
def get_all_last_updates():
    updates = db.session.execute(
        db.select(
            InstUpdate.instcode, db.func.max(InstUpdate.last_update).label('last_update')
        ).group_by(InstUpdate.instcode)
    ).all()

    return updates

# Update user settings
def update_user_settings(form, user):
    db.session.execute(db.delete(UserDay).where(UserDay.user == user.id))  # delete all the user's days
    days = [
        (0, form.data['sunday']),
        (1, form.data['monday']),
        (2, form.data['tuesday']),
        (3, form.data['wednesday']),
        (4, form.data['thursday']),
        (5, form.data['friday']),
        (6, form.data['saturday']),
    ]
    for day in days:  # for each day
        if day[1] is True:
            user_day = UserDay(user=user.id, day=day[0])  # create a new user day
            db.session.add(user_day)  # add the user day to the database

    db.session.execute(db.delete(StatusUser).where(StatusUser.user == user.id))  # delete all the user's statuses
    statuses = [
        ('AUTOMATIC_RENEW', form.data['AUTOMATIC_RENEW']),
        ('AUTO_WILL_SUPPLY', form.data['AUTO_WILL_SUPPLY']),
        ('BAD_CITATION', form.data['BAD_CITATION']),
        ('CANCEL_NOT_ACCEPTED', form.data['CANCEL_NOT_ACCEPTED']),
        ('CANCELLED', form.data['CANCELLED']),
        ('CLAIMED', form.data['CLAIMED']),
        ('CONDITIONAL', form.data['CONDITIONAL']),
        ('REQUEST_CREATED_BOR', form.data['REQUEST_CREATED_BOR']),
        ('DAMAGED_COMMUNICATED', form.data['DAMAGED_COMMUNICATED']),
        ('DECLARED_LOST', form.data['DECLARED_LOST']),
        ('RECEIVED_DIGITALLY', form.data['RECEIVED_DIGITALLY']),
        ('EXPIRED', form.data['EXPIRED']),
        ('EXTERNALLY_OBTAINED', form.data['EXTERNALLY_OBTAINED']),
        ('LENDER_CHECK_IN', form.data['LENDER_CHECK_IN']),
        ('LOCAL_HOLDING', form.data['LOCAL_HOLDING']),
        ('LOCATE_FAILED', form.data['LOCATE_FAILED']),
        ('LOCATE_IN_PROCESS', form.data['LOCATE_IN_PROCESS']),
        ('LOST_AND_FEE_PAID', form.data['LOST_AND_FEE_PAID']),
        ('LOST_COMMUNICATED', form.data['LOST_COMMUNICATED']),
        ('MANUAL_RENEW', form.data['MANUAL_RENEW']),
        ('MEDIATED_RENEWAL', form.data['MEDIATED_RENEWAL']),
        ('OVERDUE_ITEM', form.data['OVERDUE_ITEM']),
        ('PENDING_APPROVAL', form.data['PENDING_APPROVAL']),
        ('REACTIVATED', form.data['REACTIVATED']),
        ('READY_TO_SEND', form.data['READY_TO_SEND']),
        ('RECALLED_BOR', form.data['RECALLED_BOR']),
        ('REJECT', form.data['REJECT']),
        ('RENEW_NOT_ACCEPTED', form.data['RENEW_NOT_ACCEPTED']),
        ('RENEW_REQUESTED', form.data['RENEW_REQUESTED']),
        ('RENEWED', form.data['RENEWED']),
        ('DAMAGED', form.data['DAMAGED']),
        ('REPORT_LOST', form.data['REPORT_LOST']),
        ('REQUEST_ACCEPTED', form.data['REQUEST_ACCEPTED']),
        ('REQUEST_SENT', form.data['REQUEST_SENT']),
        ('RETURNED_TO_PARTNER', form.data['RETURNED_TO_PARTNER']),
        ('SHIPPED_DIGITALLY', form.data['SHIPPED_DIGITALLY']),
        ('SHIPPED_PHYSICALLY', form.data['SHIPPED_PHYSICALLY']),
        ('CANCEL_REPLY', form.data['CANCEL_REPLY']),
        ('RECEIVE_DIGITALLY_REPLY', form.data['RECEIVE_DIGITALLY_REPLY']),
        ('WILL_SUPPLY', form.data['WILL_SUPPLY']),
    ]
    for status in statuses:  # for each status
        if status[1] is True:
            status_user = StatusUser(status=status[0], user=user.id)  # create a new status user
            db.session.add(status_user)  # add the status user to the database

    db.session.commit()  # commit changes to the database


import _io
import logging

import numpy as np
import pandas as pd

from ordered_set import OrderedSet

from socket import gethostname
from argparse import ArgumentParser, Namespace

# this can be removed, it was used to distinguish between the local machine and
# the (remote) machine where we run the final experiments
if gethostname() != 'BigMedilytics':
    from ctgan import CTGANSynthesizer

from purify.dataset.metadata import Metadata
from purify.dataset.profiling import profiler
from purify.dataset.processors import PreProcessor
from purify.encoders import label_encoders_fit_transform, label_encoders_inverse_transform
from purify.encoders import get_dummies_fit_transform, get_dummies_inverse_transform
from purify.generation.tabulator import TabularDataGenerator

from typing import List, Dict, Tuple, Union

pd.set_option('display.max_rows', None)
pd.set_option('display.max_columns', None)
pd.set_option('display.width', None)
pd.set_option('display.max_colwidth', None)

DATASETS: Dict[str, int] = {
    'adult': 30162, 'breast': 569, 'cover': 581012, 'credit': 30000, 'eeg': 14980, 'iris': 150, 'iris_sample': 12,
    'letter': 20000, 'mushroom': 8124, 'news': 39644, 'spam': 4601, 'wine-red': 1599, 'wine-white': 4898, 'yeast': 1484
}

def run_CTGAN(dataset: str = 'adult',
              n_epochs: int = 10,
              n_samples: int = 100,
              in_folder: str = './datasets',
              out_folder: str = './experiments',
              verbose: bool = False) -> Tuple[pd.DataFrame, pd.DataFrame]:
    # df_raw: pd.DataFrame = load_demo() if dataset == 'adult' else pd.read_csv(
    #     filepath_or_buffer=f"./datasets/{dataset}.csv", na_values='?')
    df_raw: pd.DataFrame = pd.read_csv(
        filepath_or_buffer=f"{in_folder}/{dataset}.csv", skipinitialspace=True, na_values='?', skip_blank_lines=True)
    df_pre: pd.DataFrame  # pandas DataFrame to hold preprocessed data
    synthesizer: CTGANSynthesizer  # the CTGAN data synthesizer
    df_sam: pd.DataFrame  # to store the samples (i.e., the synthetic data)
    filename: str = f"{dataset}_CTGAN_{n_epochs}"

    # data preprocessing
    df_pre = PreProcessor.drop_vars(dataset=dataset, df=df_raw)
    df_pre = PreProcessor.replace_miss_values_by_nans(df=df_pre, dataset=dataset)
    df_pre = PreProcessor.drop_nans(df=df_pre)
    # create the synthesizer
    synthesizer = CTGANSynthesizer(epochs=n_epochs)
    # learn from the data distribution
    synthesizer.fit(train_data=df_pre, discrete_columns=Metadata.discrete_vars(dataset=dataset, df=df_pre))
    # generate synthetic data
    df_sam = synthesizer.sample(n=n_samples)
    if verbose:
        log: _io.TextIOWrapper = open(file=f"{out_folder}/{filename}.txt", mode='w+')

        log.write(f"{'--- CTGAN ---' * 3}\n")
        log.write(f"dataset: {dataset}\n")
        log.write(f"raw shape: {df_raw.shape}\n")
        log.write(f"preprocessing shape: {df_pre.shape}\n")
        log.write(f"n_epochs: {n_epochs}\n")
        log.write(f"n_samples: {n_samples}\n")
        # log.write("samples:\n")
        # log.write(df_sam.head())
        # log.write("\n")
        # log.write("...\n")
        # log.write(df_sam.tail())
        # log.write("\n")
        log.write(f"profiling: {profiler(df=df_sam, discrete_vars=Metadata.discrete_vars(dataset=dataset, df=df_sam))}")
        log.write("\n")
        # log.write(f"are the pandas dataframes equals? {df_pre.equals(other=df_sam)}\n")
    # open(file=f"{out_folder}/{filename}_pre.txt", mode='w+').write(
    #     f"profiling: {profiler(df=df_pre, discrete_vars=Metadata.discrete_vars(dataset=dataset, df=df_pre))}")
    # open(file=f"{out_folder}/{filename}_sam.txt", mode='w+').write(
    #     f"profiling: {profiler(df=df_sam, discrete_vars=Metadata.discrete_vars(dataset=dataset, df=df_sam))}")
    df_pre.to_csv(path_or_buf=f"{out_folder}/{filename}_pre.csv", index=False)
    df_sam.to_csv(path_or_buf=f"{out_folder}/{filename}_sam.csv", index=False)
    return df_pre, df_sam

def run_tabulator(dataset: str = 'adult',
                  ampu_rate: float = 0.2,
                  encoder_type: str = 'label',
                  algo: str = 'tabulator',
                  loss: str = 'mse',
                  batch_size: int = 128,
                  n_iterations: int = 1000,
                  n_samples: int = 100,
                  in_folder: str = './datasets',
                  out_folder: str = './experiments',
                  verbose: bool = False) -> Tuple[pd.DataFrame, pd.DataFrame]:
    # df_raw: pd.DataFrame = load_demo() if dataset == 'adult' else pd.read_csv(
    #     filepath_or_buffer=f"./datasets/{dataset}.csv")
    df_raw: pd.DataFrame = pd.read_csv(
        filepath_or_buffer=f"{in_folder}/{dataset}.csv", skipinitialspace=True, na_values='?', skip_blank_lines=True)
    df_pre: pd.DataFrame  # pandas DataFrame to hold preprocessed data
    df_enc: pd.DataFrame  # pandas DataFrame to hold the encoded and the non-encoded data
    df_sam: pd.DataFrame  # to store the samples (i.e., the synthetic data) in a pandas DataFrame
    samples: np.ndarray  # the samples (i.e., the synthetic data)
    generator: TabularDataGenerator
    new_discrete_vars: Union[List[str], List[int]]
    filename: str = f"{dataset}_{ampu_rate}_{encoder_type}_{algo}_{batch_size}_{loss}_{n_iterations}"

    # data preprocessing
    df_pre = PreProcessor.drop_vars(dataset=dataset, df=df_raw)
    df_pre = PreProcessor.replace_miss_values_by_nans(df=df_pre, dataset=dataset)
    df_pre = PreProcessor.drop_nans(df=df_pre)
    # encoding the discrete variables
    if encoder_type == 'one-hot':
        # data transformation that looks like one-hot encoding
        df_enc = get_dummies_fit_transform(
            data=df_pre, discrete_vars=Metadata.discrete_vars(dataset=dataset, df=df_pre))
        # list of the new discrete variables, which came from the original discrete variables
        new_discrete_vars = [var for var in df_enc if var not in df_pre.columns]
        # replace each zero of one-hot encoding with minus one
        df_enc = PreProcessor.replace_values(
            df=df_enc, to_replace={new_discrete_var: {0: -1} for new_discrete_var in new_discrete_vars})
    else:  # 'label' --> default encoder
        df_enc, label_encoders = label_encoders_fit_transform(
            data=df_pre, discrete_vars=Metadata.discrete_vars(dataset=dataset, df=df_pre))
    # create an instance of the generator
    generator = TabularDataGenerator(
        data=df_enc.to_numpy(),
        algo=algo,
        algo_parameters={'miss_rate': ampu_rate, 'batch_size': batch_size, 'loss': loss, 'n_iterations': n_iterations})
    # logging some execution info
    if verbose:
        logging.basicConfig(filename=f"{out_folder}/{filename}.txt", level=logging.INFO)
        logging.info(f"{'--- tabulatorSGAIN ---' * 3}")
        logging.info(f"dataset: {dataset}")
        logging.info(f"amputation rate: {ampu_rate}")
        logging.info(f"encoder type: {encoder_type}")
        logging.info(f"raw shape: {df_raw.shape}")
        logging.info(f"preprocessing shape: {df_pre.shape}")
        logging.info(f"encoded shape: {df_enc.shape}")
        logging.info(f"algorithm: {algo}")
        logging.info(f"batch size: {batch_size}")
        logging.info(f"loss: {loss}")
        logging.info(f"n_iterations: {n_iterations}")
        logging.info(f"n_samples: {n_samples}")
    # sampling (i.e., get the samples)
    samples = generator.sampler(n_samples=n_samples)
    # decoding the discrete variables
    if encoder_type == 'one-hot':
        df_sam = pd.DataFrame(data=samples, columns=df_enc.columns)
        # invert the replacement of each zero of one-hot encoding with minus one
        df_sam = PreProcessor.replace_values(
            df=df_sam, to_replace={new_discrete_var: {-1: 0} for new_discrete_var in new_discrete_vars})
        # data transformation to invert (i.e., to revert) the one that looks line one-hot encoding
        df_sam = get_dummies_inverse_transform(
            dataset=dataset,
            # data=pd.DataFrame(data=samples, columns=df_enc.columns),
            data=df_sam,
            discrete_vars=Metadata.discrete_vars(dataset=dataset, df=df_pre),
            vars_order=df_pre.columns)
    else:  # 'label' --> default encoder
        df_sam = label_encoders_inverse_transform(
            dataset=dataset,
            data=pd.DataFrame(data=samples, columns=df_enc.columns),
            label_encoders=label_encoders)
    if verbose:
        # logging.info("samples:")
        # logging.info(df_sam.head())
        # logging.info("...")
        # logging.info(df_sam.tail())
        logging.info(
            f"profiling: {profiler(df=df_sam, discrete_vars=Metadata.discrete_vars(dataset=dataset, df=df_sam))}")
        # logging.info(f"are the pandas dataframes equals? {df_pre.equals(other=df_sam)}")
    # open(file=f"{out_folder}/{filename}_pre.txt", mode='w+').write(
    #     f"profiling: {profiler(df=df_pre, discrete_vars=Metadata.discrete_vars(dataset=dataset, df=df_pre))}")
    # open(file=f"{out_folder}/{filename}_sam.txt", mode='w+').write(
    #     f"profiling: {profiler(df=df_sam, discrete_vars=Metadata.discrete_vars(dataset=dataset, df=df_sam))}")
    df_pre.to_csv(path_or_buf=f"{out_folder}/{filename}_pre.csv", index=False)
    df_sam.to_csv(path_or_buf=f"{out_folder}/{filename}_sam.csv", index=False)
    return df_pre, df_sam

def main(args: Namespace) -> None:
    algos: OrderedSet[str] = OrderedSet([algo.strip() for algo in args.algos.split(',')])
    # TODO: GET RID OF HARDCODED
    algos_set: OrderedSet[str] = OrderedSet(['CTGAN', 'tabulator', 'tabulator-CP', 'tabulator-GP'])
    datasets: OrderedSet[str] = OrderedSet([dataset.strip() for dataset in args.datasets.split(',')])
    # TODO: GET RID OF HARDCODED
    datasets_set: OrderedSet[str] = OrderedSet(['breast', 'credit', 'eeg', 'iris', 'letter', 'news',
                                                'spam', 'wine-red', 'wine-white', 'yeast'])
    ampu_rates: OrderedSet[str] = OrderedSet([ampu_rate.strip() for ampu_rate in args.ampu_rates.split(',')])
    dataset: str
    algo: str
    ampu_rate: str
    ampu_rate_tmp: float
    df_pre: pd.DataFrame
    df_sam: pd.DataFrame

    if not algos.issubset(algos_set):
        raise ValueError(f"In terms of algorithms, expecting a subset of {algos_set} but got: {algos}.")
    if not datasets.issubset(datasets_set):
        raise ValueError(f"In terms of datasets, expecting a subset of {datasets_set} but got: {datasets}.")
    for dataset in datasets:
        for ampu_rate in ampu_rates:
            try:
                ampu_rate_tmp = float(ampu_rate)
                if ampu_rate_tmp < 0.00 or ampu_rate_tmp > 1.00:
                    raise ValueError(
                        f"Expecting an amputation rate within the interval [0.00, 1.00] but got: {ampu_rate_tmp}.")
            except ValueError:
                print(f"Expecting an amputation rate within the interval [0.00, 1.00] but got: {ampu_rate}.")
                exit(1)
            for algo in algos:
                print(f"{dataset} :: {ampu_rate} :: {algo}")
                df_pre, df_sam = run_tabulator(dataset=dataset,
                                               ampu_rate=ampu_rate_tmp,
                                               encoder_type='one-hot',
                                               algo=algo,
                                               batch_size=128,
                                               n_iterations=1000,
                                               n_samples=DATASETS[dataset],
                                               verbose=False)

if __name__ == "__main__":
    parser: ArgumentParser = ArgumentParser()

    parser.add_argument(
        '--algos',
        help="a csv list of the algorithms to run (e.g., 'CTGAN, tabulator, tabulator-CP, tabulator-GP')",
        # choices=['CTGAN', 'tabulator', 'tabulator-CP', 'tabulator-GP'],
        default='tabulator',
        type=str)
    parser.add_argument(
        '--datasets',
        help="a csv list of datasets short names",
        # choices=['breast', 'cover-type', 'credit', 'eeg', 'iris', 'letter', 'mushroom',
        #          'news', 'spam', 'wine-red', 'wine-white', 'yeast'],
        default='letter',
        type=str)
    parser.add_argument(
        '--ampu_rates',
        help="a csv list of amputation rates ([0.00, 1.00])",
        default='0.20',
        type=str)
    parser.add_argument(
        '--batch_size',
        help="number of samples in mini-batch",
        default=128,
        type=int)
    parser.add_argument(
        '--hint_rate',  # NOTE: the algorithms SGAIN, WSGAIN-CP, and WSGAIN-GP do NOT use this parameter,
        help='hint probability',  # it is here just because the GAIN algorithm requires the `hint_rate` parameter
        default=0.9,
        type=float)
    parser.add_argument(
        '--alpha',
        help="hyper-parameter to compute generator's loss",
        default=100,
        type=float)
    parser.add_argument(
        '--lambd',
        help="hyper-parameter to compute critic's loss",
        default=10,
        type=float)
    parser.add_argument(
        '--clip_value',
        help="clip (penalty) value",
        default=0.01,
        type=float)
    parser.add_argument(
        '--optimizer',
        help="solvers' optimizer",
        choices=['Adam', 'GDA', 'RMSProp'],
        default='Adam',
        type=str)
    parser.add_argument(
        '--learn_rate',
        help="optimizer's learning rate",
        default=1e-3,
        type=float)
    parser.add_argument(
        '--beta_1',
        help="Adam optimizer's hyper-parameter (1st moment estimates)",
        default=0.900,
        type=float)
    parser.add_argument(
        '--beta_2',
        help="Adam optimizer's hyper-parameter (2nd moment estimates)",
        default=0.999,
        type=float)
    parser.add_argument(
        '--decay',
        help="RMSProp optimizer's hyper-parameter (discounting factor for the history/coming gradient)",
        default=0.900,
        type=float)
    parser.add_argument(
        '--momentum',
        help="RMSProp optimizer's hyper-parameter (a scalar tensor)",
        default=0.000,
        type=float)
    parser.add_argument(
        '--epsilon',
        help="Adam hyper-parameter to ensure numerical stability or RMSProp hyper-parameter to avoid zero denominator",
        default=1e-08,
        type=float)
    parser.add_argument(
        '--n_iterations',
        help="number of training iterations",
        default=10000,
        type=int)
    parser.add_argument(
        '--n_critic',
        help="number of additional iterations to train the critic",
        default=5,
        type=int)
    parser.add_argument(
        '--n_runs',
        help="number of runs",
        default=3,
        type=int)
    parser.add_argument(
        '--verbose',
        help="to control verbosity",
        choices=['False', 'True'],  # `bool` type does NOT work as expected
        default='False',  # `bool` type does NOT work as expected
        type=str)  # `bool` type does NOT work as expected

    main(args=parser.parse_args())  # rock 'n roll

# python main.py --algos="CTGAN,tabulator,tabulator-CP,tabulator-GP" --datasets="iris,yeast" --ampu_rates="0.20" --optimizer=GDA --learn_rate=0.001 --n_iterations=1000 --n_runs=3


input_file = 'input.txt'
XRegister = 1
Cycle = 0
CycleDict = {}

def SaveCycle():
    global XRegister
    global Cycle
    global CycleDict
    CycleDict[Cycle] = XRegister

def AddInstruction(value):
    global XRegister
    global Cycle
    NoOperationInstruction()
    Cycle += 1
    SaveCycle()
    XRegister += value

def NoOperationInstruction():
    global XRegister
    global Cycle
    Cycle += 1
    SaveCycle()

instructions = [i.split(' ') for i in open(input_file).read().split('\n')]

for i in instructions:
    if i[0] == 'noop':
        NoOperationInstruction()
    elif i[0] == 'addx':
        AddInstruction(int(i[1]))

start = 20
step = 40
importantCycles = {}
for i in range(start, 9999999, step):
    if(i not in CycleDict):
        break
    importantCycles[i] = CycleDict[i]

s = 0
for i in importantCycles:
    s += i * importantCycles[i]

print(s)

# print(XRegister, Cycle, '\n', importantCycles)

# print('\n\n\n', CycleDict)


# Definition for singly-linked list.
class ListNode:
    def __init__(self, val=0, next=None):
        self.val = val
        self.next = next

class Solution:
    def checkPalindrome(self, l1: ListNode):
      t=[]
      while l1 is not None:
        t.append(l1.val)
        l1=l1.next
      s="".join(t)
      return (s==s[::-1])


# Databricks notebook source
from pyspark.sql import SparkSession
from pyspark.sql.functions import *

#COMMENTS -----------CREATING SPARK SESSION OBJECT
spark_sc=SparkSession.Builder().appName('Ex01').master('local[*]').getOrCreate()

# COMMAND ----------READ FILE FROM DATA BRICKS LOCATION SAVED FILE

read_data=spark_sc.read.format('csv').option('header',True).option('inferschema',True).load('/FileStore/tables/5000_Sales_Records.csv')

# COMMAND ----------DROPPING FEW UNWANTED COLUMNS FROM DATA

read_filter = read_data.drop('Region', 'Country', 'Order Priority', 'Ship Date', 'Total Profit', 'Total Cost', 'Unit Cost')

# COMMAND ----------RENAMING COLUMN NAMES BY REPLACING SPACES WITH UNDERSCORE AND USING LOWER CASES

df_1 = read_filter.withColumnRenamed('Order Id','order_id') \
.withColumnRenamed('Item Type','item_type') \
.withColumnRenamed('Sales Channel','sales_channel') \
.withColumnRenamed('Order Date','order_date') \
.withColumnRenamed('Units Sold','units_sold') \
.withColumnRenamed('Unit Price','unit_price') \
.withColumnRenamed('Total Revenue','total_revenue')

# COMMAND ----------CONVERTING DATA TYPES FOR COLUMNS

df_2 = df_1.withColumn('units_sold',col('units_sold').cast('float')) \
.withColumn('unit_price',col('unit_price').cast('float')) \
.withColumn('total_revenue',col('total_revenue').cast('float')) \
.withColumn('order_date',to_date(col('order_date'), 'M/d/yyyy'))

# COMMAND ----------CREATING TABLE IN DATABRICKS

df_2.repartition(3).write.mode('overwrite').saveAsTable('Table_Sales')


from gtts import gTTS
import googletrans
from ui import Ui_MainWindow
from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtWidgets import QMessageBox
import sys
from playsound import playsound
import os
from threading import Thread
import PyPDF2

def ReverseDict(d):
    key_dict = []
    value_dict = []
    for i in d:
        value_dict.append(i)
        key_dict.append(d.get(i))

    return dict(zip(key_dict, value_dict))

def Capitilization(s):
	place = s[1:]
	s = s[0].upper() + place
	return s

class GttsWindow(QtWidgets.QMainWindow):

	def __init__(self,*args,**kwargs):
		super().__init__(*args,**kwargs)
		self.langs = ReverseDict(googletrans.LANGUAGES)
		self.ui = Ui_MainWindow()
		self.ui.setupUi(self)

		for i in self.langs:
			self.ui.Language_Selector.addItem(Capitilization(i))

		self.ui.Convert.clicked.connect(self.ConvertFile)
		self.ui.ReadPDFButton.clicked.connect(self.ReadFromPDFFile)
		self.ui.ReadFileButton.clicked.connect(self.ReadFromTextFile)

	def ConvertFile(self):

		msg = QMessageBox()
		msg.setWindowTitle("gTTs Error")
		msg.setIcon(QMessageBox.Critical)
		msg.setStandardButtons(QMessageBox.Close)
		msg.setDefaultButton(QMessageBox.Close)
		icon = QtGui.QIcon()
		icon.addPixmap(QtGui.QPixmap("gtts.ico"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
		msg.setWindowIcon(icon)

		if(str(self.ui.textEdit.toPlainText())!=""):
			print("here")
			tts = gTTS(str(self.ui.textEdit.toPlainText()) , lang=self.langs.get(self.ui.Language_Selector.currentText().lower()))
			print("here")
			self.audiopath = str(QtWidgets.QFileDialog.getSaveFileName(self,"Save MP3 File" , filter="MP3 Files (*.mp3)")[0])
			self.audiopath =  self.audiopath.replace(r"/" , chr(92))
			if(self.audiopath != ""):
				self.__SaveThread(tts)
				m = QMessageBox()
				m.setWindowTitle("gTTs Preview")
				m.setText(f"Do you want to preview '{str(self.audiopath.split(chr(92))[len(self.audiopath.split(chr(92)))-1])}' ?")
				m.setIcon(QMessageBox.Question)
				m.setStandardButtons(QMessageBox.No | QMessageBox.Yes)
				m.setDefaultButton(QMessageBox.No)
				icon = QtGui.QIcon()
				icon.addPixmap(QtGui.QPixmap("gtts.ico"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
				m.setWindowIcon(icon)
				m.buttonClicked.connect(self.PreviewSound)
				m.exec_()
			else:
				m = QMessageBox()
				m.setWindowTitle("gTTs Error")
				m.setText(f"File not Saved")
				m.setIcon(QMessageBox.Critical)
				m.setStandardButtons(QMessageBox.Close)
				m.setDefaultButton(QMessageBox.Close)
				icon = QtGui.QIcon()
				icon.addPixmap(QtGui.QPixmap("gtts.ico"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
				m.setWindowIcon(icon)
				m.buttonClicked.connect(self.PreviewSound)
				m.exec_()
		else:
			msg.setText("No Text Inputted Into Input Field.")
			msg.exec_()

	def PreviewSound(self,pt):
		if(pt.text() == "&Yes"):
			if("tts.mp3" in list(os.listdir())):
				os.remove("tts.mp3")
			with open(str(self.audiopath) , "rb") as g:
				with open("tts.mp3" , "wb") as h:
					h.write(g.read())
					h.close()
					g.close()
			playsound("tts.mp3", False)
		else:
			pass

	def ReadFromTextFile(self):
		path = str(QtWidgets.QFileDialog.getOpenFileName(self , caption="Choose Text File" , filter="Text files (*.txt)")[0]).replace(r"/" , chr(92))
		try:
			with open(path , "r") as f:
				self.ui.textEdit.setText(f.read())
				f.close()
		except:
				m = QMessageBox()
				m.setWindowTitle("gTTs Error")
				m.setText("Text File Not Chosen.")
				m.setIcon(QMessageBox.Critical)
				m.setStandardButtons(QMessageBox.Close)
				m.setDefaultButton(QMessageBox.Close)
				icon = QtGui.QIcon()
				icon.addPixmap(QtGui.QPixmap("gtts.ico"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
				m.setWindowIcon(icon)
				m.exec_()

	def ReadFromPDFFile(self):
		path = str(QtWidgets.QFileDialog.getOpenFileName(self , caption="Choose PDF File" , filter="PDF files (*.pdf)")[0]).replace(r"/" , chr(92))
		try:
			otpt = """

			"""
			with open(path, "rb") as pdf_file:
				pdf_reader = PyPDF2.PdfFileReader(pdf_file)
				for i in range(0,pdf_reader.getNumPages()):
					otpt = otpt + pdf_reader.getPage(i).extractText()+"\n"
			self.ui.textEdit.setText(otpt)
		except:
				m = QMessageBox()
				m.setWindowTitle("gTTs Error")
				m.setText("PDF File Not Chosen.")
				m.setIcon(QMessageBox.Critical)
				m.setStandardButtons(QMessageBox.Close)
				m.setDefaultButton(QMessageBox.Close)
				icon = QtGui.QIcon()
				icon.addPixmap(QtGui.QPixmap("gtts.ico"), QtGui.QIcon.Normal, QtGui.QIcon.Off)
				m.setWindowIcon(icon)
				m.exec_()

	def __SaveThread(self,tts_object):
		print("saving")
		tts_object.save(self.audiopath)

if __name__ == "__main__":
	app = QtWidgets.QApplication(sys.argv)
	ui = GttsWindow()
	ui.show()
	sys.exit(app.exec_())


from __future__ import annotations
from typing import TYPE_CHECKING, Optional
import logging
from ezdxf.lldxf import validator, const
from ezdxf.lldxf.attributes import (
    DXFAttributes,
    DefSubclass,
    DXFAttr,
    RETURN_DEFAULT,
    XType,
    group_code_mapping,
)
from ezdxf.math import NULLVEC, Z_AXIS
from .dxfentity import base_class
from .dxfgfx import acdb_entity
from .factory import register_entity
from .polygon import DXFPolygon
from .gradient import Gradient, GradientType

if TYPE_CHECKING:
    from ezdxf.lldxf.tagwriter import AbstractTagWriter
    from ezdxf.colors import RGB

__all__ = ["MPolygon"]
logger = logging.getLogger("ezdxf")

acdb_mpolygon = DefSubclass(
    "AcDbMPolygon",
    {
        # MPolygon: version
        "version": DXFAttr(
            70,
            default=1,
            validator=validator.is_integer_bool,
            fixer=RETURN_DEFAULT,
        ),
        # x- and y-axis always equal 0, z-axis represents the elevation:
        "elevation": DXFAttr(10, xtype=XType.point3d, default=NULLVEC),
        "extrusion": DXFAttr(
            210,
            xtype=XType.point3d,
            default=Z_AXIS,
            validator=validator.is_not_null_vector,
            fixer=RETURN_DEFAULT,
        ),
        # Hatch pattern name:
        "pattern_name": DXFAttr(2, default=""),
        # Solid fill color as ACI
        "fill_color": DXFAttr(
            63,
            default=const.BYLAYER,
            optional=True,
            validator=validator.is_valid_aci_color,
            fixer=RETURN_DEFAULT,
        ),
        # MPolygon: Solid-fill flag:
        # 0 = lacks solid fill
        # 1 = has solid fill
        "solid_fill": DXFAttr(
            71,
            default=0,
            validator=validator.is_integer_bool,
            fixer=RETURN_DEFAULT,
        ),
        # Hatch style tag is not supported for MPOLYGON!?
        # I don't know in which order this tag has to be exported.
        # TrueView does not accept this tag in any place!
        # BricsCAD supports this tag!
        "hatch_style": DXFAttr(
            75,
            default=const.HATCH_STYLE_NESTED,
            validator=validator.is_in_integer_range(0, 3),
            fixer=RETURN_DEFAULT,
            optional=True,
        ),
        # Hatch pattern type ... see HATCH
        "pattern_type": DXFAttr(
            76,
            default=const.HATCH_TYPE_PREDEFINED,
            validator=validator.is_in_integer_range(0, 3),
            fixer=RETURN_DEFAULT,
        ),
        # Hatch pattern angle (pattern fill only) in degrees:
        "pattern_angle": DXFAttr(52, default=0),
        # Hatch pattern scale or spacing (pattern fill only):
        "pattern_scale": DXFAttr(
            41,
            default=1,
            validator=validator.is_not_zero,
            fixer=RETURN_DEFAULT,
        ),
        # MPolygon: Boundary annotation flag:
        # 0 = boundary is not an annotated boundary
        # 1 = boundary is an annotated boundary
        "annotated_boundary": DXFAttr(
            73,
            default=0,
            optional=True,
            validator=validator.is_integer_bool,
            fixer=RETURN_DEFAULT,
        ),
        # Hatch pattern double flag (pattern fill only) .. see HATCH
        "pattern_double": DXFAttr(
            77,
            default=0,
            validator=validator.is_integer_bool,
            fixer=RETURN_DEFAULT,
        ),
        # see ... HATCH
        "pixel_size": DXFAttr(47, optional=True),
        "n_seed_points": DXFAttr(
            98,
            default=0,
            validator=validator.is_greater_or_equal_zero,
            fixer=RETURN_DEFAULT,
        ),
        # MPolygon: offset vector in OCS ???
        "offset_vector": DXFAttr(11, xtype=XType.point2d, default=(0, 0)),
        # MPolygon: number of degenerate boundary paths (loops), where a
        # degenerate boundary path is a border that is ignored by the hatch:
        "degenerated_loops": DXFAttr(99, default=0),
    },
)
acdb_mpolygon_group_code = group_code_mapping(acdb_mpolygon)

@register_entity
class MPolygon(DXFPolygon):
    """DXF MPOLYGON entity

    The MPOLYGON is not a core DXF entity, and requires a CLASS definition.

    """

    DXFTYPE = "MPOLYGON"
    DXFATTRIBS = DXFAttributes(base_class, acdb_entity, acdb_mpolygon)
    MIN_DXF_VERSION_FOR_EXPORT = const.DXF2000
    LOAD_GROUP_CODES = acdb_mpolygon_group_code

    def preprocess_export(self, tagwriter: AbstractTagWriter) -> bool:
        if self.paths.has_edge_paths:
            logger.warning(
                "MPOLYGON including edge paths are not exported by ezdxf!"
            )
            return False
        return True

    def export_entity(self, tagwriter: AbstractTagWriter) -> None:
        """Export entity specific data as DXF tags."""
        super().export_entity(tagwriter)
        tagwriter.write_tag2(const.SUBCLASS_MARKER, acdb_mpolygon.name)
        dxf = self.dxf
        dxf.export_dxf_attribs(
            tagwriter,
            [  # tag order is important!
                "version",
                "elevation",
                "extrusion",
                "pattern_name",
                "solid_fill",
            ],
        )
        self.paths.export_dxf(tagwriter, self.dxftype())
        # hatch_style not supported?
        dxf.export_dxf_attribs(
            tagwriter,
            [
                # "hatch_style",  # not supported by MPolygon ???
                "pattern_type",
            ],
        )
        if dxf.solid_fill == 0:  # export pattern
            dxf.export_dxf_attribs(
                tagwriter, ["pattern_angle", "pattern_scale", "pattern_double"]
            )

        dxf.export_dxf_attribs(
            tagwriter,
            [
                "annotated_boundary",
                "pixel_size",
            ],
        )
        if dxf.solid_fill == 0:  # export pattern
            if self.pattern:
                self.pattern.export_dxf(tagwriter, force=True)
            else:  # required pattern length tag!
                tagwriter.write_tag2(78, 0)
        if tagwriter.dxfversion > const.DXF2000:
            dxf.export_dxf_attribs(tagwriter, "fill_color")
        dxf.export_dxf_attribs(tagwriter, "offset_vector")
        self.export_degenerated_loops(tagwriter)
        self.export_gradient(tagwriter)

    def export_degenerated_loops(self, tagwriter: AbstractTagWriter):
        self.dxf.export_dxf_attribs(tagwriter, "degenerated_loops")

    def export_gradient(self, tagwriter: AbstractTagWriter):
        if tagwriter.dxfversion < const.DXF2004:
            return
        if self.gradient is None:
            self.gradient = Gradient(kind=0, num=0, type=0)
        self.gradient.export_dxf(tagwriter)

    def set_solid_fill(
        self, color: int = 7, style: int = 1, rgb: Optional[RGB] = None
    ):
        """Set :class:`MPolygon` to solid fill mode and removes all gradient and
        pattern fill related data.

        Args:
            color: :ref:`ACI`, (0 = BYBLOCK; 256 = BYLAYER)
            style: hatch style is not supported by MPOLYGON, just for symmetry
                to HATCH
            rgb: true color value as (r, g, b)-tuple - has higher priority
                than `color`. True color support requires DXF R2004+

        """
        # remove existing gradient and pattern fill
        self.gradient = None
        self.pattern = None
        self.dxf.solid_fill = 1

        # if a gradient is present, the solid fill_color is ignored
        self.dxf.fill_color = color
        self.dxf.pattern_name = "SOLID"
        self.dxf.pattern_type = const.HATCH_TYPE_PREDEFINED
        if rgb is not None:
            self.set_solid_rgb_gradient(rgb)

    def set_solid_rgb_gradient(self, rgb: RGB):
        """Set solid fill color as gradient of a single RGB color.
        This disables pattern fill!

        (internal API)
        """
        self.gradient = Gradient(kind=1, num=2, type=GradientType.LINEAR)
        self.gradient.color1 = rgb
        self.gradient.color2 = rgb


import math
class Point:
    __x = 0
    __y = 0
    def __init__(self,x,y):
        self.__x = x
        self.__y = y
    def distance(self,point):
        distance = math.sqrt(math.pow((point.__x-self.__x),2)+math.pow((point.__y-self.__y),2))
        return distance
    def __add__(self,point):
        x = self.__x + point.__x
        y = self.__y + point.__y
        print('(',x,',',y,')')

p1 = Point(1,1)
p2 = Point(2,2)

print(p1.distance(p2))
p1+p2


from eos.tests import TestBase

class Test(TestBase):
    def setUp(self):
        TestBase.setUp(self)
        self.skill = "Gas Cloud Harvesting"

    # Allows use of one gas cloud harvester per level.

    def test_maxGroupActive_moduleGasCloudHarvester(self):
        self.buildTested = 0
        attr = "maxGroupActive"
        item = "Gas Cloud Harvester I"
        iLvl = 1
        iIngame = 1
        fLvl = 4
        fIngame = 4
        iEos = self.getItemAttr(attr, item, skill=(self.skill, iLvl))
        fEos = self.getItemAttr(attr, item, skill=(self.skill, fLvl))
        dIngame = fIngame - iIngame
        dEos = fEos - iEos
        self.assertAlmostEquals(dEos, dIngame)

    def test_maxGroupActive_moduleOther(self):
        self.buildTested = 0
        attr = "maxGroupActive"
        item = "10MN Afterburner I"
        iLvl = 1
        iIngame = 0
        fLvl = 4
        fIngame = 0
        iEos = self.getItemAttr(attr, item, skill=(self.skill, iLvl))
        fEos = self.getItemAttr(attr, item, skill=(self.skill, fLvl))
        dIngame = fIngame - iIngame
        dEos = fEos - iEos
        self.assertAlmostEquals(dEos, dIngame)


import sys

input = lambda: sys.stdin.readline().rstrip()

def make_false(sq, arr, start, end):
    if start%sq==0:
        i=start//sq
    else:
        i=start//sq+1

    while True:
        tmp = sq * i
        if tmp > end:
            break
        arr[tmp - start] = False
        i += 1

if __name__ == "__main__":
    start, end = map(int, input().split())
    arr = [True for _ in range(end - start + 1)]
    i = 2
    while True:
        sq = i ** 2
        if sq > end:
            break
        make_false(sq, arr, start, end)
        i += 1

    count = 0
    for i in range(end - start + 1):
        if arr[i]:
            count += 1

    print(count)

백준/Gold/1016. 제곱 ㄴㄴ 수/제곱 ㄴㄴ 수.py


#!/usr/bin/env python
# _*_ coding: utf-8 _*_
#luz maria calderon aguilera
#1-b
import matplotlib.pyplot as plt
import numpy as np
import math
x=np.linspace(-1,5,100)
g=2*x**2-8*x-11
plt.plot(g,x, linewidth=3,color='b',label='grafica')
plt.legend()
plt.title('$2x^2-8x-11$')
plt.xlabel('algo')
plt.ylabel('una funcion de algo')
plt.show()
plt.savefig('grafica.png')