from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time

import lib.db
from lib.helper import remove_tags, open_selenium
from lib.log import log_text as log

url = "https://2e.aonprd.com/Ancestries.aspx"

def upload_heritage_data():
    log("Starting Heritage Upload Preperation")
    heritage_data = organize_heritage_data()
    log("Preparation Done")

    log("Clearing Table")
    conn, row_count, result = lib.db.query_database("DELETE FROM official_heritages;", get_result=True, close_conn=False)

    log("Starting INSERT Process")
    for heritage in heritage_data:
        log("Inserting " + heritage + " Into Database")
        conn = lib.db.query_database("INSERT INTO official_heritages VALUES (" + heritage + ");", connection=conn, close_conn=False)[0]

    log("Commiting Database Changes")
    conn.commit()
    log("Closing Connection")
    conn.close()

def grab_heritage_data():
    heritage_output = []

    log("Opening Browser")
    driver = open_selenium()
    log("Going to Page: " + url)
    driver.get(url)
    log("Waiting for Page to Load")
    time.sleep(5)

    log("Getting Page Source")
    html = driver.page_source
    log("Setting up BeautifulSoup with Source")
    soup = BeautifulSoup(html, "html.parser")

    log("Finding Initial HTML Container")
    container = soup.find(id="ctl00_RadDrawer1_Content_MainContent_DetailedOutput")
    log("Finding All Categories in Container")
    name_list = container.find_all("h2")

    for item in name_list:
      log("Grabbing Name in Category")
      elements = item.text.split("\n")
      log("Found: " + elements[0])

      log("Getting All Links in Category")
      links = item.find_all("a")
      output_link = ""
      log("Finding Ancestry Page Link")
      for link in links:
        if link.get("href").startswith("Ancestries.aspx"):
          output_link = "https://2e.aonprd.com/" + link.get("href")
          log("Found: " + output_link)
          break

      log("Opening Ancestry Page")
      ancestry_driver = open_selenium()
      ancestry_driver.get(output_link)
      log("Waiting for Page to Load")
      time.sleep(5)

      log("Getting Ancestry Page Source")
      ancestry_html = ancestry_driver.page_source
      log("Setting up BeautifulSoup with Page Source")
      ancestry_soup = BeautifulSoup(ancestry_html, "html.parser")

      log("Finding Sub Navigation")
      sub_nav_container = ancestry_soup.find(id="ctl00_RadDrawer1_Content_MainContent_SubNavigation")
      sub_nav_list = sub_nav_container.find_all("h2")
      log("Getting All Sub Navigation Headings")

      heritage_list_link = ""

      log("Searching Headings for Heritage Link")
      for nav in sub_nav_list:
          nav_links = nav.find_all("a")
          for n in nav_links:
              if n.get("href").startswith("Heritages.aspx"):
                  heritage_list_link = "https://2e.aonprd.com/" + n.get("href")
                  log(f"Found Heritage Link for {elements[0]}: {heritage_list_link}")

      log("Closing Ancestry Browser. Opening Heritage Browser")
      ancestry_driver.close()
      heritage_driver = open_selenium()
      heritage_driver.get(heritage_list_link)
      log("Waiting for Page to Load")
      time.sleep(5)

      log("Setting up BeautifulSoup with Page Source")
      heritage_html = heritage_driver.page_source
      heritage_soup = BeautifulSoup(heritage_html, "html.parser")

      log("Getting Heritage List Container")
      heritage_container = heritage_soup.find(id="ctl00_RadDrawer1_Content_MainContent_DetailedOutput")
      log("Getting All Headings")
      heritage_list = heritage_container.find_all("h2")

      heritage_name = ""
      heritage_link = ""
      heritage_summary = ""

      log("Starting Search for Heritages")
      i = 0
      for heritage in heritage_list:
          heritage_links = heritage.find_all("a")
          for l in heritage_links:
              if l.get("href").startswith("Heritages.aspx"):
                  heritage_name = l.text.split("\n")[0]

                  log("Found Heritage: " + heritage_name)
                  heritage_link = "https://2e.aonprd.com/" + l.get("href")
                  link_pos = heritage_html.find(l.get("href"))
                  print(f"Link Pos: {link_pos}")

                  versatile_heritage_pos = heritage_html.index("
Versatile Heritages
")
                  half_human_heritage_pos = heritage_html.find("
Half-Human Heritages")

                  if half_human_heritage_pos == -1 or link_pos < half_human_heritage_pos:
                      start_pos = heritage_html.index("
", link_pos) + len("
")
                  else:
                      first_break_pos = heritage_html.index("
", link_pos) + len("
")
                      start_pos = heritage_html.index("
", first_break_pos) + len("
")

                  h3_pos = heritage_html.find("<h3", start_pos)
                  br_pos = heritage_html.find("
", start_pos)
                  end_pos = 0
                  print(f"H3 Pos: {h3_pos}; BR Pos: {br_pos}")

                  if h3_pos < br_pos and h3_pos != -1:
                      end_pos = h3_pos
                  elif br_pos < h3_pos and br_pos != -1:
                      end_pos = br_pos
                  elif br_pos != -1 and h3_pos == -1:
                      end_pos = br_pos
                  elif h3_pos != -1 and br_pos == -1:
                      end_pos = h3_pos

                  if end_pos > versatile_heritage_pos:
                      end_pos = versatile_heritage_pos

                  if start_pos > versatile_heritage_pos:
                      break

                  print(f"End Pos: {end_pos}; Next 50 Characters: {heritage_html[end_pos: end_pos + 50]}")

                  heritage_summary = heritage_html[start_pos:end_pos].strip()
                  print(heritage_summary)

                  if heritage_summary.find("Source") > -1:
                      end_pos += 3
                      temp_pos = heritage_html.find("Source", start_pos)
                      start_pos = heritage_html.find("
", temp_pos)
                      h3_pos = heritage_html.find("<h3", end_pos)
                      br_pos = heritage_html.find("
", end_pos)

                      if h3_pos < br_pos and h3_pos != -1:
                          end_pos = h3_pos
                      elif br_pos < h3_pos and br_pos != -1:
                          end_pos = br_pos

                      if end_pos > versatile_heritage_pos:
                          end_pos = versatile_heritage_pos

                      if start_pos > versatile_heritage_pos:
                          break

                      heritage_summary = heritage_html[start_pos:end_pos].strip()

                  if heritage_summary.find("PFS Note") > -1:
                      end_pos += 3
                      temp_pos = heritage_html.find("PFS Note", start_pos)
                      start_pos = heritage_html.find("
", temp_pos)
                      h3_pos = heritage_html.find("<h3", end_pos)
                      br_pos = heritage_html.find("
", end_pos)

                      if h3_pos < br_pos and h3_pos != -1:
                          end_pos = h3_pos
                      elif br_pos < h3_pos and br_pos != -1:
                          end_pos = br_pos

                      if end_pos > versatile_heritage_pos:
                          end_pos = versatile_heritage_pos

                      if start_pos > versatile_heritage_pos:
                          break

                      heritage_summary = heritage_html[start_pos:end_pos].strip()

                  heritage_summary = remove_tags(heritage_summary, tag_to_remove="h2", remove_inside=True)
                  heritage_summary = remove_tags(heritage_summary, tag_to_remove="table", remove_inside=True)
                  heritage_summary = remove_tags(heritage_summary, tag_to_remove="i")
                  heritage_summary = remove_tags(heritage_summary, tag_to_remove="u")
                  heritage_summary = remove_tags(heritage_summary, tag_to_remove="b")
                  heritage_summary = remove_tags(heritage_summary, tag_to_remove="a")

                  log(str([heritage_name, heritage_link, elements[0], heritage_summary]))
                  heritage_output.append([heritage_name, heritage_link, elements[0], heritage_summary])

    nav_container = soup.find(id="ctl00_RadDrawer1_Content_MainContent_Navigation")
    nav_links = nav_container.find_all("a")
    for link in nav_links:
        if link.get("href").endswith("Versatile=true"):
            versatile_heritage_link = "https://2e.aonprd.com/" + link.get("href")

            log("Opening Versatile Heritage Browser")
            versatile_heritage_driver = open_selenium()
            versatile_heritage_driver.get(versatile_heritage_link)
            log("Waiting for Page to Load")
            time.sleep(5)

            log("Setting up BeautifulSoup with Page Source")
            versatile_heritage_html = versatile_heritage_driver.page_source
            versatile_heritage_soup = BeautifulSoup(versatile_heritage_html, "html.parser")

            log("Getting Heritage List Container")
            versatile_heritage_container = versatile_heritage_soup.find(id="ctl00_RadDrawer1_Content_MainContent_DetailedOutput")
            log("Getting All Headings")
            versatile_heritage_list = versatile_heritage_container.find_all("h2")

            versatile_heritage_name = ""
            versatile_heritage_link = ""
            versatile_heritage_summary = ""

            log("Searching For Versatile Heritages")
            for heritage in versatile_heritage_list:
                vh_links = heritage.find_all("a")
                for l in vh_links:
                    if l.get("href").startswith("Ancestries.aspx"):
                        versatile_heritage_name = l.text.split("\n")[0]
                        log("Found Heritage: " + versatile_heritage_name)
                        vh_ancestry_link = "https://2e.aonprd.com/" + l.get("href")

                        log("Opening Versatile Heritage Ancestry Browser")
                        vh_ancestry_driver = open_selenium()
                        vh_ancestry_driver.get(vh_ancestry_link)
                        log("Waiting for Page to Load")
                        time.sleep(5)

                        log("Setting up BeautifulSoup with Page Source")
                        vh_ancestry_html = vh_ancestry_driver.page_source
                        vh_ancestry_soup = BeautifulSoup(vh_ancestry_html, "html.parser")

                        content_pos = vh_ancestry_soup.find(id="ctl00_RadDrawer1_Content_MainContent_DetailedOutput").sourcepos
                        vh_h1_pos = vh_ancestry_html.index("
Versatile Heritage
", content_pos)

                        vh_h2_pos = vh_ancestry_html.index("
", vh_h1_pos) + len("
")
                        break_pos_1 = vh_ancestry_html.index("
", vh_h2_pos) + len("
")
                        break_pos_2 = vh_ancestry_html.index("
", break_pos_1) + len("
")
                        break_pos_3 = vh_ancestry_html.index("
", break_pos_2) + len("
")

                        end_pos = 0
                        span_pos = vh_ancestry_html.find("", break_pos_3)
                        h3_pos =  vh_ancestry_html.find("<h3 class", break_pos_3)

                        if h3_pos == -1:
                            end_pos = span_pos
                        else:
                            if span_pos < h3_pos and span_pos != -1:
                                end_pos = span_pos
                            elif h3_pos < span_pos and h3_pos != -1:
                                end_pos = h3_pos

                        versatile_heritage_summary = vh_ancestry_html[break_pos_3:end_pos]
                        versatile_heritage_summary = remove_tags(versatile_heritage_summary, tag_to_remove="h2", remove_inside=True)
                        versatile_heritage_summary = remove_tags(versatile_heritage_summary, tag_to_remove="table", remove_inside=True)
                        versatile_heritage_summary = remove_tags(versatile_heritage_summary, tag_to_remove="i")
                        versatile_heritage_summary = remove_tags(versatile_heritage_summary, tag_to_remove="a")

                        log(str([versatile_heritage_name, vh_ancestry_link, "Versatile", versatile_heritage_summary]))
                        heritage_output.append([versatile_heritage_name, vh_ancestry_link, "Versatile", versatile_heritage_summary])

    return heritage_output

def organize_heritage_data():
    log("Getting Heritage Data")
    output = grab_heritage_data()

    organized_data = []

    log("Starting to Organize Heritage Data")
    for heritage in output:
        organized_data.append(f"\"{heritage[0]}\", \"{heritage[1]}\", \"{heritage[2]}\", \"{heritage[3]}\"")
        log(f"Added \"{heritage[0]}\", \"{heritage[1]}\", \"{heritage[2]}\", \"{heritage[3]}\" to Organized Data")

    return organized_data


import jax.numpy as np
from jax import grad, nn, random, jit
from jax.experimental import stax, optimizers
from jax.experimental.optimizers import l2_norm
from jax.numpy import linalg
from jax.experimental.stax import Dense, Relu, Tanh, Conv, MaxPool, Flatten, Softmax, LogSoftmax, Sigmoid
from jax.tree_util import tree_flatten, tree_unflatten, tree_map
from jax.nn import log_sigmoid

from mnist import mnist

from tqdm import tqdm
import itertools
import pickle

LogSigmoid = elementwise(log_sigmoid)

def model(rng):
  """Feature extraction network."""
  init_params, forward = stax.serial(
    Conv(16, (8, 8), padding='SAME', strides=(2, 2)),
    Relu,
    MaxPool((2, 2), (1, 1)),
    Conv(32, (4, 4), padding='VALID', strides=(2, 2)),
    Relu,
    MaxPool((2, 2), (1, 1)),
    Flatten,
    Dense(32),
    Relu,
    Dense(1),
    LogSigmoid,
  )
  temp, rng = random.split(rng)
  params = init_params(temp, (-1, 28, 28, 1))[1]
  return params, forward

def data_stream(rng, batch_size, X, y):
  num_complete_batches, leftover = divmod(X.shape[0], batch_size)
  num_batches = num_complete_batches + bool(leftover)
  while True:
    temp, rng = random.split(rng)
    perm = random.permutation(temp, X.shape[0])
    for i in range(num_batches):
      batch_idx = perm[i*batch_size:(i+1)*batch_size]
      yield X[batch_idx], y[batch_idx]

if __name__ == "__main__":
  rng = random.PRNGKey(0)

  X, y, X_test, y_test = mnist()
  X, X_test = X.reshape(-1, 28, 28, 1), X_test.reshape(-1, 28, 28, 1)
  y, y_test = (np.argmax(y, 1) % 2 == 1).astype(np.float32), (np.argmax(y_test, 1) % 1 == 1).astype(np.float32)

  temp, rng = random.split(rng)
  params, predict = model(temp)

  def loss(params, batch, l2=0.05):
    X, y = batch
    y_hat = predict(params, X).reshape(-1)
    return -np.mean(y * np.log(y_hat) + (1. - y) * np.log(1. - y_hat))

  @jit
  def update(i, opt_state, batch):
    params = get_params(opt_state)
    return opt_update(i, grad(loss)(params, batch), opt_state)

  iterations = 5000
  batch_size = 64
  step_size = 0.001

  opt_init, opt_update, get_params = optimizers.adam(step_size)
  opt_state = opt_init(params)

  temp, rng = random.split(rng)
  batches = data_stream(temp, batch_size, X, y)

  for i in tqdm(range(iterations)):
    opt_state = update(i, opt_state, next(batches))
    if i % 1000 == 0:
      params = get_params(opt_state)
      print('Loss: {:.4f}'.format(loss(params, (X, y))))
  params = get_params(opt_state)
  exit()

  pickle.dump(lr_params, open('logistic_regression_params.pkl', 'wb'))
  pickle.dump(logistic_regression, open('logistic_regression.pkl', 'wb'))
  pickle.dump(fe_params, open('feature_extractor_params.pkl', 'wb'))
  pickle.dump(feature_extractor, open('feature_extractor.pkl', 'wb'))


from collections import ChainMap

import yaml

with open('eve_static_data/invFlags.yaml') as flags_file:
    INV_FLAGS = {item['flagID']: item for item in yaml.full_load(flags_file)}

INVENTORY_POSITIONS = [
    *range(92, 99+1),  # Rigs
    *range(27, 34+1),  # High Slots
    *range(19, 26+1),  # Med Slots
    *range(11, 18+1),  # Low Slots
    0                # Everything Else
]


import copy
from typing import List, Optional

def deep_merge_dicts(original: dict, new_dict: dict) -> dict:
    """
    Overview:
        Merge two dicts by calling ``deep_update``
    Arguments:
        - original (:obj:`dict`): Dict 1.
        - new_dict (:obj:`dict`): Dict 2.
    Returns:
        - merged_dict (:obj:`dict`): A new dict that is d1 and d2 deeply merged.
    """
    original = original or {}
    new_dict = new_dict or {}
    merged = copy.deepcopy(original)
    if new_dict:  # if new_dict is neither empty dict nor None
        deep_update(merged, new_dict, True, [])
    return merged

def deep_update(
    original: dict,
    new_dict: dict,
    new_keys_allowed: bool = False,
    whitelist: Optional[List[str]] = None,
    override_all_if_type_changes: Optional[List[str]] = None
):
    """
    Overview:
        Update original dict with values from new_dict recursively.
    Arguments:
        - original (:obj:`dict`): Dictionary with default values.
        - new_dict (:obj:`dict`): Dictionary with values to be updated
        - new_keys_allowed (:obj:`bool`): Whether new keys are allowed.
        - whitelist (:obj:`Optional[List[str]]`):
            List of keys that correspond to dict
            values where new subkeys can be introduced. This is only at the top
            level.
        - override_all_if_type_changes(:obj:`Optional[List[str]]`):
            List of top level
            keys with value=dict, for which we always simply override the
            entire value (:obj:`dict`), if the "type" key in that value dict changes.

    .. note::

        If new key is introduced in new_dict, then if new_keys_allowed is not
        True, an error will be thrown. Further, for sub-dicts, if the key is
        in the whitelist, then new subkeys can be introduced.
    """
    whitelist = whitelist or []
    override_all_if_type_changes = override_all_if_type_changes or []

    for k, value in new_dict.items():
        if k not in original and not new_keys_allowed:
            raise RuntimeError("Unknown config parameter `{}`. Base config have: {}.".format(k, original.keys()))

        # Both original value and new one are dicts.
        if isinstance(original.get(k), dict) and isinstance(value, dict):
            # Check old type vs old one. If different, override entire value.
            if k in override_all_if_type_changes and \
                    "type" in value and "type" in original[k] and \
                    value["type"] != original[k]["type"]:
                original[k] = value
            # Whitelisted key -> ok to add new subkeys.
            elif k in whitelist:
                deep_update(original[k], value, True)
            # Non-whitelisted key.
            else:
                deep_update(original[k], value, new_keys_allowed)
        # Original value not a dict OR new value not a dict:
        # Override entire value.
        else:
            original[k] = value
    return original


import argparse
import copy
import csv
import os
import warnings

import numpy
import torch
import tqdm
import yaml
from torch.utils import data

from nets import nn
from utils import util
from utils.dataset import Dataset

warnings.filterwarnings("ignore")

def learning_rate(args, params):
    def fn(x):
        return (1 - x / args.epochs) * (1.0 - params['lrf']) + params['lrf']

    return fn

def train(args, params):
    # Model
    model = nn.yolo_v5_n(len(params['names'].values())).cuda()

    # Model attributes
    params['box'] *= 3 / model.head.nl
    params['obj'] *= (args.input_size / 640) ** 2 * 3 / model.head.nl
    params['cls'] *= len(params['names'].values()) / 80 * 3 / model.head.nl

    # Optimizer
    accumulate = max(round(64 / (args.batch_size * args.world_size)), 1)
    params['weight_decay'] *= args.batch_size * args.world_size * accumulate / 64

    p = [], [], []
    for v in model.modules():
        if hasattr(v, 'bias') and isinstance(v.bias, torch.nn.Parameter):
            p[2].append(v.bias)
        if isinstance(v, torch.nn.BatchNorm2d):
            p[1].append(v.weight)
        elif hasattr(v, 'weight') and isinstance(v.weight, torch.nn.Parameter):
            p[0].append(v.weight)

    optimizer = torch.optim.SGD(p[2], params['lr0'], params['momentum'], nesterov=True)

    optimizer.add_param_group({'params': p[0], 'weight_decay': params['weight_decay']})
    optimizer.add_param_group({'params': p[1]})
    del p

    # Scheduler
    lr = learning_rate(args, params)
    scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr, last_epoch=-1)

    # EMA
    ema = util.EMA(model) if args.local_rank == 0 else None

    filenames = []
    with open('../Dataset/COCO/train2017.txt') as reader:
        for filename in reader.readlines():
            filename = filename.rstrip().split('/')[-1]
            filenames.append('../Dataset/COCO/images/train2017/' + filename)

    dataset = Dataset(filenames, args.input_size, params, True)

    if args.world_size <= 1:
        sampler = None
    else:
        sampler = data.distributed.DistributedSampler(dataset)

    loader = data.DataLoader(dataset, args.batch_size, sampler is None, sampler,
                             num_workers=8, pin_memory=True, collate_fn=Dataset.collate_fn)

    util.check_anchors(dataset, model, args, params)
    model.half().float()  # pre-reduce anchor precision

    if args.world_size > 1:
        # DDP mode
        model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model)
        model = torch.nn.parallel.DistributedDataParallel(module=model,
                                                          device_ids=[args.local_rank],
                                                          output_device=args.local_rank)

    # Start training
    best = 0
    num_batch = len(loader)
    amp_scale = torch.cuda.amp.GradScaler()
    criterion = util.ComputeLoss(model, params)
    num_warmup = max(round(params['warmup_epochs'] * num_batch), 100)
    with open('weights/step.csv', 'w') as f:
        if args.local_rank == 0:
            writer = csv.DictWriter(f, fieldnames=['epoch', 'mAP@50', 'mAP'])
            writer.writeheader()
        for epoch in range(args.epochs):
            model.train()

            m_loss = util.AverageMeter()
            if args.world_size > 1:
                sampler.set_epoch(epoch)
            p_bar = enumerate(loader)
            if args.local_rank == 0:
                print(('\n' + '%10s' * 3) % ('epoch', 'memory', 'loss'))
            if args.local_rank == 0:
                p_bar = tqdm.tqdm(p_bar, total=num_batch)  # progress bar

            optimizer.zero_grad()

            for i, (samples, targets, _) in p_bar:
                x = i + num_batch * epoch  # number of iterations
                samples = samples.cuda().float() / 255
                targets = targets.cuda()

                # Warmup
                if x <= num_warmup:
                    xp = [0, num_warmup]
                    fp = [1, 64 / (args.batch_size * args.world_size)]
                    accumulate = max(1, numpy.interp(x, xp, fp).round())
                    for j, y in enumerate(optimizer.param_groups):
                        if j == 0:
                            fp = [params['warmup_bias_lr'], y['initial_lr'] * lr(epoch)]
                        else:
                            fp = [0.0, y['initial_lr'] * lr(epoch)]
                        y['lr'] = numpy.interp(x, xp, fp)
                        if 'momentum' in y:
                            fp = [params['warmup_momentum'], params['momentum']]
                            y['momentum'] = numpy.interp(x, xp, fp)

                # Forward
                with torch.cuda.amp.autocast():
                    outputs = model(samples)  # forward
                loss = criterion(outputs, targets)

                m_loss.update(loss.item(), samples.size(0))

                loss *= args.batch_size  # loss scaled by batch_size
                loss *= args.world_size  # gradient averaged between devices in DDP mode

                # Backward
                amp_scale.scale(loss).backward()

                # Optimize
                if x % accumulate == 0:
                    amp_scale.step(optimizer)  # optimizer.step
                    amp_scale.update()
                    optimizer.zero_grad()
                    if ema:
                        ema.update(model)

                # Log
                if args.local_rank == 0:
                    memory = f'{torch.cuda.memory_reserved() / 1E9:.3g}G'  # (GB)
                    s = ('%10s' * 2 + '%10.4g') % (f'{epoch + 1}/{args.epochs}', memory, m_loss.avg)
                    p_bar.set_description(s)
            # Scheduler
            scheduler.step()

            if args.local_rank == 0:
                # mAP
                last = test(args, params, ema.ema)
                writer.writerow({'mAP': str(f'{last[1]:.3f}'),
                                 'epoch': str(epoch + 1).zfill(3),
                                 'mAP@50': str(f'{last[0]:.3f}')})
                f.flush()

                # Update best mAP
                if last[1] > best:
                    best = last[1]

                # Save model
                ckpt = {'model': copy.deepcopy(ema.ema).half()}

                # Save last, best and delete
                torch.save(ckpt, './weights/last.pt')
                if best == last[1]:
                    torch.save(ckpt, './weights/best.pt')
                del ckpt

    if args.local_rank == 0:
        util.strip_optimizer('./weights/best.pt')  # strip optimizers
        util.strip_optimizer('./weights/last.pt')  # strip optimizers

    torch.cuda.empty_cache()

@torch.no_grad()
def test(args, params, model=None):
    filenames = []
    with open('../Dataset/COCO/val2017.txt') as reader:
        for filename in reader.readlines():
            filename = filename.rstrip().split('/')[-1]
            filenames.append('../Dataset/COCO/images/val2017/' + filename)

    dataset = Dataset(filenames, args.input_size, params, False)
    loader = data.DataLoader(dataset, 4, False, num_workers=4,
                             pin_memory=True, collate_fn=Dataset.collate_fn)

    if model is None:
        model = torch.load('./weights/best.pt', map_location='cuda')['model']

    model.half()

    # Configure
    model.eval()
    iou_v = torch.linspace(0.5, 0.95, 10).cuda()  # iou vector for mAP@0.5:0.95
    n_iou = iou_v.numel()

    m_pre = 0.
    m_rec = 0.
    map50 = 0.
    mean_ap = 0.
    metrics = []
    p_bar = tqdm.tqdm(loader, desc=('%10s' * 3) % ('precision', 'recall', 'mAP'))
    for samples, targets, shapes in p_bar:
        samples = samples.cuda()
        targets = targets.cuda()

        samples = samples.half()  # uint8 to fp16/32
        samples = samples / 255.0  # 0 - 255 to 0.0 - 1.0

        _, _, h, w = samples.shape  # batch size, channels, height, width

        # Inference
        outputs = model(samples)

        # NMS
        targets[:, 2:] *= torch.tensor((w, h, w, h)).cuda()  # to pixels
        outputs = util.non_max_suppression(outputs, 0.001, 0.6)

        # Metrics
        for i, output in enumerate(outputs):
            labels = targets[targets[:, 0] == i, 1:]
            correct = torch.zeros(output.shape[0], n_iou, dtype=torch.bool).cuda()

            if output.shape[0] == 0:
                if labels.shape[0]:
                    metrics.append((correct, *torch.zeros((3, 0)).cuda()))
                continue

            detections = output.clone()
            util.scale(detections[:, :4], samples[i].shape[1:], shapes[i][0], shapes[i][1])

            # Evaluate
            if labels.shape[0]:
                tbox = labels[:, 1:5].clone()  # target boxes
                tbox[:, 0] = labels[:, 1] - labels[:, 3] / 2  # top left x
                tbox[:, 1] = labels[:, 2] - labels[:, 4] / 2  # top left y
                tbox[:, 2] = labels[:, 1] + labels[:, 3] / 2  # bottom right x
                tbox[:, 3] = labels[:, 2] + labels[:, 4] / 2  # bottom right y
                util.scale(tbox, samples[i].shape[1:], shapes[i][0], shapes[i][1])

                correct = numpy.zeros((detections.shape[0], iou_v.shape[0]))
                correct = correct.astype(bool)

                t_tensor = torch.cat((labels[:, 0:1], tbox), 1)
                iou = util.box_iou(t_tensor[:, 1:], detections[:, :4])
                correct_class = t_tensor[:, 0:1] == detections[:, 5]
                for j in range(len(iou_v)):
                    x = torch.where((iou >= iou_v[j]) & correct_class)
                    if x[0].shape[0]:
                        matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1)
                        matches = matches.cpu().numpy()
                        if x[0].shape[0] > 1:
                            matches = matches[matches[:, 2].argsort()[::-1]]
                            matches = matches[numpy.unique(matches[:, 1], return_index=True)[1]]
                            matches = matches[numpy.unique(matches[:, 0], return_index=True)[1]]
                        correct[matches[:, 1].astype(int), j] = True
                correct = torch.tensor(correct, dtype=torch.bool, device=iou_v.device)
            metrics.append((correct, output[:, 4], output[:, 5], labels[:, 0]))

    # Compute metrics
    metrics = [torch.cat(x, 0).cpu().numpy() for x in zip(*metrics)]  # to numpy
    if len(metrics) and metrics[0].any():
        tp, fp, m_pre, m_rec, map50, mean_ap = util.compute_ap(*metrics)

    # Print results
    print('%10.3g' * 3 % (m_pre, m_rec, mean_ap))

    # Return results
    model.float()  # for training
    return map50, mean_ap

@torch.no_grad()
def demo(args):
    import cv2

    # Load model
    model = torch.load('./weights/best.pt', map_location='cuda')['model'].float()
    model.half()
    model.eval()

    camera = cv2.VideoCapture(0)
    # Check if camera opened successfully
    if not camera.isOpened():
        print("Error opening video stream or file")
    # Read until video is completed
    while camera.isOpened():
        # Capture frame-by-frame
        success, frame = camera.read()
        if success:
            image = frame.copy()
            shape = image.shape[:2]

            r = args.input_size / max(shape[0], shape[1])
            if r != 1:
                resample = cv2.INTER_LINEAR if r > 1 else cv2.INTER_AREA
                image = cv2.resize(image, dsize=(int(shape[1] * r), int(shape[0] * r)), interpolation=resample)
            height, width = image.shape[:2]

            # Scale ratio (new / old)
            r = min(1.0, args.input_size / height, args.input_size / width)

            # Compute padding
            pad = int(round(width * r)), int(round(height * r))
            w = (args.input_size - pad[0]) / 2
            h = (args.input_size - pad[1]) / 2

            if (width, height) != pad:  # resize
                image = cv2.resize(image, pad, interpolation=cv2.INTER_LINEAR)
            top, bottom = int(round(h - 0.1)), int(round(h + 0.1))
            left, right = int(round(w - 0.1)), int(round(w + 0.1))
            image = cv2.copyMakeBorder(image, top, bottom, left, right, cv2.BORDER_CONSTANT)  # add border

            # Convert HWC to CHW, BGR to RGB
            x = image.transpose((2, 0, 1))[::-1]
            x = numpy.ascontiguousarray(x)
            x = torch.from_numpy(x)
            x = x.unsqueeze(dim=0)
            x = x.cuda()
            x = x.half()
            x = x / 255
            # Inference
            outputs = model(x)
            # NMS
            outputs = util.non_max_suppression(outputs, 0.25, 0.7)
            for output in outputs:
                output[:, [0, 2]] -= w  # x padding
                output[:, [1, 3]] -= h  # y padding
                output[:, :4] /= min(height / shape[0], width / shape[1])

                output[:, 0].clamp_(0, shape[1])  # x1
                output[:, 1].clamp_(0, shape[0])  # y1
                output[:, 2].clamp_(0, shape[1])  # x2
                output[:, 3].clamp_(0, shape[0])  # y2

                for box in output:
                    box = box.cpu().numpy()
                    x1, y1, x2, y2, score, index = box
                    cv2.rectangle(frame, (int(x1), int(y1)), (int(x2), int(y2)), (0, 255, 0), 2)

            cv2.imshow('Frame', frame)
            # Press Q on keyboard to  exit
            if cv2.waitKey(25) & 0xFF == ord('q'):
                break
        # Break the loop
        else:
            break
    # When everything done, release the video capture object
    camera.release()

    # Closes all the frames
    cv2.destroyAllWindows()

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--input-size', default=640, type=int)
    parser.add_argument('--batch-size', default=32, type=int)
    parser.add_argument('--local_rank', default=0, type=int)
    parser.add_argument('--epochs', default=600, type=int)
    parser.add_argument('--train', action='store_true')
    parser.add_argument('--test', action='store_true')
    parser.add_argument('--demo', action='store_true')

    args = parser.parse_args()

    args.local_rank = int(os.getenv('LOCAL_RANK', 0))
    args.world_size = int(os.getenv('WORLD_SIZE', 1))

    if args.world_size > 1:
        torch.cuda.set_device(device=args.local_rank)
        torch.distributed.init_process_group(backend='nccl', init_method='env://')

    if args.local_rank == 0:
        if not os.path.exists('weights'):
            os.makedirs('weights')

    util.setup_seed()
    util.setup_multi_processes()

    with open(os.path.join('utils', 'args.yaml'), errors='ignore') as f:
        params = yaml.safe_load(f)

    if args.train:
        train(args, params)
    if args.test:
        test(args, params)
    if args.demo:
        demo(args)

if __name__ == "__main__":
    main()


import pandas as pd
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings('ignore')

class DynamicEvolutionStats:
    def __init__(self, seeds, specialist_type):
        self.seeds = seeds
        self.specialist_type = specialist_type
        self.init_data()

    def init_data(self):
        self.data = {}
        for seed in self.seeds:
            self.data[seed] = pd.DataFrame(columns=['generation', 'score', 'cycle'])

    def get_data(self, suffix='score'):
        for seed in self.seeds:
            df = pd.read_csv(f'../../data/specialist/dynamic_evolution/{self.specialist_type}/{seed}_{suffix}.csv')
            self.data[seed] = pd.concat([self.data[seed], df]).query("generation >= 1600")

    def get_seed(self, seed):
        return self.data.get(seed)

    def describe_seeds(self):
        describe = []
        for seed in self.seeds:
            df = self.get_seed(seed)
            describe.append([
                df.score.mean(),
                len(df.query('cycle == "score"')),
                len(df.query('cycle == "fit"')),
            ])

        return pd.DataFrame(
            describe,
            columns=['score', 'score_time', 'fit_time']
        )

    def plot_seeds_scatter(self):
        for seed in self.seeds:
            df = self.get_seed(seed)
            plt.scatter(df.generation, df.score, s=1)
        plt.legend(self.seeds)
        plt.title(f'All Seeds Specialist Score')
        plt.xlabel('generation')
        plt.ylabel('score')
        plt.show()

    def describe_score(self):
        df = self.describe_seeds()
        plt.boxplot(df.score, labels=['mean'])
        plt.title(f'All Seeds Specialist Score Mean')
        plt.ylabel('score')
        plt.show()

    def describe_cycles(self):
        df = self.describe_seeds()
        plt.boxplot(df[['score_time', 'fit_time']], labels=['score_time', 'fit_time'])
        plt.title(f'All Seeds Specialist Cycles')
        plt.xlabel('process')
        plt.ylabel('cycles')
        plt.show()


#!/usr/bin/env python
import argparse

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="split blast results by organism")
    parser.add_argument("blast", type=argparse.FileType("r"))
    args = parser.parse_args()

    blast = [x.split("\t") for x in args.blast.readlines()]

    for row in blast:
        if "<>" in row[24]:
            for i in row[24].strip().split("<>"):
                row_copy = row
                row_copy[24] = i
                print("\t".join(row_copy).strip())
        else:
            print("\t".join(row).strip())


import os
import math
import copy
import codecs
import numpy as np
import srt
import subprocess
import datetime

from utils import mkdir, basename_without_ext
from voice_detector import VoiceDetector
from tqdm import tqdm

def shift_by_delay(bin_arr2, delay_by_frames):
    if delay_by_frames < 0:
        return bin_arr2[abs(delay_by_frames):]

    return np.concatenate([np.zeros(delay_by_frames).astype(np.uint8), bin_arr2])

def make_list_length_equal(lst1, lst2):
    len_lst1 = lst1.shape[0]
    len_lst2 = lst2.shape[0]
    max_len = max(len_lst1, len_lst2)
    return np.concatenate([lst1, np.zeros(max_len - len_lst1).astype(np.uint8)]), np.concatenate([lst2, np.zeros(max_len - len_lst2).astype(np.uint8)])

def error(bin_arr1, bin_arr2):
    # MAE
    return np.sum(bin_arr1.astype(np.uint8) ^ bin_arr2.astype(np.uint8)) / float(len(bin_arr1))

def get_err(tmp_bin_arr1, tmp_bin_arr2, delay_by_frames):
    #tmp_bin_arr1 = arr1[:]
    #tmp_bin_arr2 = arr2[:]

    # shift by delay
    tmp_bin_arr2 = shift_by_delay(tmp_bin_arr2, delay_by_frames)

    # align arrays
    tmp_bin_arr1, tmp_bin_arr2 = make_list_length_equal(tmp_bin_arr1, tmp_bin_arr2)

    # calculate error
    tmp_err = error(tmp_bin_arr1, tmp_bin_arr2)

    return delay_by_frames, tmp_err

class GetSub:
    def __init__(self, aggressiveness, frame_duration_ms, padding_duration_ms):
        self.vad = VoiceDetector(
            aggressiveness, frame_duration_ms, padding_duration_ms)

    def timedelta_to_frame(self, td):
        ms = float(td.seconds) * 1000.0 + float(td.microseconds) * 0.001
        return int(ms / self.vad.frame_duration_ms)

    def binary_array_from_srt(self, srt_path):
        common_encodings = ['latin1', 'utf-8', 'utf-16', 'cp1252']

        subs = []

        for encoding in common_encodings:
            try:
                srt_file = codecs.open(srt_path, 'r', encoding=encoding)
                srt_string = srt_file.read()
                srt_file.close()

                subs = np.array(list(srt.parse(srt_string)))

                break
            except BaseException as error:
            	pass
                # print('An exception occurred: {}'.format(error))

        start_end_pairs = [(self.timedelta_to_frame(sub.start), self.timedelta_to_frame(sub.end)) for sub in subs]

        # convert seconds and microseconds to milliseconds
        first_sub_frame = start_end_pairs[0][0]
        last_sub_frame = start_end_pairs[-1][1]

        bin_array = np.zeros(last_sub_frame).astype(np.uint8)

        print('Creating Binary Array from SRT..')

        for start_frame, end_frame in tqdm(start_end_pairs):
            bin_array[start_frame:end_frame] = 1

        # TODO
        five_second_delay = int(5 * 1000 / self.vad.frame_duration_ms)

        # set max delay to 5% of video
        max_delay = max(five_second_delay, int(len(bin_array) * 0.05))

        return bin_array, -first_sub_frame, max_delay, subs

    def chunks(self, lst, n):
        """Yield successive n-sized chunks from lst."""
        for i in range(0, len(lst), n):
            yield lst[i:i + n]

    def find_best_delay_milliseconds(self, bin_arr1, bin_arr2, delay_range_start, delay_range_end, error_csv_out):

        err = math.inf
        best_delay = 0

        delay_range_len = delay_range_end - delay_range_start
        rows = np.zeros((delay_range_len, 2))
        early_stop = False

        print('Finding Best Delay..')
        #with Parallel(n_jobs=cpus, prefer="threads") as parallel:

        for i, delay_by_frames in tqdm(enumerate(range(delay_range_start, delay_range_end)), total=delay_range_len):
            delay_by_frames, tmp_err = get_err(
                bin_arr1,
                bin_arr2,
                delay_by_frames,
            )
            if tmp_err < err:
                err = tmp_err
                best_delay = delay_by_frames

            rows[i][0] = delay_by_frames * self.vad.frame_duration_ms * 0.001
            rows[i][1] = tmp_err

            percent_change = (tmp_err - err) / err

            if percent_change > 0.1:
                early_stop = True
                rows = rows[:(i + 1)]
                break

        if early_stop:
            print('stopping early at', str(int(i / delay_range_len * 100.0)) + '%')

        #df = pd.DataFrame(rows, columns=["delay_in_seconds", "MAE"])
        #df.set_index("delay_in_seconds", inplace=True)
        #df.to_csv(error_csv_out)

        return best_delay * self.vad.frame_duration_ms

    def align(self, vid_file_path, srt_path, out_dir, original_name):
        bin_arr1 = np.array(list(self.vad.detect(vid_file_path))).astype(np.uint8)
        bin_arr2, delay_range_start, delay_range_end, subs = self.binary_array_from_srt(srt_path)

        best_delay_ms = self.find_best_delay_milliseconds(
            bin_arr1,
            bin_arr2,
            delay_range_start,
            delay_range_end,
            os.path.join(out_dir, original_name + "_error.csv"),
        )
        best_delay_sec = best_delay_ms * 0.001
        print(f"best delay: {best_delay_sec}s")

        out_path = os.path.join(out_dir, original_name + "_synced.srt")

        td_to_shift = datetime.timedelta(seconds=best_delay_sec)

        print('Shifting Subtitles..')
        for subtitle in tqdm(subs):
            subtitle.start += td_to_shift
            subtitle.end += td_to_shift

        with open(out_path, 'w') as file:
            file.write(srt.compose(subs))

        print('output aligned subs to:', out_path)

    def download(self, vid_file_path, language):

        out_dir = os.path.dirname(vid_file_path)
        temp_dir = "/temp/"

        mkdir(out_dir)
        mkdir(temp_dir)

        command1 = "python OpenSubtitlesDownload.py --cli --auto {} --output {} --lang {}"
        command1_list = command1.format(vid_file_path, temp_dir, language).split(" ")
        subprocess.call(command1_list)

        original_name = basename_without_ext(vid_file_path)
        srt_path = os.path.join(temp_dir, original_name + ".srt")

        # save original file as 'filename_unsynced.srt'
        out_path_unsynced = os.path.join(out_dir, original_name + "_unsynced.srt")

        command2 = "cp {} {}"
        command2_list = command2.format(srt_path, out_path_unsynced).split(" ")
        subprocess.call(command2_list)

        print('downloaded subs:', srt_path)

        self.align(vid_file_path, srt_path, out_dir, original_name)


from Crypto.Util.number import  getPrime
from Crypto.Util.number import inverse
import hashlib
import socket
from threading import Thread

host = 'localhost'
port = 6000

mysocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

try :
    mysocket.connect((host, port))

except socket.error :
    print("connexion echouer avec le serveur ")
    exit()

print("connexion etablie avec le serveur")

def gen_rsa_keypair (bits):
    p=getPrime(bits//2)
    q=getPrime(bits//2)
    n=p*q
    e=655337
    d=inverse(e, (p-1)*(q-1))
    return((e,n), (d,n)) #cle pub et cle priv

key  = gen_rsa_keypair(256)

def rsa(m,key):
    return pow(m,key[0],key[1])

def rsa_enc(msg, key):
    m = int.from_bytes(msg.encode('utf-8'),'big')
    c = rsa(m, key)
    return c

def rsa_dec(msg, key):
    txt_clair = rsa(msg, key)
    return txt_clair.to_bytes((txt_clair.bit_length()+7) // 8,'big').decode('utf-8')

class Send(Thread):

    def __init__(self,arg):
        Thread.__init__(self)
        #super(Send, self).__init__()
        self.arg = arg

    def run(self):
        continuer = True

        while(continuer):

            message = input()

            message1 = self.arg.sendall(repr(key[0]).encode('utf8'))#cle

            try:
                enchifrer = rsa_enc(message, key[0])

                #print("enchiffreeer = ",enchifrer)

                #self.arg.send(repr(enchifrer).encode('utf-8'))
                dechiffrer = rsa_dec(enchifrer, key[1])
                #print("dechiffrer ", dechiffrer)
                self.arg.send(dechiffrer.encode('utf-8'))

            except socket.error:
                continuer = False
                break

        self.arg.close()

class receive(Thread):

    def __init__(self,arg):
        Thread.__init__(self)
       # super(receive, self).__init__()
        self.arg = arg

    def run(self):

        continuer = True

        while(continuer):
            try:

                message = self.arg.recv(1024).decode('utf-8')
            except socket.error:
                    continuer = False
                    break

            else :
                print(">>>>>> {0}".format(message))

        self.arg.close()

if __name__ == "__main__":

    sn = Send(mysocket)
    sn.start()

    rv = receive(mysocket)
    rv.start()


import pika
import sys

conn = pika.BlockingConnection(pika.URLParameters('amqp://guest:guest@localhost:25672/%2F'))
channel = conn.channel()

channel.exchange_declare(exchange='direct_logs', exchange_type='direct')

severity = sys.argv[1] if len(sys.argv) > 1 else 'info'
message = ' '.join(sys.argv[2:]) or "Hello World!"
channel.basic_publish(
    exchange='direct_logs', routing_key=severity, body=message
)
print(f" [*] Sent {severity}:{message}")
conn.close()


from multiprocessing import Process,Array
from time import time
import sqlite3

from .config import KASTEN_ANZ,VOK_DIR

class vokabelKartei(Process):
    def __init__(self):
        self.conn = sqlite3.connect(VOK_DIR+"kartei.sqlite")
        self.conn.text_factory = str
        self.c = self.conn.cursor()
        self.c.execute("""CREATE TABLE IF NOT EXISTS sprachen
            (id INTEGER PRIMARY KEY, name TEXT, spr1 TEXT,
          spr2 TEXT)""")
        self.c.execute("""CREATE TABLE IF NOT EXISTS kapitel
            (id INTEGER PRIMARY KEY,  name TEXT, spr_id INT)""")
        self.c.execute("""CREATE TABLE IF NOT EXISTS vokabeln
            (id INTEGER PRIMARY KEY,  spr1 TEXT, spr2 TEXT,
          kap_id INT, kasten INT, spr_id INT, last_date INT)""")
        self.COMMIT_MODE = True
        self.DEBUG_MODE = False

    def close(self):
        self.c.close()

    def commit(self):
        if self.COMMIT_MODE == True and self.DEBUG_MODE == False:
            self.conn.commit()

    def execute(self,query_str,args=()):
        if self.DEBUG_MODE == True:
            print(query_str, args)
        self.c.execute(query_str,args)

    def set_commit_mode(self,mode):
        if mode == True and self.COMMIT_MODE == False:
            self.COMMIT_MODE = True
            self.commit()
        elif mode == False and self.COMMIT_MODE == True:
            self.COMMIT_MODE = False

    def get_kapitel(self,sprache,kap_id=-1):
        if kap_id != -1:
            self.execute("SELECT * FROM kapitel WHERE spr_id=? AND id=?",
                         (sprache,kap_id))
        else:
            self.execute("SELECT * FROM kapitel WHERE spr_id=?", (sprache,))
        return self.c.fetchall()

    def get_vok(self,vok_id):
        self.execute("SELECT * FROM vokabeln WHERE id=?", (vok_id,))
        return list(self.c.fetchall()[0])

    def get_sprachen(self,spr_id=None):
        if spr_id != None:
            self.execute("SELECT * FROM sprachen WHERE id=?", (spr_id,))
        else:
            self.execute("SELECT * FROM sprachen ORDER BY name ASC")
        return [list(x) for x in self.c.fetchall()]

    def get_stapel(self,sprache,kapitel=-1,kasten=0):
        if kapitel != -1 and kasten != 0:
            self.execute("""SELECT * FROM vokabeln
                            WHERE spr_id=? AND kap_id=? AND kasten=?""",
                         (sprache,kapitel,kasten))
        elif kapitel != -1:
            self.execute("""SELECT * FROM vokabeln
                            WHERE spr_id=? AND kap_id=?""",
                         (sprache,kapitel))
        elif kasten != 0:
            self.execute("""SELECT * FROM vokabeln
                            WHERE spr_id=? AND kasten=?""",
                         (sprache,kasten))
        else:
            self.execute("SELECT * FROM vokabeln WHERE spr_id=?", (sprache,))
        return self.c.fetchall()

    def rem_vok(self,vokids):
        if list != type(vokids):
            vokids = [vokids]
        for vok in vokids:
            self.execute("""DELETE FROM vokabeln WHERE id=?""", (vok,))
        self.commit()

    def rem_kap(self,kap_id):
        self.execute("""DELETE FROM kapitel WHERE id=?""", (kap_id,))
        self.execute("""DELETE FROM vokabeln WHERE kap_id=?""", (kap_id,))
        self.commit()

    def rem_sprache(self,spr_id):
        self.execute("""DELETE FROM sprachen WHERE id=?""", (spr_id,))
        self.execute("""DELETE FROM vokabeln WHERE spr_id=?""", (spr_id,))
        self.execute("""DELETE FROM kapitel WHERE spr_id=?""", (spr_id,))
        self.commit()

    def add_vok(self,*vok):
        kapitel = vok[3]
        if vok[3] == -1:
            kapitel = 0
        self.execute("""INSERT INTO vokabeln(spr1,spr2,kap_id,kasten,spr_id)
                        VALUES (?,?,?,?,?)""",
                     (vok[0],vok[1],kapitel,1,vok[2]))
        self.commit()
        return self.c.lastrowid

    def add_sprache(self,name,spr1,spr2):
        self.execute("""INSERT INTO sprachen(name,spr1,spr2)
                        VALUES (?,?,?)""",
                     (name,spr1,spr2))
        self.commit()
        return self.c.lastrowid

    def add_kapitel(self,name,spr_id):
        self.execute("""INSERT INTO kapitel(name,spr_id)
                        VALUES (?,?)""",
                     (name,spr_id))
        self.commit()
        return self.c.lastrowid

    def edit_sprache(self,spr_id,name,spr1,spr2):
        self.execute("""UPDATE sprachen SET name=?,spr1=?,spr2=?
                        WHERE id=?""",
                     (name,spr1,spr2,spr_id))
        self.commit()

    def edit_kapitel(self,kap_id,name):
        self.execute("""UPDATE kapitel SET name=? WHERE id=?""", (name,kap_id))
        self.commit()

    def edit_vok(self,vok_id,spr1,spr2):
        self.execute("""UPDATE vokabeln SET spr1=?,spr2=?
                        WHERE id=?""",
                     (spr1,spr2,vok_id))
        self.commit()

    def count_vok(self,sprache,kapitel=0,kasten=0):
        if kapitel != 0 and kasten != 0:
            self.execute("""SELECT COUNT(*) FROM vokabeln
                            WHERE spr_id=? AND kap_id=? AND kasten=?""",
                         (sprache,kapitel,kasten))
        elif kasten != 0:
            self.execute("""SELECT COUNT(*) FROM vokabeln
                            WHERE spr_id=? AND kasten=?""",
                         (sprache,kasten))
        elif kapitel != 0:
            self.execute("""SELECT COUNT(*) FROM vokabeln
                            WHERE spr_id=? AND kap_id=?""",
                         (sprache,kapitel))
        else:
            self.execute("""SELECT COUNT(*) FROM vokabeln
                            WHERE spr_id=?""",
                         (sprache,))
        return self.c.fetchall()[0][0]

    def change_kasten(self,vok_id,kasten):
        if kasten <= KASTEN_ANZ:
            self.execute("""UPDATE vokabeln SET kasten=?
                            WHERE id=?""",
                         (kasten,vok_id))
            self.commit()

    def touch_vok(self,vok_id,null=False):
        timestamp = int(time())
        if null:
            timestamp = 0
        self.execute("""UPDATE vokabeln SET last_date=?
                        WHERE id=?""",
                     (timestamp,vok_id))
        self.commit()

    def change_kap(self,vok_id,kapitel):
        self.execute("""UPDATE vokabeln SET kap_id=?
                        WHERE id=?""",
                     (kapitel,vok_id))
        self.commit()

    def get_duplicate(self,spr1,spr_id,kap_id=-1):
        if kap_id != -1:
            self.execute("""SELECT * FROM vokabeln
                            WHERE spr1=? AND spr_id=? AND kap_id=?""",
                         (spr1,spr_id,kap_id))
        else:
            self.execute("""SELECT * FROM vokabeln
                            WHERE spr1=? AND spr_id=?""",
                         (spr1,spr_id))
        ergebnis = self.c.fetchall()
        if len(ergebnis) == 0:
            return None
        else:
            return list(ergebnis[0])


from . import pblm
import sys
import torch
import torch.nn as nn

class CNN_A(pblm.PrebuiltLightningModule):
    def __init__(self, classes):
        super().__init__(self.__class__.__name__)

        # Model Layer Declaration
        self.conv1 = nn.Conv1d(1, 16, kernel_size=5, stride=2)
        self.conv2 = nn.Conv1d(16, 32, kernel_size=5, stride=2)
        self.conv3 = nn.Conv1d(32, 64, kernel_size=5, stride=2)
        self.dense1 = nn.Linear(64 * 309, 512)
        self.dense2 = nn.Linear(512, 256)
        self.dense3 = nn.Linear(256, classes)

    def forward(self, x):

        x = x.reshape(x.shape[0], 1, -1)

        # Convolutional Layer
        x = self.conv1(x)
        x = nn.functional.relu(x)
        x = self.conv2(x)
        x = nn.functional.relu(x)
        x = self.conv3(x)
        x = nn.functional.relu(x)

        # Flattening
        x = x.reshape(x.shape[0], -1)

        # Dense Layers
        x = self.dense1(x)
        x = nn.functional.relu(x)
        x = self.dense2(x)
        x = nn.functional.relu(x)
        x = self.dense3(x)

        return x

if __name__ == "__main__":
    model = CNN_A(4)


import cv2
import pandas as pd
import numpy as np
from PIL import Image
from torch.utils.data import Dataset

class COVIDChestXRayDataset(Dataset):
    def __init__(self, path, size=128, augment=None):
        super(COVIDChestXRayDataset, self).__init__()
        print('{} initialized with size={}, augment={}'.format(self.__class__.__name__, size, augment))
        print('Dataset is located in {}'.format(path))
        self.size = size
        self.augment = augment

        image_dir = path / 'images'
        metadata_path = path / 'metadata.csv'

        df_metadata = pd.read_csv(metadata_path, header=0)
        # Drop CT scans
        df_metadata = df_metadata[df_metadata['modality'] == 'X-ray']
        # Keep only PA/AP/AP Supine, drop Axial, L (lateral)
        allowed_views = ['PA', 'AP', 'AP Supine']
        df_metadata = df_metadata[df_metadata['view'].isin(allowed_views)]

        # COVID-19 = 1, SARS/ARDS/Pneumocystis/Streptococcus/No finding = 0
        self.labels = (df_metadata.finding == 'COVID-19').values.reshape(-1, 1)
        images = df_metadata.filename
        images = images.apply(lambda x: image_dir / x).values.reshape(-1, 1)

        self.df = pd.DataFrame(np.concatenate((images, self.labels), axis=1), columns=['image', 'label'])

        del images

        print("Dataset: {}".format(self.df))

    @staticmethod
    def _load_image(path, size):
        img = Image.open(path)
        img = cv2.resize(np.array(img), (size, size), interpolation=cv2.INTER_AREA)
        if len(img.shape) == 2:
            img = np.expand_dims(img, axis=2)
            img = np.dstack([img, img, img])
        else:
            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

        # size, size, chan -> chan, size, size
        img = np.transpose(img, axes=[2, 0, 1])

        return img

    def __getitem__(self, index):
        row = self.df.iloc[index]
        img = self._load_image(row['image'], self.size)
        label = row['label']

        if self.augment is not None:
            img = self.augment(img)

        return img, label

    def __len__(self):
        return self.df.shape[0]


import cv2 as cv
import numpy as np

from process import Resize, NormalizeImage

class PicoDetProcess():
    def __init__(self,
                 trainsize=[320,320],
                 mean=[0.485,0.456,0.406],
                 std=[0.229,0.224,0.225],
                 score_threshold=0.4,
                 nms_threshold=0.5
                 ):
        self.score_threshold = score_threshold
        self.nms_threshold = nms_threshold
        self.resize =Resize(trainsize)
        self.normalizeImage = NormalizeImage(mean = mean,std =std)

    def preprocess(self, images):
        input_im_lst = []
        input_im_info_lst = []
        for im in images:
            im, im_info = self.processim(im)
            input_im_lst.append(im)
            input_im_info_lst.append(im_info)
        inputs = self.create_inputs(input_im_lst, input_im_info_lst)
        return inputs

    def create_inputs(self, imgs, im_info):
        """generate input for different model type
        Args:
            imgs (list(numpy)): list of images (np.ndarray)
            im_info (list(dict)): list of image info
        Returns:
            inputs (dict): input of model
        """
        inputs = {}
        im_shape = []
        scale_factor = []
        if len(imgs) == 1:
            inputs['image'] = np.array((imgs[0], )).astype('float32')
            inputs['im_shape'] = np.array(
                (im_info[0]['im_shape'], )).astype('float32')
            inputs['scale_factor'] = np.array(
                (im_info[0]['scale_factor'], )).astype('float32')
            return inputs

        for e in im_info:
            im_shape.append(np.array((e['im_shape'], )).astype('float32'))
            scale_factor.append(np.array((e['scale_factor'], )).astype('float32'))

        inputs['im_shape'] = np.concatenate(im_shape, axis=0)
        inputs['scale_factor'] = np.concatenate(scale_factor, axis=0)

        imgs_shape = [[e.shape[1], e.shape[2]] for e in imgs]
        max_shape_h = max([e[0] for e in imgs_shape])
        max_shape_w = max([e[1] for e in imgs_shape])
        padding_imgs = []
        for img in imgs:
            im_c, im_h, im_w = img.shape[:]
            padding_im = np.zeros(
                (im_c, max_shape_h, max_shape_w), dtype=np.float32)
            padding_im[:, :im_h, :im_w] = img
            padding_imgs.append(padding_im)
        inputs['image'] = np.stack(padding_imgs, axis=0)
        return inputs

    def processim(self, im):
            # process image by preprocess_ops
        im_info = {
            'scale_factor': np.array(
                [1., 1.], dtype=np.float32),
            'im_shape': None,
        }
        im_info['im_shape'] = np.array(im.shape[:2], dtype=np.float32)
        im_info['scale_factor'] = np.array([1., 1.], dtype=np.float32)
        im = cv.cvtColor(im, cv.COLOR_BGR2RGB)
        im,im_info = self.resize(im,im_info)
        im,im_info = self.normalizeImage(im,im_info)
        # im = im.transpose((2, 0, 1)).copy()
        return im, im_info

    def postprocess(self, inputs, scale_factor):
        bboxs = inputs['bboxes']
        scores = inputs['scores']
        bbox,score = self.nms(bboxs[0],scores[0][0])
        for box in bbox:
            box[0] = box[0] / scale_factor[1]
            box[1] = box[1] / scale_factor[0]
            box[2] = box[2] / scale_factor[1]
            box[3] = box[3] / scale_factor[0]
        outputs =  dict(bboxes=np.array(bbox), scores=np.array(score))
        return outputs

    def nms(self, bounding_boxes, confidence_score):
        '''
        :param bounding_boxes: 候选框列表，[左上角坐标, 右下角坐标], [min_x, min_y, max_x, max_y], 原点在图像左上角
        :param confidence_score: 候选框置信度
        :param threshold: IOU阈值
        :return: 抑制后的bbox和置信度
        '''
        picked = []

        for i in range(confidence_score.shape[-1]):
            if confidence_score[i] > self.score_threshold:
                    picked.append(i)
        bounding_boxes = bounding_boxes[picked,:]
        confidence_score = confidence_score[picked]
        # 如果没有bbox，则返回空列表
        if len(bounding_boxes) == 0:
            return [], []

        # bbox转为numpy格式方便计算
        boxes = np.array(bounding_boxes)

        # 分别取出bbox的坐标
        start_x = boxes[:, 0]
        start_y = boxes[:, 1]
        end_x = boxes[:, 2]
        end_y = boxes[:, 3]

        # 置信度转为numpy格式方便计算
        score = np.array(confidence_score)  # [0.9  0.75 0.8  0.85]

        # 筛选后的bbox和置信度
        picked_boxes = []
        picked_score = []

        # 计算每一个框的面积
        areas = (end_x - start_x + 1) * (end_y - start_y + 1)

        # 将score中的元素从小到大排列，提取其对应的index(索引)，然后输出到order
        order = np.argsort(score)   # [1 2 3 0]

        # Iterate bounding boxes
        while order.size > 0:

            # The index of largest confidence score
            # 取出最大置信度的索引
            index = order[-1]

            # Pick the bounding box with largest confidence score
            # 将最大置信度和最大置信度对应的框添加进筛选列表里
            picked_boxes.append(bounding_boxes[index])
            picked_score.append(confidence_score[index])

            # 求置信度最大的框与其他所有框相交的长宽，为下面计算相交面积做准备
            # 令左上角为原点，
            # 两个框的左上角坐标x取大值，右下角坐标x取小值，小值-大值+1==相交区域的长度
            # 两个框的左上角坐标y取大值，右下角坐标y取小值，小值-大值+1==相交区域的高度
            # 这里可以在草稿纸上画个图，清晰明了
            x1 = np.maximum(start_x[index], start_x[order[:-1]])
            x2 = np.minimum(end_x[index], end_x[order[:-1]])
            y1 = np.maximum(start_y[index], start_y[order[:-1]])
            y2 = np.minimum(end_y[index], end_y[order[:-1]])

            # 计算相交面积，当两个框不相交时，w和h必有一个为0，面积也为0
            w = np.maximum(0.0, x2 - x1 + 1)
            h = np.maximum(0.0, y2 - y1 + 1)
            intersection = w * h

            # 计算IOU
            ratio = intersection / (areas[index] + areas[order[:-1]] - intersection)

            # 保留小于阈值的框的索引
            left = np.where(ratio < self.nms_threshold)
            # 根据该索引修正order中的索引（order里放的是按置信度从小到大排列的索引）
            order = order[left]

        return picked_boxes, picked_score


#%%
import numpy as np
import matplotlib.pyplot as plt

#%%
x = np.arange(0, 6 * np.pi, 0.025)
y_true = np.sin(x)
y = y_true + np.random.normal(scale=1, size=len(x))

plt.scatter(x, y, color="k")
plt.plot(x, y_true, color="red")

#%%
np.random.seed(42)
from sklearn.ensemble import HistGradientBoostingRegressor
from sklearn.neural_network import MLPRegressor
from sklearn.ensemble import RandomForestRegressor

model = HistGradientBoostingRegressor(random_state=42, max_iter=20, max_leaf_nodes=64, min_samples_leaf=30)
model.fit(x.reshape(-1, 1), y)
preds = model.predict(x.reshape(-1, 1))

plt.scatter(x, y)
plt.plot(x, preds, color="red")

#%%

def gen_one_frame(use_fraction: float, left_to_right: bool):

    use_fraction = round(use_fraction, 3)

    print(use_fraction)

    if left_to_right:
        visible_idx = np.arange(0, len(preds) * use_fraction).astype("int")
    else:
        visible_idx = np.arange(len(preds) * use_fraction, len(preds)).astype("int")

    fig, ax = plt.subplots(figsize=(10, 5))
    ax.scatter(x, y, color="k", alpha=0.1)
    ax.plot(x[visible_idx], preds[visible_idx], color="blue")
    ax.set_title(f"frac = {use_fraction}")
    fig.savefig(
        f"ML-Basics/frames/{'ltr' if left_to_right else 'rtl'}_frame_{use_fraction}.png"
    )
    plt.close()

for f in np.arange(0.01, 1, 0.005):
    gen_one_frame(use_fraction=f, left_to_right=True)
for f in np.arange(0.01, 1, 0.005):
    gen_one_frame(use_fraction=f, left_to_right=False)

#%%
import glob
from PIL import Image

# filepaths
fp_in = "ML-Basics/frames/*.png"
fp_out = "ML-Basics/out_gif.gif"

imgs = (Image.open(f) for f in sorted(glob.glob(fp_in)))
img = next(imgs)  # extract first image from iterator
img.save(fp=fp_out, format="GIF", append_images=imgs, save_all=True, duration=100, loop=0)


#!/usr/bin/env python3

import telebot
from telebot import types
import sqlite3

sqll = [0]

bot = telebot.TeleBot("TOKEN", parse_mode=None)

conn = sqlite3.connect('SQLdb.db', check_same_thread=False)
cursor = conn.cursor()

def updateUserBalance (id: int, balans: int):
    cursor.execute('UPDATE users SET balans=? WHERE id=?', (balans, id))
    conn.commit()

def createUser (id: int, user_name: str, user_login: str, balans: int):
    cursor.execute('INSERT INTO users (id, user_name, user_login, balans) VALUES (?, ?, ?, ?)', (id, user_name, user_login, balans))
    conn.commit()

def getUserBalans (id: int):
    balans = cursor.execute('SELECT balans FROM users WHERE id = ?', (id,))
    conn.commit()
    return balans.fetchone()[0]

def getUserName (id: int):
    userData = cursor.execute('SELECT user_name FROM users WHERE id = ?', (id,))
    conn.commit()
    return userData.fetchone()[0]

def selectAll_id ():
    all_id = cursor.execute('SELECT id FROM users')
    conn.commit()
    return all_id.fetchall()

def idINFOMRER (ID):        # вытаскиваем список ID-шников из картежа sqlite
    allin = selectAll_id()
    print(allin)
    num = 0
    # usId = call.from_user.id
    print(ID, '- user')
    new_base = []
    for el in allin:
        print(num)
        print(ID, allin[num][0])
        new_base.insert(num, allin[num][0]) #такой способ вытащить ID из кортежа
        num = num+1
        print(new_base)
        print('==========================================')
    return new_base

def select_all_base ():
    all_base = cursor.execute('SELECT * FROM users')
    conn.commit()
    return all_base.fetchall()

@bot.message_handler(commands=['start', 'help'])
def send_welcome(message):
    bot.reply_to(message, "Этот бот призван читать круче Кендрика Ламара.")
    stick = open('sticker.webp', 'rb')
    bot.send_sticker(message.chat.id, stick)
    markup = types.ReplyKeyboardRemove(selective=False)
    markup = types.InlineKeyboardMarkup(row_width=1)
    itembtm1 = types.InlineKeyboardButton('Правила', callback_data='inc_1')
    itembtm2 = types.InlineKeyboardButton('Пользовательское соглашение', callback_data='inc_2')
    itembtm3 = types.InlineKeyboardButton('Пополнение баланса', callback_data='inc_3')
    itembtm4 = types.InlineKeyboardButton('Личный кабинет', callback_data='inc_4')
    markup.add(itembtm1, itembtm2, itembtm3, itembtm4)
    bot.send_message(message.chat.id, "Меню:", reply_markup=markup)

"""--------------------------- Обработчик кнопок -----------------------------------------------------------"""
@bot.callback_query_handler(func=lambda call: True)
def test_callback(call):
    if call.message:
        if call.data == "inc_0":
            markup = types.InlineKeyboardMarkup(row_width=1)
            itembtm1 = types.InlineKeyboardButton('Правила', callback_data='inc_1')
            itembtm2 = types.InlineKeyboardButton('Пользовательское соглашение', callback_data='inc_2')
            itembtm3 = types.InlineKeyboardButton('Пополнение баланса', callback_data='inc_3')
            itembtm4 = types.InlineKeyboardButton('Личный кабинет', callback_data='inc_4')
            markup.add(itembtm1, itembtm2, itembtm3, itembtm4)
            bot.edit_message_text("Меню:", call.message.chat.id, call.message.message_id, reply_markup=markup)

        elif call.data == "inc_1":
            user_ID = call.from_user.id
            if user_ID in idINFOMRER(user_ID):
                print('Hello, friend')
            else:
                us_id = call.from_user.id
                us_name = call.from_user.first_name
                us_sname = call.from_user.username
                createUser(id=us_id, user_name=us_name, user_login=us_sname, balans=0)
                print('new user')
                # bot.send_message(call.from_user.id, 'Привет! Ваше имя добавленно в базу данных!')

            markup = types.InlineKeyboardMarkup(row_width=1)
            item1 = types.InlineKeyboardButton('Отправить отзыв', callback_data='la_2')
            item2 = types.InlineKeyboardButton('Назад', callback_data='inc_0')
            markup.add(item1, item2)
            bot.edit_message_text("Мы тут не в игры играем, никаких правил", call.message.chat.id, call.message.message_id, reply_markup=markup)

        elif call.data == "inc_2":
            user_ID = call.from_user.id
            if user_ID in idINFOMRER(user_ID):
                print('Hello, friend')
            else:
                us_id = call.from_user.id
                us_name = call.from_user.first_name
                us_sname = call.from_user.username
                createUser(id=us_id, user_name=us_name, user_login=us_sname, balans=0)
                print('new user')
                # bot.send_message(call.from_user.id, 'Привет! Ваше имя добавленно в базу данных!')
            markup = types.InlineKeyboardMarkup(row_width=1)
            item_for_block_1 = types.InlineKeyboardButton('Инверсировать игру', callback_data='item_block_2')
            item_for_block_2 = types.InlineKeyboardButton('Назад', callback_data='inc_0')
            markup.add(item_for_block_1, item_for_block_2)
            bot.edit_message_text("Можем просто заблокировать, если ты нам не понравишься", call.message.chat.id, call.message.message_id, reply_markup=markup)

        elif call.data == "inc_3":
            user_ID = call.from_user.id
            if user_ID in idINFOMRER(user_ID):

                print('Hello, friend')
            else:
                us_id = call.from_user.id
                us_name = call.from_user.first_name
                us_sname = call.from_user.username
                createUser(id=us_id, user_name=us_name, user_login=us_sname, balans=0)
                # bot.send_message(call.from_user.id, 'Привет! Ваше имя добавленно в базу данных!')
                print('new user')

            us_id = call.from_user.id
            us_name = call.from_user.first_name
            us_sname = call.from_user.username
            currentUserBalance = getUserBalans(us_id)
            # print(currentUserBalance.fetchone()[0])
            currentUserBalance = currentUserBalance+100
            updateUserBalance(id=us_id, balans=currentUserBalance)
            markup = types.InlineKeyboardMarkup(row_width=1)
            balansbtn = types.InlineKeyboardButton('Назад', callback_data='inc_0')
            markup.add(balansbtn)
            bot.edit_message_text("Баланс пополнен!", call.message.chat.id, call.message.message_id, reply_markup=markup)

        elif call.data == "inc_4":
            user_ID = call.from_user.id
            if user_ID in idINFOMRER(user_ID):
                print('Hello, friend')
            else:
                us_id = call.from_user.id
                us_name = call.from_user.first_name
                us_sname = call.from_user.username
                createUser(id=us_id, user_name=us_name, user_login=us_sname, balans=0)
                print('new user')
                # bot.send_message(call.from_user.id, 'Привет! Ваше имя добавленно в базу данных!')
            if user_ID == 795675764 or user_ID == 5510951877:
                markup = types.InlineKeyboardMarkup(row_width=1)
                lkbtn1 = types.InlineKeyboardButton('Назад', callback_data='inc_0')
                lkbtn2 = types.InlineKeyboardButton('База данных', callback_data='admin_base')
                markup.add(lkbtn1, lkbtn2)
                us_id = call.from_user.id
                bot.edit_message_text('Привет, создатель!\n'+'Игрок: '+str(getUserName(us_id))+'\nБаланс: '+str(getUserBalans(us_id)),
                    call.message.chat.id, call.message.message_id, reply_markup=markup)
            else:
                markup = types.InlineKeyboardMarkup(row_width=1)
                lkbtn1 = types.InlineKeyboardButton('Назад', callback_data='inc_0')
                markup.add(lkbtn1)
                us_id = call.from_user.id
                bot.edit_message_text('Игрок: '+str(getUserName(us_id))+'\nБаланс: '+str(getUserBalans(us_id)),
                    call.message.chat.id, call.message.message_id, reply_markup=markup)

        elif call.data == "la_2":
            markup = types.InlineKeyboardMarkup(row_width=1)
            otzyv = types.InlineKeyboardButton('Назад', callback_data='inc_0')
            markup.add(otzyv)
            bot.edit_message_text("info@xcloudclub.com", call.message.chat.id, call.message.message_id, reply_markup=markup)

        elif call.data == 'item_block_2':
            markup = types.InlineKeyboardMarkup(row_width=1)
            inv_1 = types.InlineKeyboardButton('Назад', callback_data='inc_0')
            markup.add(inv_1)
            bot.edit_message_text("Инверсированно", call.message.chat.id, call.message.message_id, reply_markup=markup)

        elif call.data == 'admin_base':
            my_list = []
            for x in select_all_base():
                my_list.append(''.join(str(x)))    #https://ru.stackoverflow.com/questions/1178388
            my_str = '\n'.join(my_list)
            # print(select_all_base())
            # for item in select_all_base():
            #     print(item)
            markup = types.InlineKeyboardMarkup(row_width=1)
            button_back = types.InlineKeyboardButton('Назад', callback_data='inc_0')
            markup.add(button_back)
            bot.edit_message_text('База данных\n'+my_str, call.message.chat.id, call.message.message_id, reply_markup=markup)

"""------------------------- Обработчики текста и стикеров --------------------------------------------------"""
@bot.message_handler(content_types=['text'])
def text_up(message):
    bot.reply_to(message, message.text)
    bot.send_message(message.chat.id, "Давай без самодеятельности. Мы для кого кнопки сделали?")

@bot.message_handler(content_types=['sticker'])
def text_down(message):
    bot.send_message(message.chat.id, "Козырный стикер!")

"""--------------------------------- Start ------------------------------------------------------------------"""
bot.infinity_polling()


import os
import ast
import subprocess
import uuid
import json
import hashlib
import socket
import psutil
from ipykernel.ipkernel import IPythonKernel

def make_except_safe(code):
    code = code.replace('\n', '\n ')
    code = 'try:\n ' + code
    code = code + '\nexcept: pass\n'
    try:
        ast.parse(code)
        return code
    except:
        return ''

SCIUNIT_HOME = os.path.expanduser('~/sciunit/')
SCIUNIT_PROJECT_FILE = os.path.join(SCIUNIT_HOME, '.activated')
SCIUNIT_SOCKET_FILE = os.path.join(SCIUNIT_HOME, 'listener.socket')

class SciunitKernel(IPythonKernel):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        implementation = super().implementation + ' sciunit'

        if (os.path.exists(SCIUNIT_PROJECT_FILE)):
            self.project = open(SCIUNIT_PROJECT_FILE).read().strip()
            self.project_name = os.path.basename(os.path.normpath(self.project))
            if (os.path.exists(os.path.join(self.project, 'kernel'))):
                self.recording = False
            else:
                self.recording = True
                open(os.path.join(self.project, 'kernel'), 'w').write(json.dumps([]))

        else:
            self.project_name = 'Project_' + str(uuid.uuid4())
            self.project = os.path.join(SCIUNIT_HOME, self.project_name)
            subprocess.run(['sciunit', 'create', self.project_name])
            self.recording = True
            open(os.path.join(self.project, 'kernel'), 'w').write(json.dumps([]))

        self.eid = 1
        self.file = os.path.join(self.project, 'run.py')
        self.valid = True

        files = psutil.Process().open_files()
        for file in files:
            os.close(file.fd)

        criu_path = os.path.join(self.project, 'criu0')
        data = ['Dump', os.getpid(), os.getppid(), criu_path, 0]
        client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        client.connect(SCIUNIT_SOCKET_FILE)
        client.sendall(json.dumps(data).encode())
        client.close()

    def do_execute(self, code, silent, store_history=True, user_expressions=None, allow_stdin=False):
        criu_path = os.path.join(self.project, f'criu{self.eid}')
        if (os.path.exists(criu_path)): self.recording = False
        hashes = json.loads(open(os.path.join(self.project, 'kernel')).read())
        if not self.recording and (len(hashes) == self.eid - 1): self.valid = False

        data = []
        if self.valid:
            with open(self.file[1], 'a') as file:
                safe_code = make_except_safe(code)
                if safe_code:
                    if self.recording:
                        print('Recording e{}'.format(self.eid))
                        open(self.file, 'a').write(safe_code)
                        subprocess.Popen(['sciunit', 'exec', 'python3', self.file], stdout=subprocess.PIPE).communicate()

                        hashes.append(hashlib.sha256(safe_code.encode()).hexdigest())
                        open(os.path.join(self.project, 'kernel'), 'w').write(json.dumps(hashes))

                        data = ['Dump', os.getpid(), os.getppid(), criu_path, self.eid]
                    else:
                        if (hashlib.sha256(safe_code.encode()).hexdigest() != hashes[self.eid - 1]):
                            print('Invalid, stopped repeating')
                            self.valid = False
                        else:
                            print('Valid, repeating e{}'.format(self.eid))
                            subprocess.Popen(['sciunit', 'repeat', 'e{}'.format(self.eid)], stdout=subprocess.PIPE).communicate()
                            data = ['Restore', os.getpid(), os.getppid(), criu_path, self.eid]

                    self.eid += 1

        output = super().do_execute(code, silent, False, user_expressions, allow_stdin)

        if data:
            client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
            client.connect(SCIUNIT_SOCKET_FILE)
            client.sendall(json.dumps(data).encode())
            client.close()
            # TODO: Wait without Socket

        return output

if __name__ == '__main__':
    from ipykernel.kernelapp import IPKernelApp
    IPKernelApp.launch_instance(kernel_class=SciunitKernel)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import urllib.request
import re
import os
import random
import time
from time import sleep
import json
from collections import OrderedDict

def dropSameListEle(inList):
	outList=[]
	for x in inList:
		if x not in outList and x != '':
			outList.append(x)
	return outList

class FWIKI:

	#初始化，传入起始页码,截止页码
	def __init__(self):
		self.baseUrl="http://fgowiki.com/guide/petdetail/"

	#抓取页面
	def getPage(self,url):
		try:
			request=urllib.request.Request(url)
			response=urllib.request.urlopen(request)
			page=response.read().decode('utf-8')
			return page
		except (urllib.request.URLError,e):
			print('erro')
			if hasattr(e,'reason'):
				print('reason',e.reason)
				return None

	#提取信息
	def getInf(self,regExpress,page,pos):
		pattern=re.compile(regExpress,re.S)
		result=re.search(pattern,page)
		if result:
			result = result.group(pos).strip()
			result = re.sub(r'・',r'·',result)
			result = re.sub(r'〔(.*?)〕',r'(\1)',result)
			result = re.sub(r'（(.*?)）',r'(\1)',result)
			return result
		else:
			return None

f=FWIKI()
whiteList=[83,149,151,152,168]
startPage=1
endPage=182

skillList=[]
pSkillList=[]
NPList=[]
nameDict=OrderedDict()

while startPage<=endPage:
	try:
		if startPage in whiteList:
			startPage = startPage + 1
			continue

		url=f.baseUrl+str(startPage)
		page=f.getPage(url)
		page=page.encode().decode('unicode_escape')

		name=f.getInf(r'"NAME":"(.*?)"',page,1)
		nameDict[startPage]=name

		skill=f.getInf(r'"SKILL_R1":"(.*?)"',page,1)
		skillList.append(skill)
		skill=f.getInf(r'"SKILL_R2":"(.*?)"',page,1)
		skillList.append(skill)
		skill=f.getInf(r'"SKILL_R3":"(.*?)"',page,1)
		skillList.append(skill)

		np=	f.getInf(r'"T_NAME":"(.*?)"',page,1)
		np = re.sub(r'\(.*?\)','',np)
		NPList.append(np)

		pSkill=f.getInf(r'"CSKILL_R1":"(.*?)"',page,1)
		pSkillList.append(pSkill)
		pSkill=f.getInf(r'"CSKILL_R2":"(.*?)"',page,1)
		pSkillList.append(pSkill)
		pSkill=f.getInf(r'"CSKILL_R3":"(.*?)"',page,1)
		pSkillList.append(pSkill)
		pSkill=f.getInf(r'"CSKILL_R4":"(.*?)"',page,1)
		pSkillList.append(pSkill)

		print(str(startPage))
		if startPage <= endPage:
			sleep(random.uniform(3,5))
		startPage = startPage + 1
	except Exception as e:
		print('Error:',e)
		if startPage<=endPage:
			sleep(random.uniform(2,3))

NPList=dropSameListEle(NPList)
skillList=dropSameListEle(skillList)
pSkillList=dropSameListEle(pSkillList)

lines='var servantsDict = {\n'
for x in nameDict:
	lines+='\t"'+str(x)+'" : "'+nameDict[x]+'",\n'
lines+='};\n\n\n\n'

lines+='var noblePhantasmsDict = {\n'
for x in NPList:
	lines+='\t"" : "'+str(x)+'",\n'
lines+='};\n\n\n\n'

lines+='var skillsDict = {\n'
for x in skillList:
	lines+='\t"" : "'+str(x)+'",\n'
lines+='};\n\n\n\n'

lines+='var passiveSkillsDict = {\n'
for x in pSkillList:
	lines+='\t"" : "'+str(x)+'",\n'
lines+='};\n\n\n\n'

with open('servants_new.json','w+',encoding='utf-8') as wpoint:
	wpoint.write(lines)

print('Task is finished!')


# -*- coding: utf-8 *-
import pprint
import re
import sys
import importlib
from Symfopy.Component.HttpFoundation import Request, Response

class Router(object):

    var_regex = re.compile(r'\{(\w+)(?::([^}]+))?\}')

    def __init__(self, routes = {}):
        self.routes = dict()
        for name in routes:
            vars = routes[name].get('defaults', {})
            self.add_route(name, routes[name]['route'],\
                    routes[name]['controller'], **vars)

    def load_controller(self, string):
        module_name, func_name = string.split(':', 1)
        module = importlib.import_module(module_name)
        #__import__(module_name)
        #module = sys.modules[module_name]
        func = getattr(module, func_name)
        return func

    def add_route(self, name, route, controller, **vars):
        #if isinstance(controller, basestring):
        #    controller = self.load_controller(controller)
        self.routes[name] = (re.compile(self.template_to_regex(route)),
                controller, vars)

    @staticmethod
    def template_to_regex(template):
        regex = ''
        last_pos = 0
        for match in Router.var_regex.finditer(template):
            regex += re.escape(template[last_pos:match.start()])
            var_name = match.group(1)
            expr = match.group(2) or '[^/]+'
            expr = '(?P<%s>%s)' % (var_name, expr)
            regex += expr
            last_pos = match.end()
        regex += re.escape(template[last_pos:])
        regex = '^%s$' % regex
        return regex

    def __str__(self):
        return pprint.pformat(self.__dict__)

    @staticmethod
    def notfound(message = None, **kwargs):
        content = ['
Not Found
']
        if isinstance(message, basestring):
            content.append('
'+ message + '
')
        elif isinstance(message, list):
            for x in message:
                if isinstance(x, basestring):
                    content.append('
'+ x + '
')
        return Response(content, 404)

def rest_controller(cls):
    def replacement(request, **urlvars):
        action = urlvars.get('action', None)
        if action:
            action += '_' + request.get_method().lower()
            urlvars.pop('action')
        else:
            if isinstance(action, basestring):
                urlvars.pop('action')
            action = request.get_method().lower()
        instance = cls(**urlvars)
        try:
            method = getattr(instance, action)
        except Exception:
            return Router.notfound('No action ' + action)
        return method(request)
    return replacement

def rest_controller_template(cls):
    def replacement(request, template = None, **urlvars):
        action = urlvars.get('action', None)
        if action:
            action += '_' + request.get_method().lower()
            urlvars.pop('action')
        else:
            if isinstance(action, basestring):
                urlvars.pop('action')
            action = request.get_method().lower()
        instance = cls(**urlvars)
        try:
            method = getattr(instance, action)
        except Exception:
            return Router.notfound('No action ' + action)
        if template:
            return method(request, template)
        else:
            return method(request)
    replacement.member_func = cls
    return replacement