import numpy as np
import csv
import math

from sklearn.utils import shuffle

# Tien xu ly du lieu tu file csv
def loadData(source):
    file = open(source, 'r')
    data = csv.reader(file)
    data = np.array(list(data))
    # Xoa dong dau tien chua tieu de
    data = np.delete(data, 0, 0)
    # Xoa cot dau tien chua chi so
    data = np.delete(data, 0, 1)
    # Xao tron ngau nhien du lieu
    np.random.shuffle(data)
    file.close()
    # Tap huan luyen gom 100 quan sat dau tien
    trainSet = data[ : 100]
    # Tap kiem tra gom 51 quan sat con lai
    testSet = data[100 : ]
    return trainSet, testSet

# Tinh khoang cach giua 2 quan sat
def countDistance(a, b, features = 4):
    result = 0
    for i in range(features):
        result += (float(a[i]) - float(b[i])) ** 2
    result = math.sqrt(result)
    return result

# Thuat toan KNN tim k quan sat gan diem du lieu nhat
def KNN(trainSet, point , k):
    distances = []
    for instance in trainSet:
        distances.append({
            "label" : instance[-1], # Nhan cua moi quan sat
            "distance": countDistance(instance, point) # Khoang cach giua quan sat voi diem dang xet
        })
    # Sap xep khoang cach tang dan
    distances.sort(key = lambda x : x["distance"])
    labels = [instance["label"] for instance in distances]
    # Chon k quan sat co khoang cach nho nhat
    return labels[ : k]

# Chon nhan xuat hien nhieu nhat trong k quan sat
def selectLabel(array):
    labels = set(array)
    result = ""
    countMax = 0
    for label in labels:
        count = array.count(label)
        if(count > countMax):
            countMax = count
            result = label
    return result

# Ham main dua ra ket qua du doan voi tap kiem tra
if __name__ == "__main__":
    trainSet, testSet = loadData("Iris.csv")
    correctPredict = 0
    for instance in testSet:
        knn = KNN(trainSet, instance, 5)
        result = selectLabel(knn)
        if(result == instance[-1]):
            correctPredict += 1
        # In ra ket qua du doan va ket qua thuc te voi moi quan sat
        print(f"Kết quả dự đoán: {result} ; Két quả mong muốn: {instance[-1]}")
    print(f"Độ chính xác: {float(correctPredict / len(testSet))}")


"""# Import libraries and methods"""

import nltk
nltk.download('punkt')
from scipy import spatial
import statistics
#from word_vectors import WordVectors
import numpy as np
import pickle
import pandas as pd
import torch
import torch.utils.data

from typing import List, \
    Union
import os
import sys

import matplotlib.pyplot as plt
import seaborn as sns
import random
from collections import defaultdict
from collections import Counter
from current_files_gensim.word_vectors import WordVectors
from pytorch.model_torch import Word2VecModel
from pytorch.dataset_torch import create_skipgram, read_corpus

def read_corpus(path, min_len = 3):
    text = []
    #cut_min = lambda x: x if len(x) > min_len
    try:
        with open(path, 'r') as f:
            #text = [nltk.tokenize.word_tokenize(x.strip()) for x in f.readlines()]
            for x in f.readlines():
                res = nltk.tokenize.word_tokenize(x.strip())
                if len(res) > min_len:
                    text.append(res)
    except Exception as e:
        print(e)
    return text

torch.set_default_tensor_type(torch.cuda.FloatTensor if torch.cuda.is_available()
                                                    else torch.FloatTensor)

import datetime;

def split_given_size(a, size):
    return np.split(a, np.arange(size, len(a), size))

def get_emb_og(wv, word):
    """
    Function allowing to get the embedding of a given word.
    word: word of interest
    return: word embedding
    """

    # same as wv[word]
    return wv._syn0_final[wv._rev_vocab[word]]

def cos_similarity(tar, att):
    '''
    Calculates the cosine similarity of the target variable vs the attribute
    '''
    score = np.dot(tar, att) / (np.linalg.norm(tar) * np.linalg.norm(att))
    return score

def mean_cos_similarity(tar, att):
    '''
    Calculates the mean of the cosine similarity between the target and the range of attributes
    '''
    mean_cos = np.mean([cos_similarity(tar, attribute) for attribute in att])
    return mean_cos

def association(tar, att1, att2):
    '''
    Calculates the mean association between a single target and all of the attributes
    '''
    association = mean_cos_similarity(
        tar, att1) - mean_cos_similarity(tar, att2)

    return association

def effect_sizev2(t1, t2, att1, att2):
    '''
    Calculates the effect size (d) between the two target variables and the attributes
    Parameters:
        t1 (np.array): first target variable matrix
        t2 (np.array): second target variable matrix
        att1 (np.array): first attribute variable matrix
        att2 (np.array): second attribute variable matrix
    Returns:
        effect_size (float): The effect size, d.
    Example:
        t1 (np.array): Matrix of word embeddings for professions "Programmer, Scientist, Engineer"
        t2 (np.array): Matrix of word embeddings for professions "Nurse, Librarian, Teacher"
        att1 (np.array): matrix of word embeddings for males (man, husband, male, etc)
        att2 (np.array): matrix of word embeddings for females (woman, wife, female, etc)
    '''
    combined = np.concatenate([t1, t2])
    num1 = np.mean([association(target, att1, att2) for target in t1])
    num2 = np.mean([association(target, att1, att2) for target in t2])
    combined_association = np.array(
        [association(target, att1, att2) for target in combined])

    dof = combined_association.shape[0]
    denom = np.sqrt(
        ((dof-1)*np.std(combined_association, ddof=1) ** 2) / (dof-1))
    effect_size = (num1 - num2) / denom

    return effect_size

"""# Load data"""

BATCH_SIZE = 256
NEGATIVES = 5
EPOCHS = 3
SAMPLING_RATE = 1E-3
MIN_FREQ = 60
WINDOW_SIZE = 5
LEARNING_RATE = 1E-3
HIDDEN_SIZE = 300
# liste_termini_weat
S = ["science", "technology", "physics", "chemistry", "einstein", "nasa",
    "experiment", "astronomy"]
T = ["poetry", "art", "shakespeare", "dance", "literature", "novel", "symphony", "drama"]
A = ["male", "man", "boy", "brother", "he", "him", "his", "son"]
B = ["female", "woman", "girl", "sister", "she", "her", "hers", "daughter"]
WEATLIST = S+T+A+B

text = read_corpus('./corpus/nyt_dal_90_ad_oggi.txt')

with open("./content/data_correct_post_train/data_v2.pkl", "rb") as f:
  data = pickle.load(f)

with open("unigram_counts.pkl", "rb") as f:
  unigram_counts = pickle.load(f)

with open("vocab.pkl", "rb") as f:
  vocab = pickle.load(f)

with open("inv_vocab.pkl", "rb") as f:
  inv_vocab = pickle.load(f)

S_A = S.copy()+A.copy()
random.seed(0)
random.shuffle(S_A)
print(S_A)

def get_tuples(sentence, vocab):
  _data = []

  window = 5
  for i, t in enumerate(sentence):

      contexts = list(range(i-window, i + window+1))
      contexts = [c for c in contexts if c >=
                  0 and c != i and c < len(sentence)]
      for c in contexts:
          # TARGET, CONTEXT, nsent
          _data.append((vocab[t], vocab[sentence[c]], 1))

  return _data

sent_tuples = get_tuples(S_A, vocab)

list_index_weat = []
for word in S+T+A+B:
  list_index_weat.append(vocab[word])

list_approx_words = list(set(list_index_weat.copy()))

flat_data = [x for y in data for x in y] # 6 secondi
for tu in sent_tuples:
    flat_data.append(np.array([tu[0], tu[1], -1]))
tuple_set = {(x[0], x[1]) for x in iter(flat_data)} # 32 secondi
tuple_set_1 = [x for x in tuple_set if x[0] in list_approx_words] #1m 7s secondi

tuple_set_ord = tuple_set_1.copy()
tuple_set_ord.sort(key=lambda i:i[1],reverse=True)

list_sim_sent_retrain = []
sent_id =  -1
sent_text = text[sent_id]

log_per_steps= 1000#10000  # Every `log_per_steps` steps to log the value of loss to be minimized.

#flat_data = [x for xs in data for x in xs]
#flat_data_rt = [x for x in flat_data if x[2]!=sent_id]
data_rt = split_given_size(flat_data, BATCH_SIZE)
data_rt = [x for x in data_rt if len(x) == BATCH_SIZE]
unigram_counts_rt = unigram_counts.copy()
vocab_rt = vocab.copy()
for key in vocab:
    if key in sent_text:
        unigram_counts_rt[vocab[key]] = unigram_counts[vocab[key]]+1
inv_vocab_rt = {v: k for k, v in vocab_rt.items()}

        #data_rt, unigram_counts_rt, vocab_rt, inv_vocab_rt = create_skipgram(text_rt, WINDOW_SIZE, WEATLIST, MIN_FREQ, SAMPLING_RATE, EPOCHS, BATCH_SIZE)

word2vec = Word2VecModel(hidden_size=300,
                         batch_size=BATCH_SIZE,
                         negatives=NEGATIVES,
                         power=0.75,
                         alpha=LEARNING_RATE)

word2vec._data = data_rt
word2vec._vocab = vocab_rt
word2vec._unigram_counts = unigram_counts_rt
word2vec._vocab_size = len(unigram_counts_rt)
word2vec._inv_vocab = inv_vocab_rt
word2vec._text = text

word2vec.build_weights()
word2vec.train(EPOCHS, save=False)

print("Re-training for sentence "+str(sent_id)+" completed")

w0 = word2vec.syn0.cpu().detach().numpy()
w1 = word2vec.syn1.cpu().detach().numpy()

vocab_words = [word for word in vocab_rt]

wv = WordVectors(w0, vocab_words)

# compute original effect size and similarities:
t1 = np.array([get_emb_og(wv, word) for word in S])
t2 = np.array([get_emb_og(wv, word) for word in T])
att1 = np.array([get_emb_og(wv, word) for word in A])
att2 = np.array([get_emb_og(wv, word) for word in B])

simSA = np.array([mean_cos_similarity(tar, att1) for tar in t1]).mean()
simTA = np.array([mean_cos_similarity(tar, att1) for tar in t2]).mean()
simSB = np.array([mean_cos_similarity(tar, att2) for tar in t1]).mean()
simTB = np.array([mean_cos_similarity(tar, att2) for tar in t2]).mean()

print("effect size retraining: ", effect_sizev2(t1, t2, att1, att2))
print("similarity S-A origin: ", simSA)
print("similarity T-A origin: ", simTA)
print("similarity S-B origin: ", simSB)
print("similarity T-B origin: ", simTB)


"""Tests for basic interactions"""
import time
from typing import Any
from selenium.webdriver.common.by import By
from selenium.webdriver.remote.webdriver import WebDriver

from .helpers import (
    wait_for_tagged_element,
    get_tab,
    screenshot_exception,
)

def test_tabs_available(
    webdriver: WebDriver,
    tallymarks_dataset: Any,
    frontend_base_url: str,
    skip_tour: None,
) -> None:
    """
    test to click tabs
    """

    from ._autogenerated_ui_elements import DataTestTags

    with screenshot_exception(webdriver):
        webdriver.get(frontend_base_url)
        driver = webdriver

        wait_for_tagged_element(webdriver, DataTestTags.HELP_BUTTON)

        time.sleep(1.0)

        wait_for_tagged_element(webdriver, DataTestTags.DATAGRID_ALL_TAB_BUTTON)
        num = int(
            wait_for_tagged_element(webdriver, DataTestTags.DATAGRID_ALL_COUNT).text
        )
        assert num > 10
        tab = wait_for_tagged_element(webdriver, DataTestTags.DATAGRID_ALL_TAB_BUTTON)
        tab.click()

        wait_for_tagged_element(webdriver, DataTestTags.DATAGRID_FILTERED_TAB_BUTTON)
        num = int(
            wait_for_tagged_element(
                webdriver, DataTestTags.DATAGRID_FILTERED_COUNT
            ).text
        )
        assert num > 10
        tab = wait_for_tagged_element(
            webdriver, DataTestTags.DATAGRID_FILTERED_TAB_BUTTON
        )
        tab.click()

        wait_for_tagged_element(webdriver, DataTestTags.DATAGRID_SELECTED_TAB_BUTTON)
        num = int(
            wait_for_tagged_element(
                webdriver, DataTestTags.DATAGRID_SELECTED_COUNT
            ).text
        )
        assert num == 0
        tab = wait_for_tagged_element(
            webdriver, DataTestTags.DATAGRID_SELECTED_TAB_BUTTON
        )

        tab.click()

        tab = get_tab(driver, "Scatter Plot")
        tab.click()

        tab = get_tab(driver, "Histogram")
        tab.click()

        tab = get_tab(driver, "Similarity Map")
        tab.click()

        tab = get_tab(driver, "Inspector")
        tab.click()

def test_tagged_elements_available(
    webdriver: WebDriver,
    tallymarks_dataset: Any,
    frontend_base_url: str,
    skip_tour: None,
) -> None:
    """
    test that elements with data-test-tag are available
    """
    from ._autogenerated_ui_elements import DataTestTags

    with screenshot_exception(webdriver):
        webdriver.get(frontend_base_url)

        driver = webdriver

        wait_for_tagged_element(webdriver, DataTestTags.HELP_BUTTON)

        for tag in [
            tag for name, tag in vars(DataTestTags).items() if not name.startswith("_")
        ]:
            if tag not in [
                DataTestTags.GLOBAL_LOADING_INDICATOR,
                DataTestTags.MESHVIEW_SETTINGS_DROPDOWN,
                DataTestTags.SEQUENCEVIEW_SETTINGS_DROPDOWN,
            ]:
                element = wait_for_tagged_element(webdriver, tag, 1)
                if "dropdown" in tag or "button" in tag:
                    element.click()
                driver.find_element(by=By.XPATH, value="//body").click()


# Creates jobs to fetch tweets from twitter for the timeframe since 2016-01-01 until 2016-12-31

import argparse
from requests.utils import quote

# parse command line arguments
parser = argparse.ArgumentParser("jobcomposer")
parser.add_argument("inputfile", help="Filename to parse for headlines", type=str)
parser.add_argument("outputfile", help="Filename to parse for headlines", type=str)
parser.add_argument("database", help="Database to save to", type=str)
parser.add_argument("--limit", help="Max number of tweets to scrap", type=int, default=10000, required=False)
args = parser.parse_args()

# open files
headlines = open(args.inputfile,"r")
jobs = open(args.outputfile,"w")

# construct shell commands that use fetcher and write them to file
count = 0
for line in headlines.read().split("\n"):
    for i in range(3):
        encoded_query = quote(line) + "%20since%3A2016-01-01%20until%3A2016-12-31"
        job = "python fetcher.py " + args.database + " _" + str(count) + "_" + str(i) + " " + encoded_query + " " + str(args.limit) + "\n"
        jobs.write(job)
    count += 1

# close files
headlines.close()
jobs.close()


from django.shortcuts import render

import re
from .models import Slides, AboutUs, AboutUsImages

# Create your views here.

def home(request):
    return render(request, 'home.html', {})

def main_banner(request):
    slides = Slides.objects.all().order_by('slide_number')
    context = {
        'slides': slides
    }

    return render(request, 'main_banner.html', context)

def about_us(request):
    about_us = AboutUs.objects.first()
    about_us_images = AboutUsImages.objects.all().order_by('image_number')

    video_link = about_us.youtube_link
    regex = r'v=(.*)'
    video_id = re.findall(regex, video_link)[0]

    context = {
        'about_us': about_us,
        'images': about_us_images,
        'video_id': video_id,
    }

    return render(request, 'about_area.html', context)


from django.shortcuts import render

# Create your views here.

from django.http import HttpResponse

def index(request):
	context = {
		'title' : 'Mis Películas',
		'description' : 'Esta es mi página de películas',
		'movie' : 'Inception',
		'director' : 'Christopher Nolan',
		'genre' : 'Ciencia Ficción'
	}

	return render (request, 'prueba1/index.html', context)


from contextlib import contextmanager
from xml.etree import ElementTree as etree
from xml.sax.saxutils import quoteattr

import six

CNAME_TAGS = ('system-out', 'skipped', 'error', 'failure')
CNAME_PATTERN = '<![CDATA[{}]]>'
TAG_PATTERN = '<{tag}{attrs}>{text}</{tag}>'

@contextmanager
def patch_etree_cname(etree):
    """
    Patch ElementTree's _serialize_xml function so that it will
    write text as CDATA tag for tags tags defined in CNAME_TAGS.

    >>> import re
    >>> from xml.etree import ElementTree
    >>> xml_string = '''
    ... <testsuite name="nosetests" tests="1" errors="0" failures="0" skip="0">
    ...     <testcase classname="some.class.Foo" name="test_system_out" time="0.001">
    ...         <system-out>Some output here</system-out>
    ...     </testcase>
    ...     <testcase classname="some.class.Foo" name="test_skipped" time="0.001">
    ...         <skipped type="unittest.case.SkipTest" message="Skipped">Skipped</skipped>
    ...     </testcase>
    ...     <testcase classname="some.class.Foo" name="test_error" time="0.001">
    ...         <error type="KeyError" message="Error here">Error here</error>
    ...     </testcase>
    ...     <testcase classname="some.class.Foo" name="test_failure" time="0.001">
    ...         <failure type="AssertionError" message="Failure here">Failure here</failure>
    ...     </testcase>
    ... </testsuite>
    ... '''
    >>> tree = ElementTree.fromstring(xml_string)
    >>> with patch_etree_cname(ElementTree):
    ...    saved = str(ElementTree.tostring(tree))
    >>> systemout = re.findall(r'(<system-out>.*?</system-out>)', saved)[0]
    >>> print(systemout)
    <system-out><![CDATA[Some output here]]></system-out>
    >>> skipped = re.findall(r'(<skipped.*?</skipped>)', saved)[0]
    >>> print(skipped)
    <skipped message="Skipped" type="unittest.case.SkipTest"><![CDATA[Skipped]]></skipped>
    >>> error = re.findall(r'(<error.*?</error>)', saved)[0]
    >>> print(error)
    <error message="Error here" type="KeyError"><![CDATA[Error here]]></error>
    >>> failure = re.findall(r'(<failure.*?</failure>)', saved)[0]
    >>> print(failure)
    <failure message="Failure here" type="AssertionError"><![CDATA[Failure here]]></failure>
    """
    original_serialize = etree._serialize_xml

    def _serialize_xml(write, elem, *args, **kwargs):
        if elem.tag in CNAME_TAGS:
            attrs = ' '.join(
                ['{}={}'.format(k, quoteattr(v))
                 for k, v in sorted(elem.attrib.items())]
            )
            attrs = ' ' + attrs if attrs else ''
            text = CNAME_PATTERN.format(elem.text)
            write(TAG_PATTERN.format(
                tag=elem.tag,
                attrs=attrs,
                text=text
            ))
        else:
            original_serialize(write, elem, *args, **kwargs)

    etree._serialize_xml = etree._serialize['xml'] = _serialize_xml

    yield

    etree._serialize_xml = etree._serialize['xml'] = original_serialize

def merge_trees(*trees):
    """
    Merge all given XUnit ElementTrees into a single ElementTree.
    This combines all of the children test-cases and also merges
    all of the metadata of how many tests were executed, etc.
    """
    first_tree = trees[0]
    first_root = first_tree.getroot()

    if len(trees) == 0:
        return first_tree

    for tree in trees[1:]:
        root = tree.getroot()

        # append children elements (testcases)
        first_root.extend(root.getchildren())

        # combine root attributes which stores the number
        # of executed tests, skipped tests, etc
        for key, value in first_root.attrib.items():
            if not value.isdigit():
                continue
            combined = six.text_type(int(value) + int(root.attrib.get(key, '0')))
            first_root.set(key, combined)

    return first_tree

def merge_xunit(files, output, callback=None):
    """
    Merge the given xunit xml files into a single output xml file.

    If callback is not None, it will be called with the merged ElementTree
    before the output file is written (useful for applying other fixes to
    the merged file). This can either modify the element tree in place (and
    return None) or return a completely new ElementTree to be written.
    """
    trees = []

    for f in files:
        trees.append(etree.parse(f))

    merged = merge_trees(*trees)

    if callback is not None:
        result = callback(merged)
        if result is not None:
            merged = result

    with patch_etree_cname(etree):
        merged.write(output, encoding='utf-8', xml_declaration=True)


from django.shortcuts import render, redirect
from django.views.decorators.csrf import csrf_exempt

from .models import ToDoList
from .forms import ToDoForm

def index(request):
    item_list = ToDoList.objects.order_by('-created')

    if request.method == 'POST':
        form = ToDoForm(request.POST)
        if form.is_valid():
            form.save()
            return redirect('todo')
    form = ToDoForm()

    return render(request, "todo_app/index.html", {'form': form,
                                                   'item_list': item_list})

@csrf_exempt
def remove(request, task_id):
    item = ToDoList.objects.get(id=task_id)
    item.delete()
    return redirect('todo')


from collections import defaultdict
from datetime import datetime, timedelta
from typing import Any, Dict, List, Tuple, Union

import pytest
from core.models import AppointmentTag, CallRequest, Location, Reporter, State
from django.db.models.query import QuerySet
from django.utils import timezone
from time_machine import Coordinates

@pytest.mark.django_db()
def test_backfill_queue(ten_locations: List[Location]) -> None:
    def requests_with_reports() -> Dict[int, int]:
        report_count: Dict[int, int] = defaultdict(int)
        call_requests = CallRequest.available_requests()
        for call_request in call_requests:
            report_count[call_request.location.reports.count()] += 1
        return report_count

    assert CallRequest.available_requests().count() == 0
    CallRequest.backfill_queue(3)
    assert CallRequest.available_requests().count() == 3
    assert CallRequest.objects.count() == 3
    CallRequest.objects.all().delete()

    # Try again but mark half of the locations as 'called'; we should
    # only get uncalled locations
    reporter = Reporter.objects.get_or_create(external_id="test:1")[0]
    web = AppointmentTag.objects.get(slug="web")
    for i in range(5):
        ten_locations[i].reports.create(
            created_at=datetime.now() - timedelta(days=i),
            reported_by=reporter,
            report_source="ca",
            appointment_tag=web,
        )
    assert CallRequest.available_requests().count() == 0
    CallRequest.backfill_queue(5)
    assert CallRequest.available_requests().count() == 5
    assert CallRequest.objects.count() == 5
    assert requests_with_reports() == {0: 5}
    CallRequest.objects.all().delete()

    # Try again but ask for 8 locations; we should get 8 locations in
    # the queue, with 3 of them having reports
    assert CallRequest.available_requests().count() == 0
    CallRequest.backfill_queue(8)
    assert CallRequest.available_requests().count() == 8
    assert CallRequest.objects.count() == 8
    assert requests_with_reports() == {0: 5, 1: 3}

    # Specifically, it should be the _oldest_ reports, which are indexes 4, 3, and 2.
    assert CallRequest.objects.filter(location=ten_locations[4]).count()
    assert CallRequest.objects.filter(location=ten_locations[3]).count()
    assert CallRequest.objects.filter(location=ten_locations[2]).count()
    CallRequest.objects.all().delete()

    # Even when everything has a report, we don't error
    for location in ten_locations[5:10]:
        location.reports.create(
            reported_by=reporter,
            report_source="ca",
            appointment_tag=web,
        )
    assert CallRequest.available_requests().count() == 0
    CallRequest.backfill_queue(10)
    assert CallRequest.available_requests().count() == 10
    assert CallRequest.objects.count() == 10
    assert requests_with_reports() == {1: 10}

@pytest.mark.django_db()
def test_backfill_overfill_queue(ten_locations: List[Location]) -> None:
    # Ask for more locations than there are; we should get no duplicates
    assert CallRequest.available_requests().count() == 0
    CallRequest.backfill_queue(20)
    assert CallRequest.available_requests().count() == 10
    assert CallRequest.objects.count() == 10

    # Even when everything is already queued
    CallRequest.backfill_queue(20)
    assert CallRequest.available_requests().count() == 10
    assert CallRequest.objects.count() == 10

@pytest.mark.django_db()
def test_backfill_state(ten_locations: List[Location]) -> None:
    # Update several locations to be in CA (yes, this will lead to
    # counties' states being inconsistent) and ensure we can get both
    # CA and OR out still.
    for loc in ten_locations[0:10:2]:
        loc.state = State.objects.filter(abbreviation="CA").get()
        loc.save()

    # We should only get 5 locations
    CallRequest.backfill_queue(10, state="CA")
    assert CallRequest.objects.count() == 5
    assert CallRequest.objects.filter(location__state__abbreviation="CA").count() == 5

def enqueue(
    what: Union[Location, List[Location], QuerySet[Location]], **kwargs: Any
) -> List[CallRequest]:
    if isinstance(what, QuerySet):
        qs = what
    elif isinstance(what, List):
        qs = Location.objects.filter(id__in=[loc.id for loc in what])
    else:
        qs = Location.objects.filter(id=what.id)
    return CallRequest.insert(locations=qs, reason="New location", **kwargs)

@pytest.mark.django_db()
def test_enqueue(ten_locations: List[Location]) -> None:
    # Simple enqueue
    assert CallRequest.available_requests().count() == 0
    inserted = enqueue(ten_locations[0])
    assert len(inserted) == 1
    assert CallRequest.objects.count() == 1
    assert CallRequest.available_requests().count() == 1

    # Re-enqueing doesn't work
    inserted = enqueue(ten_locations[0])
    assert len(inserted) == 0
    assert CallRequest.objects.count() == 1
    assert CallRequest.available_requests().count() == 1

    # Marking it as resolved allows it to be enqueued again
    CallRequest.objects.all().update(completed=True)
    assert CallRequest.objects.count() == 1
    assert CallRequest.available_requests().count() == 0
    inserted = enqueue(ten_locations[0])
    assert len(inserted) == 1
    assert CallRequest.objects.count() == 2
    assert CallRequest.available_requests().count() == 1

    # Enqueueing multiple locations works
    inserted = enqueue(ten_locations[1:4])
    assert len(inserted) == 3
    assert CallRequest.objects.count() == 5
    assert CallRequest.available_requests().count() == 4

    # Enqueueing some locations which overlap with existing locations
    # enqueues just the new ones
    inserted = enqueue(ten_locations[0:8])
    assert len(inserted) == 4
    assert CallRequest.objects.count() == 9
    assert CallRequest.available_requests().count() == 8

@pytest.mark.django_db()
def test_call_enqueue_request_validity() -> None:
    def insert_fails(**kwargs: Any) -> bool:
        args = {
            "name": "Location {}".format(repr(kwargs)),
            "phone_number": "(555) 555-5555",
            "state_id": State.objects.get(abbreviation="OR").id,
            "location_type_id": 1,
            "latitude": 30,
            "longitude": 40,
        }
        args.update(kwargs)
        return enqueue(Location.objects.create(**args)) == []

    # Various things that exempt a location from being called
    assert insert_fails(do_not_call=True)
    assert insert_fails(soft_deleted=True)
    assert insert_fails(phone_number="")
    assert insert_fails(phone_number=None)
    assert insert_fails(preferred_contact_method="research_online")

@pytest.mark.django_db()
def test_call_remove_invalid_location(ten_locations: List[Location]) -> None:
    # Enqueue a _valid_, location, and ensure that changing any of the
    # above removes it
    def change_removes(**kwargs: Any) -> bool:
        location = Location.objects.create(
            name="Location {}".format(repr(kwargs)),
            phone_number="(555) 555-5555",
            state_id=State.objects.get(abbreviation="OR").id,
            location_type_id=1,
            latitude=30,
            longitude=40,
        )
        enqueue(location)
        for k, v in kwargs.items():
            setattr(location, k, v)
        location.save()
        return CallRequest.objects.filter(location=location).count() == 0

    assert change_removes(do_not_call=True)
    assert change_removes(soft_deleted=True)
    assert change_removes(phone_number="")
    assert change_removes(phone_number=None)
    assert change_removes(preferred_contact_method="research_online")

    # Ensure that completed call requests are not removed
    enqueue(ten_locations)
    CallRequest.objects.update(completed=True)
    assert CallRequest.objects.count() == 10
    assert CallRequest.available_requests().count() == 0
    enqueue(ten_locations)
    assert CallRequest.objects.count() == 20
    assert CallRequest.available_requests().count() == 10
    for location in ten_locations:
        location.do_not_call = True
        location.save()
    # We should only be left with the completed ones from earlier, and
    # none should be available.
    assert CallRequest.objects.count() == 10
    assert CallRequest.available_requests().count() == 0

@pytest.mark.django_db()
def test_get_call_request(
    ten_locations: List[Location], time_machine: Coordinates
) -> None:
    # Can't get anything from an empty queue; Note that in these
    # tests, MIN_CALL_REQUEST_QUEUE_ITEMS=0, which disables backfill.

    reporter = Reporter.objects.get_or_create(external_id="test:1")[0]
    assert CallRequest.get_call_request(claim_for=reporter) is None

    # We can pick a call request off; doing so makes it no longer
    # available
    current_time = timezone.make_aware(datetime(2021, 5, 7, 10, 0, 0))
    time_machine.move_to(current_time)
    enqueue(ten_locations)
    assert CallRequest.available_requests().count() == 10
    call_request = CallRequest.get_call_request(claim_for=reporter)
    assert call_request.claimed_by == reporter
    assert call_request.claimed_until >= current_time + timedelta(hours=1)
    assert CallRequest.available_requests().count() == 9

    # We can pick a call request off with no reporter; this leaves it unclaimed
    call_request = CallRequest.get_call_request()
    assert call_request
    assert call_request.claimed_by is None
    assert call_request.claimed_until is None
    assert CallRequest.available_requests().count() == 9

    # All of our ten_locations are in OR; we we ask for an OR site, we
    # get one
    call_request = CallRequest.get_call_request(claim_for=reporter, state="OR")
    assert call_request
    assert call_request.location.state.abbreviation == "OR"

    # Update several locations to be in CA (yes, this will lead to
    # counties' states being inconsistent) and ensure we can get both
    # CA and OR out still.
    for loc in ten_locations[:5]:
        loc.state = State.objects.filter(abbreviation="CA").get()
        loc.save()

    call_request = CallRequest.get_call_request(claim_for=reporter, state="CA")
    assert call_request.location.state.abbreviation == "CA"
    call_request = CallRequest.get_call_request(claim_for=reporter, state="OR")
    assert call_request.location.state.abbreviation == "OR"

    # If we're not limited by state, we'll get both:
    seen_states = {}
    while CallRequest.available_requests().count() > 0:
        call_request = CallRequest.get_call_request(claim_for=reporter)
        seen_states[call_request.location.state.abbreviation] = True
    assert seen_states == {"OR": True, "CA": True}
    assert CallRequest.get_call_request(claim_for=reporter) is None

    # If we move ahead by two hours, we have available requests again,
    # and we can get them
    current_time += timedelta(hours=2)
    time_machine.move_to(current_time)
    assert CallRequest.available_requests().count() == 10
    assert CallRequest.get_call_request(claim_for=reporter) is not None

@pytest.mark.django_db()
def test_get_call_request_priorities(ten_locations: List[Location]) -> None:
    reporter = Reporter.objects.get_or_create(external_id="test:1")[0]
    enqueue(ten_locations)
    for i in range(10):
        CallRequest.objects.filter(location=ten_locations[i]).update(
            priority_group=(i % 2) + 5,
            priority=i + 11,
        )
    seen_priorities: List[Tuple[int, int]] = []
    while CallRequest.available_requests().count():
        call_request = CallRequest.get_call_request(claim_for=reporter)
        seen_priorities.append((call_request.priority_group, call_request.priority))

    # Start with the lowest priority group, but the highest piority
    # within that; then go down the pririties inside the group.
    assert seen_priorities == [
        (5, 19),
        (5, 17),
        (5, 15),
        (5, 13),
        (5, 11),
        (6, 20),
        (6, 18),
        (6, 16),
        (6, 14),
        (6, 12),
    ]

@pytest.mark.django_db()
def test_call_mark_completed(
    ten_locations: List[Location], time_machine: Coordinates
) -> None:
    reporter = Reporter.objects.get_or_create(external_id="test:1")[0]
    current_time = timezone.make_aware(datetime(2021, 5, 7, 10, 0, 0))
    time_machine.move_to(current_time)

    # Enqueue after the time_machine change, so they get the right vested_at
    enqueue(ten_locations)
    assert CallRequest.available_requests().count() == 10
    call_request = CallRequest.get_call_request(claim_for=reporter)
    assert CallRequest.available_requests().count() == 9

    # We we submit a report for that location, it closes the call request
    web = AppointmentTag.objects.get(slug="web")
    report = call_request.location.reports.create(
        reported_by=reporter,
        report_source="ca",
        appointment_tag=web,
    )
    call_request.refresh_from_db()
    assert not call_request.completed
    assert call_request.completed_at is None
    assert CallRequest.objects.count() == 10
    assert CallRequest.available_requests().count() == 9

    CallRequest.mark_completed_by(report)
    assert CallRequest.objects.count() == 10
    assert CallRequest.available_requests().count() == 9
    call_request.refresh_from_db()
    assert call_request.completed
    assert call_request.completed_at is not None
    assert call_request.completed_at >= current_time
    assert report.call_request == call_request

    # Submit a report for the same location again; it has no
    # call_request, and leaves things as-is.
    report = call_request.location.reports.create(
        reported_by=reporter,
        report_source="ca",
        appointment_tag=web,
    )
    CallRequest.mark_completed_by(report)
    call_request.refresh_from_db()
    assert report.call_request is None
    assert CallRequest.objects.count() == 10
    assert CallRequest.available_requests().count() == 9

@pytest.mark.django_db()
def test_call_mark_completed_skip(ten_locations, time_machine: Coordinates) -> None:
    reporter = Reporter.objects.get_or_create(external_id="test:1")[0]
    current_time = timezone.make_aware(datetime(2021, 5, 7, 10, 0, 0))
    time_machine.move_to(current_time)

    web = AppointmentTag.objects.get(slug="web")

    # Put a skip on a location with no call report
    report = ten_locations[0].reports.create(
        reported_by=reporter,
        report_source="ca",
        appointment_tag=web,
    )
    assert CallRequest.objects.count() == 0
    vesting_at = current_time + timedelta(hours=1)
    CallRequest.mark_completed_by(report, enqueue_again_at=vesting_at)
    assert CallRequest.objects.count() == 1
    call_request = CallRequest.objects.all().get()
    assert call_request.location == report.location
    assert call_request.vesting_at == vesting_at
    assert call_request.tip_type == CallRequest.TipType.SCOOBY
    assert call_request.tip_report == report
    assert call_request.priority_group == 99
    assert call_request.priority == 0
    assert CallRequest.available_requests().count() == 0
    assert CallRequest.get_call_request(claim_for=reporter) is None

    # Jump forward after it vests, and we can claim it
    current_time += timedelta(hours=2)
    time_machine.move_to(current_time)
    assert CallRequest.available_requests().count() == 1
    assert CallRequest.get_call_request(claim_for=reporter) == call_request
    CallRequest.objects.all().delete()

    # Make two priority groups
    enqueue(ten_locations)
    for i in range(10):
        CallRequest.objects.filter(location=ten_locations[i]).update(
            priority_group=17,
            priority=i + 11,
        )
    call_request = CallRequest.get_call_request(claim_for=reporter)
    assert call_request.priority_group == 17
    assert call_request.priority == 20
    report = call_request.location.reports.create(
        reported_by=reporter,
        report_source="ca",
        appointment_tag=web,
    )

    # When we skip, we become the _lowest_ priority within the group
    CallRequest.mark_completed_by(
        report, enqueue_again_at=current_time + timedelta(hours=1)
    )
    assert CallRequest.objects.count() == 11
    assert CallRequest.available_requests().count() == 9
    call_request = CallRequest.objects.filter(
        completed=False, location=report.location
    ).get()
    assert call_request.priority_group == 17
    assert call_request.priority == 10


import collections

import numpy as np
import pandas as pd
import pandas.util.testing as pdt

import modules.pd_utils as pd_utils

def extend_cam_id_ts(df):
    """Extend df with cam ids and timestamps.

    This function extracts information about the camera id and the timestamps of
    the first and last frame from the videonames (column curr_img_name
    and succ_img_name).
    >>> data = [('Cam_0_2016-07-19T12:41:22.353295Z--2016-07-19T12:47:02.199607Z.mkv',
    ...          'Cam_0_2016-07-19T12:41:22.353295Z--2016-07-19T12:47:02.199607Z.mkv')]
    >>> df = pd.DataFrame(data = data, columns=['curr_img_name', 'succ_img_name'])
    >>> df = extend_cam_id_ts(df)
    >>> df['curr_start_ts'].loc[0]
    Timestamp('2016-07-19 12:41:22.353295+0000', tz='<iso8601.Utc>')
    """
    curr_meta = df.apply(lambda row: pd_utils.extract_metadata_videoname(row['curr_img_name']), axis=1)
    curr_meta.columns = ['curr_cam_id', 'curr_start_ts', 'curr_end_ts']
    df = df.merge(curr_meta, left_index=True, right_index=True)
    succ_meta = df.apply(lambda row: pd_utils.extract_metadata_videoname(row['succ_img_name']), axis=1)
    succ_meta.columns = ['succ_cam_id', 'succ_start_ts', 'succ_end_ts']
    df = df.merge(succ_meta, left_index=True, right_index=True)
    return df

def extend_distance(df):
    """Extend df with distance.

    >>> data = [('A', '[[1,0,3],[1,0,4]]')]
    >>> df = pd.DataFrame(data=data, columns=['foo', 'warp_matrix'])
    >>> extend_distance(df)
    >>> df['distance'].loc[0]
    5.0
    """
    def get_distance(row):
        if pd.isnull(row['warp_matrix']):
            return np.NaN
        else:
            return np.linalg.norm(pd_utils.str_to_numpy(row['warp_matrix'])[:, 2])
    df['distance'] = df.apply(lambda row: get_distance(row), axis=1)

# %%
def load_ecc_csv_to_pd(path):
    """Load the data from the csv file to a pandas df."""
    names = ['id', 'curr_img_name', 'succ_img_name', 'cc', 'warp_matrix']
    df = pd.read_csv(path, sep=';', names=names, index_col=0)
    return df
# %%

def load_and_prepare(path):
    """Load data from ecc csv file and prepare it for further analysis."""
    df = load_ecc_csv_to_pd(path)
    df = extend_cam_id_ts(df)
    extend_distance(df)
    return df

def get_intervalls(path, accuracy):
    df = load_and_prepare(path)

    pdt.assert_series_equal(df['succ_cam_id'], df['curr_cam_id'], check_names=False)

    df_prep = df.drop(['warp_matrix', 'cc', 'succ_cam_id'], axis=1)

    # `df_prep['distance']` beschreibt die Bewegung welche von `curr_start` zu `succ_start` stattgefunden hat.

    shifted = df[['curr_img_name', 'curr_start_ts', 'curr_end_ts']].shift(1)
    shifted.columns = ['pred_img_name', 'pred_start_ts', 'pred_end_ts']

    df_ext = df_prep.merge(shifted, left_index=True, right_index=True)

    df_selection = df_ext[(df_ext['distance'] > accuracy)]

    # calculate all intervals
    intervals = []
    start = None

    interval = collections.namedtuple('interval', ['start', 'end', 'info'])

    for i, (index, row) in enumerate(df_selection.iterrows()):

        # Das erste Intervall beginnt mit dem ersten Video im Jahr
        if i == 0:

            # Start mit dem ersten Frame im Jahr.
            start = df_ext.loc[0]['curr_img_name']
            end = df_ext.loc[index]['pred_img_name']
            intervals.append(interval(start, end, 'ecc stable'))

            # Unsicheres Intervall
            # Da immer nur der erste Frame eines jeden Videos mit dem ersten
            # Frame des darauffolgenden Videos verglichen wurden, kann
            # eine mögliche Bewegung in dem Video, bevor die Veränderung
            # messbar war, stattgefunden haben.
            start = df_ext.loc[index]['curr_img_name']
            end = df_ext.loc[index]['curr_img_name']
            intervals.append(interval(start, end, 'ecc possible movement'))

            start = df_ext.loc[index]['succ_img_name']

        else:
            # Überpüft ob bereits im vorherigen Video eine Bewegung stattgefunden
            # hat.
            if (index - 1) not in df_selection.index.values:
                # Sicheres Intervall
                end = df_ext.loc[index]['pred_img_name']
                intervals.append(interval(start, end, 'ecc stable'))

                # Unsicheres Intervall
                start = df_ext.loc[index]['curr_img_name']
                end = df_ext.loc[index]['curr_img_name']
                intervals.append(interval(start, end, 'ecc possible movement'))
            else:
                # einzelnes Video welches direkt auf ein Video folgt,
                # bei welchem bereits auch schon eine Bewegung stattgefunden hat.
                start = df_ext.loc[index]['curr_img_name']
                end = df_ext.loc[index]['curr_img_name']
                intervals.append(interval(start, end, 'ecc double movement'))

            start = df_ext.loc[index]['succ_img_name']

            if i == len(df_selection) - 1:
                end = df_ext.loc[max(df_ext.index.values)]['succ_img_name']
                intervals.append(interval(start, end, 'last interval'))
    return intervals


"""Utility functions for Jupyter Book."""

import string
import argparse
import os
import os.path as op
import yaml
from glob import glob
from collections import Counter
from string import ascii_lowercase
from tqdm import tqdm
import nbformat as nbf
import jupytext as jpt
from nbformat.v4.nbbase import new_markdown_cell, new_notebook
from .page import run_ntbk

from . import __version__

##############################################################################
# CLI utilities

def print_color(msg, style):
    endc = "\033[0m"
    bcolors = dict(
        blue="\033[94m",
        green="\033[92m",
        orange="\033[93m",
        red="\033[91m",
        bold="\033[1m",
        underline="\033[4m",
    )
    print(bcolors[style] + msg + endc)

def print_message_box(msg):
    border = "================================================================================"
    print_color("\n\n" + border + "\n\n", "green")
    print(msg)
    print_color("\n\n" + border + "\n\n", "green")

def _error(msg):
    msg = "\n\n\033[91m==========\033[0m\n{}\n\033[91m==========\033[0m\n".format(msg)
    return ValueError(msg)

def str2bool(msg):
    if msg.lower() in ("yes", "true", "t", "y", "1"):
        return True
    elif msg.lower() in ("no", "false", "f", "n", "0"):
        return False
    else:
        raise argparse.ArgumentTypeError("Boolean value expected. Got: {}".format(msg))

##############################################################################
# Book conversion formatting

ALLOWED_CHARACTERS = string.ascii_letters + "-_/." + string.digits

def _check_url_page(url_page, content_folder_name):
    """Check that the page URL matches certain conditions."""
    if not all(ii in ALLOWED_CHARACTERS for ii in url_page):
        raise ValueError("Found unsupported character in filename: {}".format(url_page))
    if "." in os.path.splitext(url_page)[-1]:
        raise _error(
            "A toc.yml entry links to a file directly. You should strip the file suffix.\n"
            "Please change {} to {}".format(url_page, os.path.splitext(url_page)[0])
        )
    if any(
        url_page.startswith(ii)
        for ii in [content_folder_name, os.sep + content_folder_name]
    ):
        raise ValueError(
            "It looks like you have a page URL that starts with your content folder's name."
            "page URLs should be *relative* to the content folder. Here is the page URL: {}".format(
                url_page
            )
        )
    if not url_page.startswith(os.sep):
        raise ValueError(
            "Page URLs need to start with {}. Here is the page URL: {}".format(os.sep, url_page)
        )

def _prepare_url(url):
    """Prep the formatting for a url."""
    # Strip suffixes and prefixes of the URL
    if not url.startswith("/"):
        url = "/" + url

    # Standardize the quotes character
    url = url.replace('"', "'")

    # Make sure it ends in "HTML"
    if not url.endswith(".html"):
        url = op.splitext(url)[0] + ".html"
    return url

def _file_newer_than(path1, path2):
    """Check whether file at path1 is newer than path2."""
    return os.stat(path1).st_mtime > os.stat(path2).st_mtime

def _minor_release(version):
    """Excludes the patch release from a version number, such that
    only major and minor versions face strict matching."""
    major_minor_version = version.rsplit('.', 1)[0]
    if len(major_minor_version .split('.')) != 2:
        raise ValueError("Version doesn't conform to `major.minor.patch` format.")
    return major_minor_version

def _check_book_versions(path_book):
    """Check whether the version of a book matches the version of the
    CLI that's building it.
    """

    with open(op.join(path_book, "_config.yml"), 'r') as ff:
        config_version = yaml.safe_load(ff.read()).get(
            "jupyter_book_version"
        )

    if config_version is None:
        raise _error(
            "Couldn't find the version for your Jupyter Book.\n"
            f"Try upgrading it with `jupyter-book upgrade {path_book}"
        )

    if _minor_release(config_version) != _minor_release(__version__):
        raise _error(
            f"The version of the book you are modifying doesn't match the\n"
            "version of the command-line tool that you're using. Please run\n"
            "\n"
            f"    jupyter-book upgrade {path_book} \n"
            "\n"
            "to upgrade your book to the CLI version.\n"
            "\n"
            f"This book's version: {config_version}\n"
            f"Your CLI's version: {__version__}\n"
            "\n"
            "See above for the error message."
        )

    return True

def _is_jupytext_file(ntbk):
    """Infer whether a notebook node was created from a Jupytext Markdown file.

    Right now, this just tries to guess based on whether there's a particular piece of
    metadata in the notebook.
    """
    jupytext_meta = ntbk.get('metadata', {}).get('jupytext')
    if jupytext_meta is None:
        return False
    else:
        return jupytext_meta.get('notebook_metadata_filter', '') != "-all"

def run_pages(path, kernel_name='python3'):
    "Run a collection of notebooks. Each will be run in-place."
    if not op.exists(path):
        raise ValueError(f"Couldn't find anything at the path provided: {path}")
    if path.endswith('.ipynb'):
        print('Running single notebook...')
        ipynb_files = [path]
    else:
        ipynb_files = glob(op.join(path, '**', '*.ipynb'), recursive=True)
        n_notebooks = len(ipynb_files)
        if n_notebooks == 0:
            raise ValueError(f"No notebooks were found in the provided folder: {path}")
        print(f"Running a folder of notebooks, {n_notebooks} in total.")

    failed_files = []
    for ifile in tqdm(ipynb_files):
        ntbk = nbf.read(ifile, nbf.NO_CONVERT)
        try:
            ntbk = run_ntbk(ntbk, op.dirname(ifile))
            nbf.write(ntbk, ifile)
        except Exception:
            failed_files.append(ifile)

    if len(failed_files) > 0:
        print('Finished printing with these failing pages:')
        for ifile in failed_files:
            print(ifile)

def _content_to_words(content, max_words=100):
    """Convert a string of content into a list of unique words."""
    common_words = [
        "the", "of", "to", "and", "a", "in", "is", "it", "you",
        "that", "he", "was", "for", "on", "are", "with", "as", "i",
        "his", "they", "be", "at", "one", "have", "this", "from",
        "or", "had", "by", "hot", "word", "but", "what", "some", "we",
        "can", "out", "other", "were", "all", "there", "when", "up", "use",
        "your", "how", "said", "an", "each", "she", "which", "do", "their",
        "time", "if", "will", "way", "about", "many", "then", "them", "write",
        "would", "like", "so", "these", "her", "long", "make", "thing", "see",
        "him", "two", "has", "look", "more", "day", "could", "go", "come", "did",
        "number", "sound", "no", "most", "people", "my", "over", "know",
        "water", "than", "call", "first", "who", "may", "down", "side",
        "been", "now", "find"
    ]

    # Replace characters with spaces
    words = content.replace('\n', ' ')
    space_characters = '!@#[]()-{}`:=/\\,.?'
    for char in space_characters:
        words = words.replace(char, ' ')

    # Collect a list of uncommon english words
    new_words = []
    for word in words.split():
        if 'http' in word:
            continue
        word = ''.join([char for char in word.lower() if char in ascii_lowercase])
        if (len(word) == 0) or (word in common_words):
            continue
        new_words.append(word)

    counts = Counter(new_words)
    out_words = [iword[0] for iword in counts.most_common()[:max_words]]
    return out_words

def load_ntbk(path_ntbk):
    """Load a notebook from a text or ipynb file, reading out YAML header as needed.

    Parameters
    ----------
    path_ntbk : string
        Path to an ipynb file or a text file.

    Returns
    -------
    ntbk : instance of NotebookNode
        A Jupyter Notebook created from `path_ntbk`. If `path_ntbk` is an ipynb
        file, this will simply read it in. If a text file, then Jupytext will
        be used to read in the file. If Jupytext metadata is not found, then
        the content of `path_ntbk` will be read into a single markdown cell
        in the notebook.

    yaml_extra : list of yaml metadata
        If any YAML metadata is found as a header in `path_ntbk`, it will be
        collected and returned in this object.
    """
    # Read in the notebook with Jupytext
    ntbk = jpt.read(path_ntbk)
    suff = op.splitext(path_ntbk)[-1]

    # If the first cell has YAML metadata we'll convert it into notebook metadata
    if ntbk.cells[0].source.startswith('---') and ntbk.cells[0].source.endswith('---'):
        yaml_extra = ntbk.cells.pop(0).source.replace('---', '').strip()
    else:
        yaml_extra = None

    # If the file was markdown and didn't have any jupytext frontmatter
    # Just add in the raw source
    if not _is_jupytext_file(ntbk) and suff in ['.md', 'markdown']:
        # Recover the Markdown content
        md = jpt.writes(ntbk, 'md')
        # Replace the notebook with a new one, made of just one Markdown cell
        ntbk = new_notebook(cells=[new_markdown_cell(md)])

    # Add the extra YAML header info if it's there
    if yaml_extra:
        ntbk.metadata['yaml_header'] = yaml_extra

    return ntbk


# -*- coding: utf-8 -*-
# Time : 2023/8/28 17:10
# Author : chen
# Software: PyCharm
import matplotlib.pyplot as plt
import numpy as np
from sklearn.svm import SVC
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import PolynomialFeatures
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
# File : 练习2.py
def main():
    # 1.完成数据集的加载apple.txt
    datas = np.loadtxt('apple.txt',delimiter=',')
    # 2.获取特征和标签
    x = datas[:,:2]
    y = datas[:,-1:]
    # 3.进行特征缩放
    x = StandardScaler().fit_transform(x)
    # 4.对其添加常数项
    x = PolynomialFeatures().fit_transform(x)
    # 5.将数据集洗牌
    m = len(x)
    np.random.seed(0)
    np_perm = np.random.permutation(m)
    x = x[np_perm]
    y = y[np_perm]
    # 6.合理分割成训练集和测试集
    train_x,test_x,train_y,test_y = train_test_split(x,y)
    # 7.正确调用SVM库函数（惩罚系数0.6，使用高斯核函数）
    s = SVC(
        C=0.6,
        kernel='rbf'
    )
    # 8.训练模型
    s.fit(train_x,train_y)
    # 9.进行模型预测
    test_p = s.predict(test_x)
    print(test_p)
    # 10.求出训练集准确率
    print(s.score(train_x,train_y))
    # 11.求出测试集的准确率
    print(s.score(test_x,test_y))
    # 12.输出混淆矩阵
    print(confusion_matrix(test_y,s.predict(test_x)))
    # 13.输出分类报告
    print(classification_report(test_y,s.predict(test_x)))
    # 14.输出支持向量的索引
    print(s.support_)
    # 15.输出支持向量的列表
    print(s.support_vectors_)
    # 16.输出支持向量所属每个类别的个数
    print(s.n_support_)
    # 17.绘制整个样本散点图
    plt.scatter(test_x[:,1],test_x[:,2],c='r',marker='+')
    plt.show()
if __name__ == '__main__':
    main()


# from app import db
from flask_sqlalchemy import SQLAlchemy
db = SQLAlchemy()

class Images(db.Model):
    __tablename__="images"
    id=db.Column(db.Integer,primary_key=True)
    timestamp=db.Column(db.DateTime,nullable=False)
    image_link=db.Column(db.String,nullable=False)

    def __init__(self,timestamp,image_link):
        self.timestamp=timestamp
        self.image_link=image_link


# Definition for singly-linked list.
from typing import Optional

# 合并两个有序数组

class ListNode:
    def __init__(self, val=0, next=None):
        self.val = val
        self.next = next

# 双指针， 代码较繁琐
class Solution:
    def mergeTwoLists(self, list1: Optional[ListNode], list2: Optional[ListNode]) -> Optional[ListNode]:
        a = list1
        b = list2
        t = ListNode()
        if a is None and b is None:
            return
        elif a is not None and b is None:
            t = a
            c = t
        elif a is None and b is not None:
            t = b
            c = t
        else:
            while a is not None and b is not None:
                if a.val <= b.val and t.next is None:
                    t = a
                    a = a.next
                    c = t
                elif a.val > b.val and t.next is None:
                    t = b
                    b = b.next
                    c = t
                elif a.val <= b.val:
                    t.next = a
                    a = a.next
                    t = t.next
                else:
                    t.next = b
                    b = b.next
                    t = t.next
            if a is not None:
                t.next = a
            if b is not None:
                t.next = b
        return c

    def mergeTwoLists_LC(self, l1: ListNode, l2: ListNode) -> ListNode:
        prehead = ListNode(-1)  # 头指针

        prev = prehead
        while l1 and l2:
            if l1.val <= l2.val:
                prev.next = l1
                l1 = l1.next
            else:
                prev.next = l2
                l2 = l2.next
            prev = prev.next

        prev.next = l1 if l1 is not None else l2

        return prehead.next

l1 = ListNode(1)
# l2 = ListNode(2)
# l3 = ListNode(3)

# l1.next = l2
# l2.next = l3

m1 = ListNode(0)
# m2 = ListNode(2)
# m3 = ListNode(3)

# m1.next = m2
# m2.next = m3

s = Solution()
s.mergeTwoLists(l1, None)


from kubernetes import client
import yaml

def handle_env(file):
    with open(file,'r') as f:
        env_data = yaml.safe_load(f)
    f.close()
    return [client.V1EnvVar(name=key,value=value) for key,value in env_data.items()]

def template(context):
    """
    handle yml env
    """
    name = context.get("name")
    labels = {"name": name}
    image_tag=context["name"].split('-')[1]
    image = context["image_namespace"]+image_tag+":"+context["image_branch"]
    args = [arg for arg in context["args"]] if context.get("args") else None
    limits,requests = context["resources"]["limits"],context["resources"]["requests"]
    replicas = context["replicas"][name]

    """
    handle cmdb env
    """
    filename = "env_" + name.split("-")[1] + ".yml"
    env = handle_env("/tmp/{}".format(filename))
    workingDir = context["workingDir"]

    """
    k8s yaml 组件模块
    """

    containers = [
                client.V1Container(
                    name = name,
                    image = image,
                    env = env,
                    args = args,
                    image_pull_policy = "Always",
                    # readiness_probe=client.V1Probe(_exec=client.V1ExecAction(command=['cat','/tmp/container_ready']),initial_delay_seconds=10, period_seconds=5),
                    resources = client.V1ResourceRequirements(limits = limits,requests = requests),
                    security_context = client.V1SecurityContext(privileged=True),
                    working_dir = working_dir,
                )
    ]

    template = client.V1PodTemplateSpec(
                    metadata = client.V1ObjectMeta(labels=labels),
                    spec = client.V1PodSpec(
                                containers = containers,
                                dns_policy = "ClusterFirst",
                                image_pull_secrets = [client.V1LocalObjectReference(name="image-pull-secret")],
                                restart_policy = "Always"
                            )

               )

    spec = client.V1StatefulSetSpec(
                service_name = name,
                replicas = replicas,
                selector = client.V1LabelSelector(match_labels=labels),
                template = template
           )

    return client.V1StatefulSet(
        api_version = "apps/v1",
        kind = "StatefulSet",
        metadata = client.V1ObjectMeta(name = name,labels = labels),
        spec = spec
    )


from PyQt5 import QtCore,QtWidgets

class Ui_Sure(QtWidgets.QDialog):
    def setupUi(self, Sure):
        Sure.setObjectName("Sure")
        Sure.resize(200, 100)
        self.Sure = QtWidgets.QPushButton(Sure)
        self.Sure.setGeometry(QtCore.QRect(62.5, 37.5, 75, 25))
        self.Sure.setObjectName("Sure")
        self.Sure.raise_()

        self.retranslateUi(Sure)
        QtCore.QMetaObject.connectSlotsByName(Sure)

    def retranslateUi(self, Sure):
        _translate = QtCore.QCoreApplication.translate
        Sure.setWindowTitle(_translate("Sure", "Diagol"))
        self.Sure.setText(_translate("Sure", "确定"))

# if __name__=="__main__":
#     import sys
#     app=QtWidgets.QApplication(sys.argv)
#     dialog=QtWidgets.QDialog()
#     ui=Ui_Sure()
#     ui.setupUi(dialog)
#     dialog.show()
#     sys.exit(app.exec_())


import PySimpleGUI as sg
import os.path
import textwrap
from matplotlib.backends.backend_tkagg import FigureCanvasTkAgg
import UV_Vis_reader

def draw_figure(canvas, figure):
    figure_canvas_agg = FigureCanvasTkAgg(figure, canvas)
    figure_canvas_agg.draw()
    figure_canvas_agg.get_tk_widget().pack(side='top', fill='both')
    return figure_canvas_agg

fig_canvas_agg = []
folder = []
t_unit = 's'
t_adj = 1
x_range = None
start = 0
inc = None
region_limits = []

sg.theme('LightBrown2')

file_list_column = [
    [sg.Text("UV-Vis Spectra Processor", font='Any 18')],
    #[sg.Text("This program shows multiple UV-Vis spectra on a single figure, integrate a peak in multiple UV-Vis spectra and graph these results")],
    [
        sg.Text("UV-Vis files folder"),
        sg.Input(size=(25, 1), enable_events=True, key="-FOLDER-"),
        sg.FolderBrowse()
    ],
    [sg.Listbox(values=[], enable_events=True, size=(40, 10), key="-FILE LIST-")],
    [
        sg.Text("Time unit"),
        sg.Combo(["s", "min", "h"], default_value="s", enable_events=True, key="-t_unit-")
    ],
    [sg.Text("Show absorption spectra", font=("Arial", 12, ' underline'))],
    [
        sg.Text("First spectrum"),
        sg.InputText(size=(5, 1), key="-FS-"),
        sg.Text("Spectra increment"),
        sg.InputText(size=(5, 1), key="-SI-"),
    ],
    [
        sg.Text("Lower x bound"),
        sg.InputText(size=(5, 1), key="-LB1-"),
        sg.Text("Upper x bound"),
        sg.InputText(size=(5, 1), key="-UB1-"),
    ],
    [sg.Button('Submit/Update', key="-SUBMIT1-")],
    [sg.Text("Calculate integrals and export", font=("Arial", 12, ' underline'))],
    [
        sg.Text("Lower range limit"),
        sg.InputText(size=(5, 1), key="-LB2-"),
        sg.Text("Upper range limit"),
        sg.InputText(size=(5, 1), key="-UB2-"),
    ],
    [
        sg.Text("Export path"),
        sg.Input(size=(25, 1), enable_events=True, key="-FILE-"),
        sg.FileBrowse()
    ],
    [sg.Button('Submit/Update', key="-SUBMIT2-")],
]

image_viewer_column = [
    [sg.Text('Plot', font=("Arial", 12, ' underline'))],
    [sg.Frame("Frame", [[sg.Canvas(key='-CANVAS-', expand_x=True, expand_y=True)]], size=(450, 450))]
]

layout = [
    [
        sg.Column(file_list_column),
        sg.VSeperator(),
        sg.Column(image_viewer_column),
    ]
]

window = sg.Window("UV-Vis Spectra Processor", layout, resizable=True, finalize=True)

while True:
    event, values = window.read()
    if event == "Exit" or event == sg.WIN_CLOSED:
        break
    # Folder name was filled in, make a list of files in the folder
    if event == "-FOLDER-":
        folder = values["-FOLDER-"]
        try:
            # Get list of files in folder
            file_list = os.listdir(folder)
        except:
            file_list = []

        fnames = [
            f
            for f in file_list
            if os.path.isfile(os.path.join(folder, f))
            and f.lower().endswith((".txt"))
        ]
        window["-FILE LIST-"].update(fnames)
    elif event == "-FILE-":
        exportpath = values["-FILE-"]

    elif event == "-t_unit-":
        t_unit = values["-t_unit-"]
        t_adj = UV_Vis_reader.get_t_adj(t_unit)

    elif event == "-SUBMIT1-":
        try:
            if values["-FS-"] != '': start = int(values["-FS-"])
            if values["-SI-"] != '': inc = int(values["-SI-"])
            if values["-LB1-"] != '' or values["-LB1-"] != '': x_range = tuple([float(values["-LB1-"]), float(values["-UB1-"])])
            if fig_canvas_agg: fig_canvas_agg.get_tk_widget().forget()
            fig = UV_Vis_reader.plot_spec(folder, x_range=x_range, start=start, inc=inc, t_adj=t_adj)
            fig_canvas_agg = draw_figure(window['-CANVAS-'].TKCanvas, fig)
        except:
            if not folder:
                sg.Popup('Select data folder')
            else:
                sg.Popup('Error detected. Please refer to operating manual')

    elif event == "-SUBMIT2-":
        try:
            region_limits = [tuple([float(values["-LB2-"]), float(values["-UB2-"])])]
            arr_res = UV_Vis_reader.integral_calc(folder, region_limits, t_adj=t_adj)
            UV_Vis_reader.spec_export(arr_res, region_limits, exportpath)
            if fig_canvas_agg: fig_canvas_agg.get_tk_widget().forget()
            fig = UV_Vis_reader.plot_integral(arr_res, t_unit=t_unit)
            fig_canvas_agg = draw_figure(window['-CANVAS-'].TKCanvas, fig)
        except:
            try:
                region_limits = [tuple([float(values["-LB2-"]), float(values["-UB2-"])])]
                try:
                    arr_res = UV_Vis_reader.integral_calc(folder, region_limits, t_adj=t_adj)
                    if fig_canvas_agg: fig_canvas_agg.get_tk_widget().forget()
                    fig = UV_Vis_reader.plot_integral(arr_res, t_unit=t_unit)
                    fig_canvas_agg = draw_figure(window['-CANVAS-'].TKCanvas, fig)
                except:
                    if not folder:
                        sg.Popup('Select data folder')
                    else:
                        sg.Popup('Error detected - please refer to operating manual')
            except:
                if not folder:
                    sg.Popup('Select data folder')
                elif not region_limits:
                    sg.Popup('Input lower and upper range limit')
                else:
                    sg.Popup('Error detected - please refer to operating manual')

window.close()


import math
from typing import Callable, Any
from dataclasses import dataclass
from datetime import datetime, timedelta

@dataclass
class Event:
    time_until_happen: timedelta
    callback: Callable
    data: Any

    def __repr__(self):
        return self.callback.__qualname__

    __str__ = __repr__

class EventCalendar:
    def __init__(self):
        self.calendar: list[Event] = []

    def push(self, time: float, callback: Callable, **data: Any):
        """
        Add event to calendar ordered by time
        :param time: fire time
        :param callback: callback_function
        :param data: custom calback data.
        """
        event = Event(time_until_happen=time, callback=callback, data=data)
        edge_indexes = [0, len(self.calendar)]
        while edge_indexes[0] != edge_indexes[1]:
            pivot = math.floor((edge_indexes[0]+edge_indexes[1])/2)
            if time >= self.calendar[pivot].time_until_happen:
                edge_indexes[0] = pivot + 1
            else:
                edge_indexes[1] = pivot

        self.calendar.insert(edge_indexes[0], event)

    def pop(self):
        """
        remove nearest event
        :return (float): fire time
        :return (callable): callback function
        """
        return self.calendar.pop(0)

    @property
    def is_empty(self):
        """
        Check wheter calendar is empty
        :return:
        """
        return len(self.calendar) == 0

    def update_events(self, time_step: timedelta):
        for event in self.calendar:
            event.time_until_happen -= time_step

if __name__ == '__main__':
    print("="*50)
    event_calendar = EventCalendar()
    event_calendar.push(0, lambda: print('Primeira função'))
    event_calendar.push(15, lambda: print('Terceira função'))
    event_calendar.push(2, lambda: print('Segunda função'))
    for event in event_calendar.calendar:
        event.callback()
    print("="*50)
    event_calendar.pop()
    for event in event_calendar.calendar:
        event.callback()
    print("="*50)
    print(event_calendar.is_empty)
    print("=" * 50)
    event_calendar.pop()
    event_calendar.pop()

    print(event_calendar.is_empty)


import pandas as pd
import numpy as np
import re
import datetime
import os
import sys

# print name of file to be decoded
print(sys.argv[1])

# Import Tika PDF parser
from tika import parser

# input-file
path = sys.argv[1]

# Read pdf-file
try:
    rawText = parser.from_file(path)

    rawList = rawText['content'].splitlines()
except:
    print('Ingen PDF-fil')

# Define approved grades
approved_grades = ['A', 'B', 'C', 'D', 'E', 'G', 'VG', 'MVG', 'AB', 'BA', '3', '4', '5', 'P']

# Set required hp
points_req = 15

# Set date interval
# 1 september -> 1 februari
start_date = '2019-09-1'
end_date = '2020-02-01'

def decode_ladok_extract(rawList):
    # Decode extracts from Ladok (containing all of the students records from all universites)
    completed_courses = []
    prev_line = 'None'
    prev2_line = 'None'

    for line in rawList:
        if line != '': # skip empty lines
            if bool(re.search(r'20',  line)): # check for lines containing 20, indicating a date
                splitted = line.split()
                try:
                    if splitted[-2] in approved_grades: # check if line contains a grade in the correct position
                        course_name = ' '.join(splitted[0:-4])
                        if len(course_name) < 1:
                            course_name = prev2_line + ' ' + prev_line
                        course_points = splitted[-3].strip('()')
                        course_complete_date = splitted[-1]
                        completed_courses.append([course_name, course_points, course_complete_date])
                    if splitted[-3] in approved_grades and ')' in splitted[-4]: # check if line contains a grade in the correct position
                        course_name = ' '.join(splitted[0:-5])
                        if len(course_name) < 1:
                            course_name = prev2_line + ' ' + prev_line
                        course_points = splitted[-4].strip('()')
                        course_complete_date = splitted[-2]
                        completed_courses.append([course_name, course_points, course_complete_date])
                except:
                    continue
            prev2_line = prev_line.strip()
            prev_line = line.strip()

    df = pd.DataFrame(completed_courses, columns=['Object', 'HP', 'Date'])
    df['HP'] = pd.to_numeric(df['HP'])
    df['Date'] = pd.to_datetime(df['Date'])

    approval = calculate_approval(df)

    return approval

def decode_uni_extract(rawList):
    # Decode extract from students university
    completed_courses = []
    avklarade = 0
    ej_avklarade = 0
    prev_line = 'None' # keep track of previous
    prev2_line = 'None' # keep track of the line before the previous line

    rawList_reduced = rawList[rawList.index('Resultatintyg'):]

    for line in rawList_reduced:
        if bool(re.search('Avklarade',  line)):
            avklarade = 1
            ej_avklarade = 0
        if bool(re.search('slutrapporterade', line)):
            avklarade = 0
            ej_avklarade = 1
        # check for completed hp in completed courses
        if line != '' and avklarade == 1 and ej_avklarade == 0:
            if bool(re.search(r'20',  line)):
                splitted = line.split()
                if splitted[-3] in approved_grades and '(' in splitted[-5]:
                    course_name = ' '.join(splitted[1:-5])
                    if len(course_name) < 1:
                        course_name = prev2_line + ' ' + prev_line
                    course_points = splitted[-5].strip('()').replace(',','.')
                    course_complete_date = splitted[-2].replace('\xad', '-')
                    completed_courses.append([course_name, course_points, course_complete_date])
            prev2_line = prev_line.strip()
            prev_line = line.strip()

        # check for completed hp in NON-completed courses
        if line != '' and avklarade == 0 and ej_avklarade ==1:
            if bool(re.search(r'20',  line)):
                splitted = line.split()
                if splitted[-3] in approved_grades:
                    course_name = ' '.join(splitted[1:-5])
                    if len(course_name) < 1:
                        course_name = prev2_line + ' ' + prev_line
                    course_points = splitted[-5].strip('()').replace(',','.')
                    course_complete_date = splitted[-2].replace('\xad', '-')
                    completed_courses.append([course_name, course_points, course_complete_date])
            prev2_line = prev_line.strip()
            prev_line = line.strip()

    # create dataframe of completed courses
    df = pd.DataFrame(completed_courses, columns=['Object', 'HP', 'Date'])
    df['HP'] = pd.to_numeric(df['HP'])
    df['Date'] = pd.to_datetime(df['Date'])

    approval = calculate_approval(df)

    return approval

def calculate_approval(df):
    df = df[(df['Date'] > start_date) & (df['Date'] < end_date)]

    if df.HP.sum() > points_req:
        approval = 'Yes'
    else:
        approval = 'No'

    print(df)
    print('Total HP during period:', df.HP.sum())

    print('Approved?', approval)

    return approval

# Check what type of student record to process
if 'Nationellt studieintyg' in rawList and 'Sveriges högskolor och universitet' in rawList:
    print('Ladok extract')
    decode_ladok_extract(rawList)
elif 'Registreringsintyg' in rawList:
    print('Endast nationella studieintyg eller resultatintyg godkänns, '
          'du har skickat in ett registreringsintyg.')
elif 'Resultatintyg' in rawList:
    print('Uni extract')
    decode_uni_extract(rawList)
else:
    print('Ogiltig fil')


# -*- coding: utf-8 -*-
# ------------------------------------------------------------------------------
# Created by Mingfei Chen (lasiafly@gmail.com)
# Created On: 2020-1-20
# ------------------------------------------------------------------------------
import numpy as np

from yacs.config import CfgNode as CN

INF = 1e8

_C = CN()

# working dir
_C.OUTPUT_ROOT = ''

# distribution
_C.DIST_BACKEND = 'nccl'

_C.DEVICE = 'cuda'

_C.WORKERS = 4

_C.PI = 'mAP'

_C.SEED = 42

# cudnn related params
_C.CUDNN = CN()
_C.CUDNN.BENCHMARK = True
_C.CUDNN.DETERMINISTIC = False
_C.CUDNN.ENABLED = True

# dataset
_C.DATASET = CN()
_C.DATASET.FILE = 'hoi_det'
_C.DATASET.NAME = 'HICODetDataset'
_C.DATASET.ROOT = ''
_C.DATASET.MEAN = []
_C.DATASET.STD = []
_C.DATASET.MAX_SIZE = 1333
_C.DATASET.SCALES = [480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800]
_C.DATASET.IMG_NUM_PER_GPU = 2
_C.DATASET.NUM_CLASSES = 91
_C.DATASET.PREFIX = ''

# model
_C.MODEL = CN()
# specific model
_C.MODEL.FILE = ''
_C.MODEL.NAME = ''
# resume
_C.MODEL.RESUME_PATH = ''
_C.MODEL.MASKS = False
_C.MODEL.MODEL_CONFIG = ''

_C.RS = CN()
_C.RS.EMB_DIM = 64
_C.RS.NIDS = 5000
_C.RS.NUM_LAYERS = 6

# backbone
_C.BACKBONE = CN()
_C.BACKBONE.NAME = 'resnet50'
_C.BACKBONE.DIALATION = False
_C.BACKBONE.DEPTH = 50
_C.BACKBONE.STRIDE_IN_1X1 = False
_C.BACKBONE.OUT_FEATURES = ["res2", "res3", "res4", "res5"]

# deformable detr
_C.DEFORMABLE = CN()
_C.DEFORMABLE.WITH_BOX_REFINE = False
_C.DEFORMABLE.TWO_STAGE = False

# transformer
_C.TRANSFORMER = CN()
_C.TRANSFORMER.POSITION_EMBEDDING = 'sine' # choices=('sine', 'learned')
_C.TRANSFORMER.POSITION_EMBEDDING_SCALE = 2 * np.pi
_C.TRANSFORMER.NUM_FEATURE_LEVELS = 4
_C.TRANSFORMER.HIDDEN_DIM = 256
_C.TRANSFORMER.ENC_LAYERS = 6
_C.TRANSFORMER.DEC_LAYERS = 6
_C.TRANSFORMER.DIM_FEEDFORWARD = 2048
_C.TRANSFORMER.DROPOUT = 0.1
_C.TRANSFORMER.NHEADS = 8
_C.TRANSFORMER.NUM_QUERIES = 300
_C.TRANSFORMER.DEC_N_POINTS = 4
_C.TRANSFORMER.ENC_N_POINTS = 4
_C.TRANSFORMER.PRE_NORM = False

# matcher
_C.MATCHER = CN()
_C.MATCHER.COST_CLASS = 1
_C.MATCHER.COST_BBOX = 5
_C.MATCHER.COST_GIOU = 2

# LOSS
_C.LOSS = CN()
_C.LOSS.AUX_LOSS = True
_C.LOSS.DICE_LOSS_COEF = 1
_C.LOSS.CLS_LOSS_COEF = 1
_C.LOSS.ID_LOSS_COEF = 1.0
_C.LOSS.OFFSET_LOSS_COEF = 1
_C.LOSS.BBOX_LOSS_COEF = 5
_C.LOSS.GIOU_LOSS_COEF = 2
_C.LOSS.FOCAL_ALPHA = 0.25
_C.LOSS.EOS_COEF = 0.1

# trainer
_C.TRAINER = CN()
_C.TRAINER.FILE = ''
_C.TRAINER.NAME = ''

# train
_C.TRAIN = CN()
_C.TRAIN.HYP_CONFIG = ''
_C.TRAIN.OPTIMIZER = ''
_C.TRAIN.LR = 0.0002
_C.TRAIN.LR_BACKBONE_NAMES = ["backbone.0"]
_C.TRAIN.LR_BACKBONE = 0.00002
_C.TRAIN.LR_PROJ_NAMES = ['reference_points', 'sampling_offsets']
_C.TRAIN.LR_TRACK_NAMES = ['']
_C.TRAIN.LR_PROJ_MULT = 0.1
_C.TRAIN.MOMENTUM = 0.9
_C.TRAIN.WEIGHT_DECAY = 0.0001
# optimizer SGD
_C.TRAIN.NESTEROV = False
# learning rate scheduler
_C.TRAIN.LR_SCHEDULER = 'MultiStepWithWarmup'
_C.TRAIN.LR_STEPS = [120000, ]
_C.TRAIN.LR_FACTOR = 0.1
_C.TRAIN.LR_DROP = 200
_C.TRAIN.CLIP_MAX_NORM = 0.1
_C.TRAIN.WARMUP_STEP = 500
_C.TRAIN.WARMUP_INIT_FACTOR = 1.0 / 3
_C.TRAIN.MAX_EPOCH = 360000
# train resume
_C.TRAIN.RESUME = False
# input size
_C.TRAIN.INPUT_MIN = 788
_C.TRAIN.INPUT_MAX = 1400
# print freq
_C.TRAIN.PRINT_FREQ = 20
# save checkpoint during train
_C.TRAIN.SAVE_INTERVAL = 5000
_C.TRAIN.SAVE_EVERY_CHECKPOINT = False
# val when train
_C.TRAIN.VAL_WHEN_TRAIN = False

# test
_C.TEST = CN()
# input size
_C.TEST.SCORE_THRESHOLD = 0.3
_C.TEST.NMS_THRESHOLD = 0.5
# test image dir
_C.TEST.IMAGE_DIR = ''
_C.TEST.TEST_SIZE = (512, 768)
_C.TEST.CROP_SZIE = (182, 182)
_C.TEST.MAX_PER_IMG = 100
_C.TEST.NMS_PRE = 1000
_C.TEST.PR_CURVE = False
_C.TEST.OUT_DIR = ''
_C.TEST.AP_IOU = 0.5

def update_config(config, args):
    config.defrost()
    # set cfg using yaml config file
    config.merge_from_file(args.yaml_file)
    # update cfg using args
    config.merge_from_list(args.opts)
    config.freeze()

if __name__ == '__main__':
    import sys
    with open(sys.argv[1], 'w') as f:
        print(_C, file=f)


"""Given a unsorted array of integers, find the longest increasing subarray.

Time: O(n)
Space: O(1)
"""

def longest_increasing_subarray(nums):
    if not nums:
        return 0

    max_length, i = 0, 0

    while i < len(nums):
        curr_length = 1
        while (i + 1) < len(nums) and nums[i] < nums[i + 1]:  # extend streak if possible
            curr_length += 1
            i += 1

        max_length = max(max_length, curr_length)  # update max length
        i += 1

    return max_length


import cv2 as cv

if __name__=="__main__":

    cam = cv.VideoCapture(-1)

    while(True):

        flag, frame = cam.read()

        if(not flag):
            print("Frame grab error")
            continue

        cv.imshow("Frame", frame)

        if(cv.waitKey(1) == ord('q')):
            break

    cv.destroyAllWindows()


"""
Module manages library tools to provide command line interface for the
calculator. Intended to be called from command line.
Contains functions:
- parse_args;
- main;
"""
import argparse
from pycalc.tools.calculator import ExpressionCalculator
from pycalc.tools.parser import ExpressionParser
from pycalc.tools.exceptions import PyCalcBaseException

def parse_args(*args):
    """
    Parse command line arguments using 'argparse' module. Yep it must contain
    'expression' positional arg but it won't work with some expressions and if
    add '--' escape into command line args '--help' won't work. This solution
    is just as good.
    :param args: inserted to call from scripts.
    :return: tuple(argparse.Namespace with module names as attributes,
             list with expression string.
    """
    parser = argparse.ArgumentParser(description='Pure-python command-line '
                                                 'calculator.')
    parser.add_argument('-m', '--use-modules', dest='module', action='append',
                        help='Additional modules to use')
    args = parser.parse_known_args(*args)
    if len(args[1]) == 0:
        raise PyCalcBaseException('No expression was provided.')
    return args

def main(*args):
    """
    Orchestrate creation of 'ExpressionParser' and 'ExpressionCalculator'
    classes and pass created instances parsed arguments.
    Print results of expression.
    :param args: inserted to call from scripts.
    """
    try:
        args = parse_args(*args)
        parser = ExpressionParser()
        calc = ExpressionCalculator(args[1][0], parser.parse_input(args[1][0]),
                                    args[0].module)
        print(calc.explore_data(calc.exp_list))
    except PyCalcBaseException as err:
        print(err)

if __name__ == '__main__':
    main()


from concurrent.futures import ProcessPoolExecutor
from requests import Session
from requests_futures.sessions import FuturesSession

def callback(future):
    print(type(future))
    response = future.result()
    print(response.json()["args"])

session = FuturesSession(executor=ProcessPoolExecutor(max_workers=10),
                         session=Session())
for i in range(10):
    future_response = session.get('http://httpbin.org/get?foo=' + str(i))
    future_response.add_done_callback(callback)

print("done")


from sorting_objects import *
from item_bin import *
import random
import sys
from itertools import permutations

#item_objects = [items('i1', 86), items('i2', 24), items('i3', 91), items('i4', 52), items('i5', 67)]
#item_objects = [items('i1', 86), items('i2', 24), items('i3', 91)]
p = 0.8

ordered_items = sorted(item_objects, key = attrgetter('volume'), reverse = True)
ordered_items2 = sorted(item_objects, key = attrgetter('volume'), reverse = True)

print("ordered items: " + str(ordered_items))

#creates new ordering by assigning a probability to each item and making a weighted selection to add to the order
new_ordering = []
while len(ordered_items2) > 0:
    total_weight = 0
    rank = {}
    weight = {}
    probability = {}
    probability_list = []
    #assigning rank and weight to each item
    for i in range(len(ordered_items2)):
        child = ordered_items2[i]
        rank[child] = i+1 #the paper uses positive integers for rank... should I use i instead of i+1? it does not seem to make a difference...
        weight[child] = (1-p)**(rank[child]) #should this be rank or -rank or i or -i?? the BubbleSearch paper says -i but that means that if 0<p<1, then the items that have a lower score (ranking) are weighted more heavily
        total_weight += weight[child]
        print("child " + str(child))
        print("weight " + str(weight))
        print("total weight " + str(total_weight))
    #calculating probability for each item
    for child in ordered_items2:
        probability[child] = weight[child] / total_weight
        print("probability " + str(probability))
        probability_list.append(weight[child] / total_weight)
    #choosing the item and adding it to the new ordering and deleting it from the original ordering
    choice = (random.choices(ordered_items2, probability_list))[0]
    ordered_items2.remove(choice)
    new_ordering.append(choice)
    print(choice)
    print(ordered_items2)
    print(new_ordering)

#calculates the kendall tau distance between the new ordering and the original ordering
print("kendall tau distance: " + str(kt_dist(ordered_items, new_ordering)))

with open('item_bin.py', 'a') as f:
    sys.stdout = f
    printout = "item_objects_permutation = ["
    for i in range(len(new_ordering)-1):
        printout+=("items" +str(new_ordering[i]) + ", ")
    printout+=("items" + str(new_ordering[len(new_ordering)-1]) + "]")
    print(printout)


#!/usr/bin/env python
# coding: utf-8

# In[21]:

import pandas as pd
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from time import sleep

playlist_URL = input("Enter the link of the playlist: ")

path = r"C:\seleniumDrivers\chromedriver.exe"
options = Options()
options.add_argument("--headless")
browser = webdriver.Chrome(executable_path = path, options = options)
browser.get(playlist_URL)
browser.maximize_window()

# In[22]:

sleep(10)
video_links = browser.find_elements(by=By.XPATH, value="//a[@class='yt-simple-endpoint inline-block style-scope ytd-thumbnail']")
video_links = [vlink.get_attribute("href") for vlink in video_links[1:]]

# stats = browser.find_element(by=By.ID, value='stats')
# stats = browser.find_element(by=By.XPATH, value="//div[@id ='stats'")

pt_title = browser.find_element(by=By.XPATH, value='//div[@class = "dynamic-text-container style-scope yt-dynamic-sizing-formatted-string"]').text

stats = browser.find_element(by=By.XPATH, value='//span[@class = "style-scope yt-formatted-string"]')
n_vid_str= stats.text
n_vid = 0
for i in list(n_vid_str):
    try:
        n = int(i)
        n_vid *=10
        n_vid += n
    except:
        continue

print("Title of the playlist: ", pt_title)
print("Total no. of videos: ", n_vid)

# In[23]:

print("==========Scrapping data==========")

idx = [i.text for i in browser.find_elements(by=By.XPATH, value="//div[@id ='index-container']")][-1]

s_time = 0
idl = []
while  int(idx) != n_vid:
    browser.execute_script("window.scrollBy(0,2000)","")
    idx = [i.text for i in browser.find_elements(by=By.XPATH, value="//div[@id ='index-container']")][-1]
    idl.append(idx)
    print(idx)
    if len(idl)>100:
        if idl[-1] == idl[-20]:
            break

# In[24]:

cmn_link_part = video_links[0][:-1:]
for i in range(len(video_links)+1, n_vid+1):
    video_links.append(cmn_link_part+str(i))

titles = [title.text for title in browser.find_elements(by=By.XPATH, value="//a[@id ='video-title']")]
times = browser.find_elements(by=By.XPATH, value="//span[@class ='style-scope ytd-thumbnail-overlay-time-status-renderer']")

time_list = [t.get_attribute("aria-label") for t in times]

data_dict = {'video_title':titles,
             'video_url':video_links[:int(idx)],
            'time_duration':time_list}

print("==========Finished Scrapping==========")

# In[25]:

spc = [r"/", "?", "+", "-", "&", "%", "$", "@", "!", "(", ")" ,"*", "|",
       "\\", "{", "}", ">", "<", ",", "~", "^"]

df = pd.DataFrame(data_dict)
for i in spc:
    pt_title = pt_title.replace(i, "_")
df.to_csv(pt_title+".csv", index=False)

# In[ ]:


NUMBER_OF_ROUNDS = 10000

inventories = [
  [57],
  [58,93,88,81,72,73,65],
  [65,95],
  [58,80,81,83],
  [58,89,90,96,55],
  [66,73, 87, 58, 62, 67],
  [85, 55, 89],
  [73, 80, 54, 94, 90, 52, 69, 58]
]
inspect = [0,0,0,0,0,0,0,0]
def HandleMoonkey0(item):
  item *= 13
  # item //= 3
  item %= (11*7*13*5*3*17*2*19)
  if item % 11 == 0:
    #throw to monkey 3
    inventories[3].append(item)
  else:
    # throw to monkey 2
    inventories[2].append(item)
def HandleMoonkey1(item):
  item += 2
  # item //= 3
  item %= (11*7*13*5*3*17*2*19)
  if item % 7 == 0:
    #throw to monkey 6
    inventories[6].append(item)
  else:
    # throw to monkey 7
    inventories[7].append(item)
def HandleMoonkey2(item):
  item += 6
  # item //= 3
  item %= (11*7*13*5*3*17*2*19)
  if item % 13 == 0:
    #throw to monkey 3
    inventories[3].append(item)
  else:
    # throw to monkey 5
    inventories[5].append(item)
def HandleMoonkey3(item):
  item *= item
  # item //= 3
  item %= (11*7*13*5*3*17*2*19)
  if item % 5 == 0:
    #throw to monkey 4
    inventories[4].append(item)
  else:
    # throw to monkey 5
    inventories[5].append(item)
def HandleMoonkey4(item):
  item += 3
  # item //= 3
  item %= (11*7*13*5*3*17*2*19)
  if item % 3 == 0:
    #throw to monkey 1
    inventories[1].append(item)
  else:
    # throw to monkey 7
    inventories[7].append(item)
def HandleMoonkey5(item):
  item *= 7
  # item //= 3
  item %= (11*7*13*5*3*17*2*19)
  if item % 17 == 0:
    #throw to monkey 4
    inventories[4].append(item)
  else:
    # throw to monkey 1
    inventories[1].append(item)
def HandleMoonkey6(item):
  item += 4
  # item //= 3
  item %= (11*7*13*5*3*17*2*19)
  if item % 2 == 0:
    #throw to monkey 2
    inventories[2].append(item)
  else:
    # throw to monkey 0
    inventories[0].append(item)
def HandleMoonkey7(item):
  item += 7
  # item //= 3
  item %= (11*7*13*5*3*17*2*19)
  if item % 19 == 0:
    #throw to monkey 6
    inventories[6].append(item)
  else:
    # throw to monkey 0
    inventories[0].append(item)

for round in range(NUMBER_OF_ROUNDS):
  while(inventories[0]):
    HandleMoonkey0(inventories[0].pop(0))
    inspect[0] += 1
  while(inventories[1]):
    HandleMoonkey1(inventories[1].pop(0))
    inspect[1] += 1
  while(inventories[2]):
    HandleMoonkey2(inventories[2].pop(0))
    inspect[2] += 1
  while(inventories[3]):
    HandleMoonkey3(inventories[3].pop(0))
    inspect[3] += 1
  while(inventories[4]):
    HandleMoonkey4(inventories[4].pop(0))
    inspect[4] += 1
  while(inventories[5]):
    HandleMoonkey5(inventories[5].pop(0))
    inspect[5] += 1
  while(inventories[6]):
    HandleMoonkey6(inventories[6].pop(0))
    inspect[6] += 1
  while(inventories[7]):
    HandleMoonkey7(inventories[7].pop(0))
    inspect[7] += 1

  # print('ROUND ' + str(round + 1))
  # for i, inv in enumerate(inventories):
  #   print(f'Monkey {i}: {inv}')
print(inspect)
s = sorted(inspect)
print(s)
print(f'[PART 2] Monkey Buisiness level {s[-1]*s[-2]}')


import re
from rest_framework import serializers
from ..models import Pokemon, PokemonStats

class PokemonStatsSerializer(serializers.ModelSerializer):
  class Meta:
    model = PokemonStats
    fields = ['pokemon_id','hp' ,'attack' ,'defense' ,'sp_attack' ,'sp_defense' ,'speed']

  def create(self, validated_data):
    id = validated_data['id']
    pokemon = Pokemon.objects.get(pk=id)

    pokemonStats = PokemonStats.objects.create(
      id=id,
      hp=validated_data['hp'],
      attack=validated_data['attack'],
      defense=validated_data['defense'],
      sp_attack=validated_data['sp_attack'],
      sp_defense=validated_data['sp_defense'],
      speed=validated_data['speed'],
      pokemon_id=pokemon.id)
    return pokemonStats

class PokemonStatsListSerializer(serializers.ListSerializer):
  child = PokemonStatsSerializer()


"""Added ProcurementInfoComputer, ProcurementInfoCPU, ProcurementInfoRAM, ProcurementInfoWindowsVersion, and ProcurementSurveyComputer model

Revision ID: 35e2030a8253
Revises: abd390311047
Create Date: 2023-02-06 15:19:24.805000

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '35e2030a8253'
down_revision = 'abd390311047'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('procurement_info_cpus',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('cpu', sa.String(), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('procurement_info_rams',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('ram', sa.String(), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('procurement_info_windows_versions',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('windows_version', sa.String(), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('procurement_info_computers',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('mac_address', sa.String(), nullable=True),
    sa.Column('computer_name', sa.String(), nullable=True),
    sa.Column('erp_code_id', sa.Integer(), nullable=True),
    sa.Column('cpu_id', sa.Integer(), nullable=True),
    sa.Column('ram_id', sa.Integer(), nullable=True),
    sa.Column('windows_version_id', sa.Integer(), nullable=True),
    sa.Column('username_id', sa.Integer(), nullable=True),
    sa.ForeignKeyConstraint(['cpu_id'], ['procurement_info_cpus.id'], ),
    sa.ForeignKeyConstraint(['erp_code_id'], ['procurement_details.id'], ),
    sa.ForeignKeyConstraint(['ram_id'], ['procurement_info_rams.id'], ),
    sa.ForeignKeyConstraint(['username_id'], ['staff_account.id'], ),
    sa.ForeignKeyConstraint(['windows_version_id'], ['procurement_info_windows_versions.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    op.create_table('procurement_survey_computers',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('surveyor_id', sa.Integer(), nullable=True),
    sa.Column('satisfaction_with_speed_of_use', sa.String(), nullable=True),
    sa.Column('satisfaction_with_continuous_work', sa.String(), nullable=True),
    sa.Column('satisfaction_with_enough_space', sa.String(), nullable=True),
    sa.Column('personal_info', sa.Boolean(), nullable=True),
    sa.Column('check_back_up_file', sa.String(), nullable=True),
    sa.Column('reason_no_backup_file', sa.String(), nullable=True),
    sa.Column('check_anti_virus_update', sa.String(), nullable=True),
    sa.Column('check_windows_update', sa.String(), nullable=True),
    sa.Column('list_software', sa.String(), nullable=True),
    sa.Column('setting_user_login', sa.String(), nullable=True),
    sa.Column('setting_screen_saver', sa.String(), nullable=True),
    sa.Column('check_ms_office_and_windows_activation', sa.String(), nullable=True),
    sa.Column('requirement', sa.Text(), nullable=True),
    sa.ForeignKeyConstraint(['surveyor_id'], ['staff_account.id'], ),
    sa.PrimaryKeyConstraint('id')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('procurement_survey_computers')
    op.drop_table('procurement_info_computers')
    op.drop_table('procurement_info_windows_versions')
    op.drop_table('procurement_info_rams')
    op.drop_table('procurement_info_cpus')
    # ### end Alembic commands ###


# ECE60146 HW1
# Zhengxin Jiang
# jiang839

class Sequence(object):
    def __init__(self, array):
        self.array = array

    def __iter__(self):
        self.idx = -1
        return self

    def __next__(self):
        self.idx += 1

        if self.idx<len(self.array):
            return self.array[self.idx]
        else:
            raise StopIteration

    def __len__(self):
         return len(self.array)

    def __gt__(self, other):
        # check if two objects have the same length
        if len(self.array) != len(other.array):
            raise ValueError('Two arrays are not equal in length !')

        count = 0
        for i in range(len(self.array)):
            if self.array[i] > other.array[i]:
                count += 1

        return count

# subclass
class Fibonacci(Sequence):

    def __init__(self, first_value, second_value):
        Sequence.__init__(self, [first_value, second_value])

    def __call__(self, length):
        # init
        self.array = self.array[:2]

        if length > 2:
            for i in range(2, length):
                self.array.append(self.array[i-1]+self.array[i-2])

        print(self.array)

# subclass
class Prime(Sequence):

    def __init__(self):
        Sequence.__init__(self, [])

    def __call__(self, length):
        # init
        self.array = []

        if length == 1:
            self.array.append(2)

        if length > 1:
            self.array.append(2)

            num = 3
            while len(self.array)<length:
                is_prime = True

                # test if a number is prime
                for i in range(2, num):
                    if num%i == 0:
                        is_prime = False
                        break

                if is_prime:
                    self.array.append(num)

                num += 1

        print(self.array)

# Main
# Task 1-4
FS = Fibonacci(1,2)
FS(length = 5)
print(len(FS))
print([n for n in FS])

# Task 5
PS = Prime()
PS(length = 8)
print(len(PS))
print([n for n in PS])

# Task 6
FS = Fibonacci(1,2)
FS(length = 8)
PS = Prime()
PS(length = 8)
print(FS>PS)
PS(length = 5)
print(FS>PS)


from Gadgets.console_design import bcolors

def setup_usernames():
    """"Setup the Instagram pages based user Input"""

    # region Static Template

    # profile_user_list = {
    #     # Key = profile name #Value = post count
    # 	"spider.models": {
    # 		"static": 0,
    # 		"updated": 0
    # 	},
    # 	"nike": {
    # 		"static": 0,
    # 		"updated": 0
    # 	},
    # 	"stabilo": {
    # 		"static": 0,
    # 		"updated": 0
    # 	}
    # }
    # # print (profile_user_list["nike"]["updated"])
    # print (profile_user_list["stabilo"]["updated"])
    # print(profile_user_list)

    # endregion Static Template

    user_input = input("Paste page usernames, separate with comma only (,)")\
                 or "spider_modelsx, nike, stabilo"
    user_input = user_input.replace(" ", "")  # Delete space
    page_user_list = user_input.split(",")
    # print("page_user_list " + str(page_user_list))
    print(f"{bcolors.Yellow}"
          f"{len(page_user_list)} users in list"
          f"{bcolors.Normal}")

    # List of strings
    # page_user_list = ["nike", "dainwalker", "_trash_baby"]

    # List of ints
    post_counters = {}
    for item in page_user_list: # Until end of list...
        # print(item)
        post_counters.update({
            item: {
                "static": 0,
                "updated": 0
            },
        }
    )
    # print(post_counters)

    # Create a zip object from two lists
    # full_page = zip(page_user_list, post_counters)
    # profile_user_list = dict(full_page)

    # print(post_counters)
    return post_counters


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

__author__ = 'ipetrash'

import sys
from PyQt5.QtWidgets import QWidget, QPushButton, QApplication, QVBoxLayout, QMessageBox
import subprocess
import os

def log_uncaught_exceptions(ex_cls, ex, tb):
    text = '{}: {}:\n'.format(ex_cls.__name__, ex)
    import traceback
    text += ''.join(traceback.format_tb(tb))

    print(text)
    QMessageBox.critical(None, 'Error', text)
    quit()

sys.excepthook = log_uncaught_exceptions

class MainWindow(QWidget):
    def __init__(self):
        super().__init__()

        self.cmd_command = "start cmd.exe @cmd /k ipconfig"

        button_run_subprocess = QPushButton('Run subprocess')
        button_run_subprocess.clicked.connect(self.btn_clicked_subprocess)

        button_os_system = QPushButton('Run os.system')
        button_os_system.clicked.connect(self.btn_clicked_subprocess)

        layout = QVBoxLayout()
        layout.addWidget(button_run_subprocess)
        layout.addWidget(button_os_system)

        self.setLayout(layout)

    def btn_clicked_subprocess(self):
        subprocess.run(self.cmd_command.split(), shell=True)

    def btn_clicked_os_system(self):
        os.system(self.cmd_command)

if __name__ == '__main__':
    app = QApplication(sys.argv)

    mw = MainWindow()
    mw.show()

    sys.exit(app.exec_())