import sys

import numpy as np
from scipy.special import softmax

sys.path.append("..")
from headers.utils import *
from math import log2
from math import sqrt
import pandas as pd
from src.Node import Node

class NCMTree:
    def __init__(self, max_depth=10, min_samples_split=2,
                 min_samples_leaf=1, random_state=None, debug=False, distance="euclidean", root_distance="mahalanobis",
                 method_subclasses="sqrt", method_split="alea", method_max_features="sqrt", root_class=None, alpha=0.95, mode_mean=None, mode_cov=None, mode_weight=None, nbgenlayer=0):
        """

        :param max_depth:
        :param min_samples_split:
        :param min_samples_leaf:
        :param random_state:
        :param debug:
        :param distance:
        :param method_subclasses:
        :param method_split:
        :param method_max_features:
        :param root_class:
        :param alpha:
        :param nbgenlayer:
        """
        self.method_max_features = method_max_features
        self.max_features = None
        self.max_depth = max_depth
        self.min_samples_split = min_samples_split
        self.min_samples_leaf = min_samples_leaf
        self.random_state = random_state
        self.root = None
        self.depth = 1
        self.debug = debug
        self.cardinality = 0
        self.distance = distance
        self.root_distance = root_distance
        self.method_subclasses = method_subclasses
        self.method_split = method_split
        self.root_class = root_class
        self.alpha=alpha
        self.nbgenlayer = nbgenlayer

        if debug:
            print('=== Initialisation NCMTree ===')
            print('\t max_features: {}'.format(self.method_max_features))
            print('\t max_depth: {}'.format(self.max_depth))
            print('\t min_samples_split: {}'.format(self.min_samples_split))
            print('\t min_samples_leaf: {}'.format(self.min_samples_leaf))
            print('\t random_state: {}'.format(self.random_state))
            print('====================================')

    def fit(self, X, y, X_nobag, y_nobag):
        """
        :param X:
        :param y:
        :return:
        """
        # number of features selected
        if self.method_max_features == 'log2':
            self.max_features = int(log2(X.shape[1]))
            assert X.shape[1] != 0, 'Null dimension'
            assert self.max_features != 0, 'self.max_features = 0'

        elif self.method_max_features == 'sqrt':
            self.max_features = int(sqrt(X.shape[1]))
            assert X.shape[1] != 0, 'Null dimension'
            assert self.max_features != 0, 'self.max_features = 0'

        elif type(self.method_max_features) == float:
            self.max_features = int(self.method_max_features * (X.shape[1]))
            assert X.shape[1] != 0, 'Null dimension'
            assert self.max_features != 0, 'self.max_features = 0'

        elif type(self.method_max_features) == int:
            self.max_features = self.method_max_features
            assert X.shape[1] != 0, 'Null dimension'
            assert self.max_features != 0, 'self.max_features = 0'

        self.root = self.build_nodes(X, y, X_nobag, y_nobag, None, 0)

    def build_nodes(self, X, y, X_nobag, y_nobag, parent, localdepth=0):
        """
        recursive function for growing tree
        :param localdepth:
        :param X:
        :param y:
        :param parent:
        :return:
        """
        # If it's a generative layer, create a generative node
        if localdepth < self.nbgenlayer:
            current_node = Node(parent, False, self.min_samples_leaf, self.max_features, self.distance, self.root_distance, self.method_subclasses, 'generative', root_class=self.root_class, alpha=self.alpha, localdepth=localdepth)
        else:
            current_node = Node(parent, False, self.min_samples_leaf, self.max_features, self.distance, self.method_subclasses,
                            self.method_split, localdepth=localdepth)
        self.cardinality += 1
        if(current_node.method_split == "generative"): #if generative layer, use unbagged X and y, to improve perf
            current_node.fit(X_nobag, y_nobag)
        else:
            current_node.fit(X, y)
        split_possible, left_index, right_index = current_node.predict_split(X)

        # ------------stopping criterion (and in Node.py, fct() predict_split )------------
        if split_possible and localdepth < self.max_depth and len(X) > self.min_samples_split:
            if self.depth < localdepth:
                self.depth = localdepth
            # LEFT NODE
            left_child = self.build_nodes(X[left_index], y[left_index], X_nobag[left_index], y_nobag[left_index], current_node, localdepth+1)
            current_node.set_left_child(left_child)
            # RIGHT NODE
            right_child = self.build_nodes(X[right_index], y[right_index], X_nobag[right_index], y_nobag[right_index], current_node, localdepth+1)
            current_node.set_right_child(right_child)
        else:
            current_node.set_leaf(True)
        return current_node

    def predict(self, X):
        """

        :param X:
        :param proba:
        :return:
        """
        return self.root.predict_all(X)


from django.urls import path
from . import views

urlpatterns = [
    path('', views.get_posts, name='get_posts'),
    path('<int:pk>/', views.post_detail, name='post_detail'),
    path('add/', views.create_or_edit_post, name='add_post'),
    path('edit/<int:pk>/', views.create_or_edit_post, name='edit_post'),
    path('delete/<int:pk>/', views.delete_post, name='delete_post'),
    path('delete-comment/<int:comment_id>/', views.delete_comment,
         name='delete_comment'),
]


def kthInt(numlist,k):
    numset=set(numlist)
    if(len(numset)<k):
        return "-1"
    else:
        i=k
        for num in numlist:
            if numlist.count(num)==1:
                if num in numset and i>=2:
                    numset.remove(num)
                    i=i-1
                elif num in numset and i==1:
                    return num
        return "-1"
t=int(input())
for i in range(t):
    n,k=map(int,input().split(" "))
    numlist=list(map(int,input().split(" ")))
    print(kthInt(numlist,k))


#9-10
from restaurant_hw import Restaurant

new = Restaurant('mike','oll day')
new.describe_restaurant()
new.open_restaurant()

#9-11
'''
from user_hw import Admin

admin_text = Admin("joe","mary")
admin_text.privilege.show_pricileges()
'''
#or

#9-12
import admin_hw

new_admin = admin_hw.Admin("joe","mary")
new_admin.privilege.show_pricileges()


import sys

input_ = sys.stdin.readline

t = int(input_())
to_print = []

def get_press_count(products, dp):
    first_left, first_right = products[0]
    dp[0][0], dp[0][1] = first_right + first_right-first_left, first_right
    for i in range(1, len(products)):
        prev_left, prev_right = products[i-1]
        left, right = products[i]
        dist = right-left
        dp[i][0] = min(dp[i-1][0]+abs(prev_left-right), dp[i-1][1]+abs(prev_right-right))+dist
        dp[i][1] = min(dp[i-1][0]+abs(prev_left-left), dp[i-1][1]+abs(prev_right-left))+dist
    return min(dp[len(products)-1])

for i in range(t):
    n, _ = map(int, input_().split())
    dp = [[0, 0] for _ in range(n)]
    products = []
    for _ in range(n):
        cur = list(map(int, input_().split()))
        products.append((min(cur), max(cur)))
    to_print.append(f'Case #{i+1}: {get_press_count(products, dp)}')

print('\n'.join(to_print))


import numpy as np
import cv2
import argparse
from functions import *

# USAGE: python road.py -i imagefilename.jpg

# NOTE: All kernel sizes are adjustable dependent to the results of testing
# but this default sizes are the ones we recommend during our testin

# Getting the image filename on the commandline
ap = argparse.ArgumentParser()
# Add the arguments to the parser
ap.add_argument("-i","--image", required=True,
   help="image filename", type=check_input)
args = vars(ap.parse_args())
path = args['image'] # filename of image
img_orig = cv2.imread(path)
#convert image to 1280, 720
img_orig = cv2.resize(img_orig, (1280,720))

# img=cv2.ximgproc.anisotropicDiffusion(img_orig, 0.25, 1,1)

#image pre-processing, converting to grayscale, histogram and applying smoothing using gaussian and erosion
img = cv2.cvtColor(img_orig, cv2.COLOR_BGR2GRAY)

#pre-process
img, v, kernel, min_value = pre_process(img, img_orig)

#performing canny edge detection
sigma = 0.33
lower = int(max(0, (1.0 - sigma) * v))
upper = lower * 3
edges = cv2.Canny(img,lower,upper,3) # default 75, 200

#thresholdings
final_contours = thresholding(edges, img_orig, kernel)
#pixel elimination
mask_contours = gray_pixel_elimination(final_contours, min_value, img_orig)
#merging and post process
bg, roi_road = post_process(img_orig, mask_contours)

# blending the red area road surface to the original image
alpha = 0.25
road_surface = cv2.addWeighted(img_orig, 1-alpha, bg, alpha, 0)

winname = "Final road surface"
cv2.imshow(winname, road_surface)
cv2.waitKey()
cv2.destroyAllWindows()


import csv
from dataclasses import dataclass
from typing import Union, Dict, List, Any
import pkgutil
from pathlib import PurePath, Path
from functools import lru_cache
import logging
import yaml
from linkml_runtime.linkml_model import SlotDefinition, ClassDefinition, SchemaDefinition, \
    PermissibleValue, EnumDefinition, TypeDefinition, SubsetDefinition, Prefix
from linkml_runtime.utils.schemaview import SchemaView

from schemasheets.conf.configschema import ColumnSettings, Shortcuts

COL_NAME = str
DESCRIPTOR = str
ROW = Dict[str, Any]

#c = ClassDefinition
T_SCHEMA = 'schema'
T_CLASS = 'class'
T_SLOT = 'slot'
T_ENUM = 'enum'
T_PV = 'permissible_value'
T_TYPE = 'type'
T_SUBSET = 'subset'
T_PREFIX = 'prefix'

tmap = {
    T_SCHEMA: SchemaDefinition,
    T_CLASS: ClassDefinition,
    T_SLOT: SlotDefinition,
    T_ENUM: EnumDefinition,
    T_PV: PermissibleValue,
    T_TYPE: TypeDefinition,
    T_SUBSET: SubsetDefinition,
    T_PREFIX: Prefix
}

@dataclass
class ColumnConfig:
    """
    Configuration for a single column in a schema sheet
    """
    name: COL_NAME
    maps_to: DESCRIPTOR = None
    settings: ColumnSettings = None
    metaslot: SlotDefinition = None
    inner_key_metaslot: SlotDefinition = None
    is_element_type: bool = None

    def merge_settings(self, settings: ColumnSettings) -> None:
        """
        merges specified settings into current settings

        :param settings: settings to be merged
        """
        for k, v in vars(settings).items():
            if v:
                setattr(self.settings, k, v)

    def add_info(self, info: Union[Dict, DESCRIPTOR]) -> None:
        """
        Adds configuration/settings in the form of a dict object.

        Information can be incrementally added:

        - the first piece of information should be the descriptor
        - after that individual settings can be added

        :param info: configuration
        :return:
        """
        logging.debug(f"Adding info to {self.name}: {info}")
        if self.maps_to is None:
            # The first descriptor row describes what the column maps to
            self.settings = ColumnSettings()
            if isinstance(info, dict):
                items = list(info.items())
                if len(items) != 1:
                    raise ValueError(f'Unexpected settings: {info}')
                else:
                    item = items[0]
                    self.maps_to = item[0]
                    if isinstance(item[1], dict):
                        settings = ColumnSettings(**item[1])
                    else:
                        raise ValueError(f'Expected dict after first element of {items}')
                    self.merge_settings(settings)
            else:
                self.maps_to = info
            mm = get_metamodel()
            snmap = mm.slot_name_mappings()
            # TODO: use alias
            snmap['uri'] = snmap['type_uri']
            if self.maps_to.startswith("metaslot."):
                maps_to = self.maps_to.replace("metaslot.", "")
                self.metaslot = snmap[maps_to]
                self.maps_to = maps_to
            elif self.maps_to in snmap and self.maps_to != 'type':
                self.metaslot = snmap[self.maps_to]
            else:
                if self.maps_to not in tmap and self.maps_to not in Shortcuts:
                    raise ValueError(f'Cannot interpret: {self.maps_to}')
        else:
            settings = ColumnSettings(**info)
            self.merge_settings(settings)
            if settings.inner_key:
                snmap = get_metamodel().slot_name_mappings()
                if settings.inner_key in snmap:
                    self.inner_key_metaslot = snmap[settings.inner_key]

@dataclass
class TableConfig:
    """
    Configuration for an entire table / schema sheet

    """
    name: str = None
    """table name"""

    columns: Dict[COL_NAME, ColumnConfig] = None
    """maps column names to config"""

    column_by_element_type: Dict[str, COL_NAME] = None
    """maps element types (schema, class, ...) to the name of the column that represents them"""

    metatype_column: COL_NAME = None
    """Column that represents the metatype designator"""

    name_column: COL_NAME = None
    """Column that represents that name of the entity"""

    def add_info(self, col: COL_NAME, info: Union[Dict, DESCRIPTOR]) -> None:
        """
        Wrapper for :ref:`ColumnConfig.add_info`

        :param col:
        :param info:
        """
        if col not in self.columns:
            self.columns[col] = ColumnConfig(col)
        #print(f'ADDING: {col}')
        self.columns[col].add_info(info)
        if self.columns[col].maps_to == 'metatype':
            if self.metatype_column and self.metatype_column != col:
                raise ValueError(f'Multiple metatype columns not allowed: {self.metatype_column}, {col}')
            self.metatype_column = col
        if self.columns[col].maps_to == 'name':
            if self.name_column:
                raise ValueError(f'Multiple name columns not allowed: {self.name_column}, {col}')
            self.name_column = col
        if self.column_by_element_type is None:
            self.column_by_element_type = {}
        for c, cc in self.columns.items():
            if cc.maps_to in tmap:
                self.column_by_element_type[cc.maps_to] = c
                cc.is_element_type = True

@dataclass
class SchemaSheet:
    """
    A SchemaSheet consists of:

    - a collection of rows, each row representing a schema element
    - a TableConfiguration
    """
    table_config: TableConfig
    rows: List[ROW]
    start_line_number: int
    table_config_rows: List[ROW] = None

    @classmethod
    def from_csv(cls, path: str, delimiter='\t'):
        with open(path, newline='') as tsv_file:
            reader = csv.DictReader(tsv_file, delimiter=delimiter)
            return cls.from_dictreader(reader)

    @staticmethod
    def from_dictreader(reader: csv.DictReader) -> "SchemaSheet":
        """
        Reads a schemasheets TSV file parsing only header info

        :param reader:
        :return:
        """
        table_config = TableConfig(columns={})
        rows = []
        line_num = 1
        table_config_rows = []
        for row in reader:
            logging.debug(f"ROW: {row}")
            # google sheets
            if "" in row:
                del row[""]
            k0 = list(row.keys())[0]
            if row[k0].startswith('>'):
                table_config_rows.append(row)
                line_num += 1
                for k, v in row.items():
                    if v is not None and v.startswith('>'):
                        v = v.replace('>', '')
                    if v:
                        meta_obj = yaml.safe_load(v)
                        table_config.add_info(k, meta_obj)
                    else:
                        if line_num == 2:
                            # TODO: consider auto-interpreting
                            raise ValueError(f'Enter an interpretation for {k}')
                        logging.debug(f'Empty val for {k} in line {line_num}')
            else:
                rows.append(row)
        return SchemaSheet(table_config=table_config,
                           table_config_rows=table_config_rows,
                           rows=rows,
                           start_line_number=line_num)

    def load_table_config(self, config: Union[dict, str, Path]) -> None:
        """
        Loads a table configuration from a file or dict

        :param config:
        :return:
        """
        if not isinstance(config, dict):
            with open(config) as f:
                config = yaml.safe_load(f)
                return self.load_table_config(config)
        for k, v in config.items():
            if isinstance(v, list):
                for v1 in v:
                    self.table_config.add_info(k, v1)
            else:
                self.table_config.add_info(k, v)

@lru_cache()
def get_metamodel() -> SchemaView:
    """
    Returns the LinkML schema metamodel as a SchemaView object

    this can be retired when https://github.com/linkml/linkml-runtime/pull/100/
    is in major release
    :return:
    """
    package = 'linkml_runtime.linkml_model.meta'
    full_path = PurePath('model') / 'schema'
    data = pkgutil.get_data(package, f'{full_path}/meta.yaml')
    return SchemaView(data.decode("utf-8"))

@lru_cache()
def get_configmodel() -> SchemaView:
    """
    Returns the Config schema metamodel as a SchemaView object

    this can be retired when https://github.com/linkml/linkml-runtime/pull/100/
    is in major release
    :return:
    """
    package = 'schemasheets.conf.configschema'
    data = pkgutil.get_data(package, f'configschema.yaml')
    return SchemaView(data.decode("utf-8"))


import pickle
from ppxgboost import BoosterParser, PaillierAPI, PPBooster
from ppxgboost.PPKey import PPBoostKey
from joblib import Parallel, delayed
import itertools
from secrets import token_bytes
from ope.pyope.ope import OPE
import pandas as pd
import numpy as np
import xgboost as xgb
from datetime import datetime
from timerit import Timerit

from train_utils import load_dataset, train_test_split_undersample, convert_to_binary, show_time_taken, get_pickled_size

def chunk_arr(arr, num_chunks):
    chunk_size = len(arr) // num_chunks
    return [(arr[i:i+chunk_size]) for i in range(0,len(arr),chunk_size)]

def dechunk_arr(chunks):
    return list(itertools.chain.from_iterable(chunks))

def encrypt_model(model, min_max, num_cores):
    # load and parse model
    print('Encrypting model...')
    start_time = datetime.now()
    encrypted_model, _, _ = BoosterParser.model_to_trees(model, min_max)

    """
    Set up encryption materials:
        column_hash_key: for hashing the columns of the data
        paillier_public_key: for encrypting the leaf node values of the model trees
        paillier_private_key: for decrypting the output of the model
        order_preserving_key: for encryption/decryption of model inputs,
                              and comparison nodes in model trees
    """
    column_hash_key = token_bytes(16) # for hashing the columns of the data
    paillier_public_key, paillier_private_key = PaillierAPI.he_key_gen() #
    order_preserving_key = OPE(token_bytes(16))
    tree_encryption_keys = PPBoostKey(paillier_public_key, column_hash_key, order_preserving_key)

    output_chunks = Parallel(n_jobs=num_cores)(
        delayed(PPBooster.enc_xgboost_model)(
            tree_encryption_keys,
            chunk,
            PPBooster.MetaData(min_max)
        )
        for chunk in chunk_arr(encrypted_model, num_cores)
    )
    encrypted_model = dechunk_arr(output_chunks)

    show_time_taken('Time taken to encrypt model:', start_time, datetime.now())
    return encrypted_model, (column_hash_key, order_preserving_key, paillier_private_key, min_max)

def test_encrypted_model(plaintext_model, encrypted_model, keys, xtest, ytest, num_cores):
    print('\nTesting encrypted model...')
    start_time = datetime.now()
    start_index = ytest.index[0]
    test_fraud_idxs = np.array(ytest[ytest==1].index)
    test_fraud_idxs = np.random.choice(test_fraud_idxs, 75, replace = False)
    test_normal_idxs = np.array(ytest[ytest==0].index)
    test_normal_idxs = np.random.choice(test_normal_idxs, 75, replace = False)
    test_idxs = np.concatenate([test_fraud_idxs, test_normal_idxs]) - start_index
    xtest = xtest.iloc[test_idxs, :]
    ytest = ytest.iloc[test_idxs]

    print('Making plaintext predictions.')
    preds_plaintext = convert_to_binary(plaintext_model.predict(xgb.DMatrix(xtest)))
    print('Plaintext predictions:', preds_plaintext)

    (column_hash_key, order_preserving_key, paillier_private_key, min_max) = keys
    encrypted_xtest = xtest.copy()
    print('Encrypting inputs.')
    encrypted_xtest = Parallel(n_jobs=num_cores)(
        delayed(PPBooster.enc_input_vector)(
            column_hash_key,
            order_preserving_key,
            xtest.columns,
            chunk,
            PPBooster.MetaData(min_max)
        )
        for chunk in chunk_arr(encrypted_xtest, num_cores)
    )
    encrypted_xtest = pd.concat(encrypted_xtest)

    print('Making encrypted predictions.')
    preds_encrypted = PPBooster.predict_binary(encrypted_model, encrypted_xtest)
    print('Decrypting results.')
    preds_decrypted = []
    for pred in preds_encrypted:
        preds_decrypted.append(PaillierAPI.decrypt(paillier_private_key, pred))
    preds_decrypted = convert_to_binary(np.array(preds_decrypted))
    print('Decrypted predictions:', preds_decrypted)

    result = np.array_equal(preds_plaintext, preds_decrypted)
    print('Success!') if result else print('Failed.')
    show_time_taken('Time taken for testing:', start_time, datetime.now())

    print('\nLatency Testing:')
    for _ in Timerit(num=100, label='Transaction Encryption', verbose=1):
        encrypted_xtest = xtest[:1].copy() # copying time was tested to be negligible
        PPBooster.enc_input_vector(column_hash_key, order_preserving_key, xtest.columns, encrypted_xtest, PPBooster.MetaData(min_max))

    for _ in Timerit(num=100, label='Plaintext Inference', verbose=1):
        convert_to_binary(plaintext_model.predict(xgb.DMatrix(xtest[:1])))

    for _ in Timerit(num=100, label='Encrypted Inference', verbose=1):
        PPBooster.predict_binary(encrypted_model, encrypted_xtest[:1])

    print('\nStorage size testing:')
    get_pickled_size(xtest[:1], 'Plaintext Transaction')
    get_pickled_size(encrypted_xtest, 'Encrypted Transaction')
    get_pickled_size(plaintext_model, 'Plaintext Model')
    get_pickled_size(encrypted_model, 'Encrypted Model')

if __name__ == '__main__':
    model_filename = dataset_name+'-xgboost.pt'
    model = pickle.load(open(model_filename, "rb"))
    # encrypted_model, keys = encrypt_model(model, 8)
    # pickle.dump(encrypted_model, open('encrypted-'+model_filename, "wb" ))
    # pickle.dump(keys, open('keys-'+model_filename, "wb" ))

    encrypted_model = pickle.load(open('encrypted-'+model_filename, "rb"))
    keys = pickle.load(open('keys-'+model_filename, "rb"))

    x, y = load_dataset('ulb')
    _, xtest, _, ytest = train_test_split_undersample(x, y, 0, test_size=0.25)

    result = test_encrypted_model(
        model,
        encrypted_model,
        keys,
        xtest,
        ytest,
        8
    )
    print(result)


import sys
import time
from random import randint

import matplotlib.pyplot as plt

from HeapSort import heap_sort
from MergeSortInsertion import merge_sort_insertion
from QuickSort import quick_sort

sys.setrecursionlimit(17500)

x, ymerge_insertion, yquick,yheap = [], [], [], []
i = 1
k = 0
while i <= 100000000:
    i *= 2
    k += 1
    arr1 = [randint(-i, i) for _ in range(i)]
    #arr = [i-j for j in range(i)]
    arr2 = arr1.copy()
    arr3 = arr1.copy()
    assert (arr1.__eq__(arr2))
    assert (arr1.__eq__(arr3))

    x.append(i)

    print("Iteração ", k, ":")
    print("i = ", i)
    a = time.perf_counter()
    heap_sort(arr3)
    b = time.perf_counter()
    yheap.append(b - a)

    print('H: ', (b - a))

    a = time.perf_counter()
    merge_sort_insertion(arr1, 10)
    b = time.perf_counter()
    ymerge_insertion.append(b-a)
    print('M: ', (b-a))

    a = time.perf_counter()
    quick_sort(arr2, 10)
    b = time.perf_counter()
    yquick.append(b - a)
    print('Q: ', (b-a))

assert (arr2.__eq__(arr1))
assert (arr3.__eq__(arr1))

plt.plot(x, yquick, label="quicksort method")
plt.plot(x, yheap, label="heap sort")
plt.plot(x, ymerge_insertion, label="mergesort with insertion on N=7")

plt.xlabel('tamanho da array')
plt.ylabel('tempo')
plt.title('sorts')
plt.legend()
plt.show()


from google.cloud import language_v1
from google.cloud import translate_v2 as translate
import requests
import docx2txt
from pptx import Presentation
from firebase_admin import firestore

class FullProcessor:

    '''
    Full processing of company description and files.

    1) Read company from Firestore DB
    2) Parse Files (PDFs, PPTs, DOCs)
    3) Call Google NLP API
    4) Generate Keywords
    5) Write keywords to Firestore DB

    '''

    def call_ocr(self, file_uri, dest_uri):
        # Single PDF OCR

        #TODO: Insert Google OCR script here with 50 page max limit

        return "hi"

    def parse_files(self, database, fs, filename, file_uri):

        pdfbox_endpoints = {
            'main':'https://us-central1-coopsightsoftware.cloudfunctions.net/backend-pdfbox',
            'staging':'https://us-central1-coopsightstaging.cloudfunctions.net/backend-pdfbox-staging',
            'vc': 'https://us-central1-coopsight-vc.cloudfunctions.net/backend-pdfbox-vc',
            'general': 'https://us-central1-coopsight-general.cloudfunctions.net/backend-pdfbox-general',
            'open': 'https://europe-west3-coopsight-open-network.cloudfunctions.net/backend-pdfbox-open'
        }

        if filename.lower().endswith(('.doc', '.docx')):
            temp_uri = file_uri.split("gs://",1)[1]
            read = fs.open(temp_uri)
            clean_text = docx2txt.process(read)

            return clean_text, True

        elif filename.lower().endswith(('.ppt', '.pptx')):
            temp_uri = file_uri.split("gs://",1)[1]
            read = fs.open(temp_uri)
            prs = Presentation(read)
            slide_number = 1
            clean_text = ""
            for slide in prs.slides:
                for shape in slide.shapes:
                    if hasattr(shape, "text"):
                        clean_text += shape.text + "\n"
                slide_number += 1

            return clean_text, True

        elif filename.lower().endswith('.pdf'):
            # print(filename)
            endpoint = pdfbox_endpoints[database]
            temp_uri = file_uri.split("appspot.com/",1)[1]
            pdfbox_params = {'uri':temp_uri}
            pdfbox_response = requests.post(endpoint, json=pdfbox_params)
            # print(pdfbox_response.text)

            # TODO: ADD OCR Integration code here. Call call_ocr function above.

            return pdfbox_response.text, True

        else:
            return '', False

    def call_nlp(self, nlp_client, test_string, desc_string, web_string):

        # Call desc_string first
        test_string = test_string.replace('\n',' ')
        word_dict = {}
        type_ = language_v1.Document.Type.PLAIN_TEXT
        language = "en"
        encoding_type = language_v1.EncodingType.UTF8

        #* Call NLP API on company description
        document = {"content": desc_string, "type_": type_, "language": language}
        nlp_response = nlp_client.analyze_entities(request = {'document': document, 'encoding_type': encoding_type})
        for entity in nlp_response.entities:
                word_dict[entity.name] = entity.salience

        #* Call NLP API on web string
        if len(web_string) > 150:
            document = {"content": web_string, "type_": type_, "language": language}
            nlp_response = nlp_client.analyze_entities(request = {'document': document, 'encoding_type': encoding_type})
            for entity in nlp_response.entities:
                    word_dict[entity.name] = entity.salience

        #* Set string upper length for test_string
        print(len(test_string))
        divisor = 100000
        count = 0
        for i in range(int(len(test_string)/divisor) + 1):
            # print("call " + str(i))
            temp_string = test_string[i*divisor:i*divisor+divisor]
            # print(len(temp_string))

            document = {"content": temp_string, "type_": type_, "language": language}

            nlp_response = nlp_client.analyze_entities(request = {'document': document, 'encoding_type': encoding_type})

            for entity in nlp_response.entities:
                word_dict[entity.name] = entity.salience

            count += 1
            if count > 1:
                break

        return word_dict

    def generate_keywords(self, nlp_response):

        # word_dict = {}
        # for entity in nlp_response.entities:
        #     word_dict[entity.name] = entity.salience

        sorted_words = sorted(nlp_response.items(), key=lambda x: x[1], reverse=True)

        # top 20 and 100
        topk = {}
        topn = {}
        counter = 0
        for w in sorted_words:
            if w[1] > 0:
                if counter < 20:
                    topk[w[0]] = w[1]
                if counter < 100:
                    topn[w[0]] = w[1]
                counter += 1

        return topk, topn

    def translate(self, translate_client, website_string):
        result = translate_client.detect_language(website_string[0:2000])
        if result['language'] == 'en':
            website_string = str(website_string.encode('ascii', errors='ignore').decode('utf-8'))
            return website_string
        else:
            print(f'translating website from {result["language"]}')
            result = translate_client.translate(website_string[0:2000], target_language='en')
            print(result["translatedText"])
            website_string = str(result["translatedText"].encode('ascii', errors='ignore').decode('utf-8'))
            return website_string

    def execute(self, database, firebase_db, nlp_client, translate_client, fs, docid, path, label=None, parent=None, pdfs=None, industry=None, website=None, WS=None):

        #* FETCH COMPANY DESCRIPTION TEXT AT PATH
        #!!! THIS IS CURCIAL <CONTAINTS DEPRECATED CONTENT>
        if path[0] == '/':
            path = path[1:]
        desc_ref = firebase_db.document(path)
        desc_doc = desc_ref.get()

        if desc_doc.exists:
            desc_string = desc_doc.to_dict()['description']
        else:
            desc_string = ''

        #* RETRIEVE LIST OF FILES
        #!!! DIDFF IMPLEMENTATION FOR DIFF DATABASE
        if database == 'staging' or database == 'open':
            files_ref = firebase_db.collection('privateBins')
            query_ref = files_ref.where('docid', '==', docid).stream()
        else:
            files_ref = firebase_db.collection('files')
            query_ref = files_ref.where('docid', '==', docid).stream()

        count = 0
        for doc in query_ref:
            query_doc = doc.to_dict()
            count += 1
            break

        #* PARSE LIST OF FILES
        parsed_list = []
        if count > 0 and pdfs is None:
            #!!!! THIS IS CURCIAL <CONTAINTS DEPRECATED CONTENT>
            if database == 'general':
                for item in query_doc['filesList']:
                    word_string, status = self.parse_files(database, fs, item['fileName'], item['fileURI'])

                    if status == True:
                        parsed_list.append(word_string)
                    else:
                        print("failed to parse " + item['fileName'])
            elif database == 'staging' or database == 'open':
                for item in query_doc['filesList']:
                    word_string, status = self.parse_files(database, fs, item['name'], item['fileURI'])
                    if status == True:
                        parsed_list.append(word_string)
                    else:
                        print("failed to parse " + item['name'])

        #* PARSE LIST OF FILES FROM INPUT PARAM
        if pdfs is not None:
            for pdf in pdfs:
                word_string, status = self.parse_files(database, fs, '.pdf', pdf)
                if status == True:
                    parsed_list.append(word_string)
                else:
                    print("failed to parse " + '.pdf')

        #* INSERT STRINGS FROM WEB SCRAPING
        web_string = ''
        if website is not None and WS is not None:
            web_string = WS.scrape(website, False)

        #* TEST DETECTION OF LANGUAGE
        web_string = self.translate(translate_client, web_string)

        #* CALL NLP
        test_string = ' '.join(parsed_list)
        nlp_response = self.call_nlp(nlp_client, test_string, desc_string, web_string)

        #* GENERATE KEYWORDS
        topk, topn = self.generate_keywords(nlp_response)

        #* SAVE KEYWORDS TO FIREBASE

        #!!!! THIS IS CURCIAL <CONTAINTS DEPRECATED CONTENT>
        if database == 'general':

            if path[0] == '/':
                path = path[1:]
            first_slash = path.find('/')
            ecoid = path[first_slash+1:path.find('/', first_slash+1)]

            files_ref = firebase_db.collection(u'ecosystemMatching/' + ecoid + '/keywords')
            query_ref = files_ref.where(u'docid', u'==', docid).limit(1).get()

            new_doc = {
                    u'topk': topk,
                    u'topn': topn,
                    u'docid': docid
                }

            count = 0
            for doc in query_ref:
                query_doc = doc.to_dict()
                doc.reference.update(new_doc)
                count += 1
                break

            if count == 0:
                files_ref.add(new_doc)

        #!!!! THIS IS CURCIAL <CONTAINTS DEPRECATED CONTENT>
        elif database == 'staging' or database == 'open':

            #* ADD KEYWORDS TO KEYWORDS COLLECTION
            files_ref = firebase_db.collection(u'keywords')
            query_ref = files_ref.where(u'docid', u'==', docid).limit(1).get()

            new_doc = {
                    u'topk': topk,
                    u'topn': topn,
                    u'docid': docid,
                    u'label': label,
                    u'parent': parent,
                    u'timestamp': firestore.SERVER_TIMESTAMP
                }

            # If there is an existing document then update
            count = 0
            for doc in query_ref:
                query_doc = doc.to_dict()
                doc.reference.update(new_doc)
                count += 1
                break

            # If there is no existing document then add
            if count == 0:
                firebase_db.collection(u'keywords').add(new_doc)

            #* ADD INDUSTRY TO INDUSTRY
            if industry is not None:
                files_ref = firebase_db.collection(u'industries')
                query_ref = files_ref.where(u'docid', u'==', docid).limit(1).get()
                new_doc = {
                        u'docid': docid,
                        u'industry': industry,
                        u'timestamp': firestore.SERVER_TIMESTAMP
                    }
                # If there is an existing document then update
                count = 0
                for doc in query_ref:
                    query_doc = doc.to_dict()
                    doc.reference.update(new_doc)
                    count += 1
                    break
                # If there is no existing document then add
                if count == 0:
                    firebase_db.collection(u'industries').add(new_doc)

        # print(topk)
        # print(topn)

        return "success", 200

# param = {'database':'general','docid':'Vn8DIhG9iVMckSbCDlUW','path':'/ecosystems/s2GDDevZc69Dfg8mOdOj/699c0550-dd7b-11ea-9705-1dd4eb3a9ec8/Vn8DIhG9iVMckSbCDlUW'}

# x = requests.post('http://localhost:8081/full_processing',json=param)
# x = requests.post('http://35.192.131.197/full_processing',json=param)


import unittest
import time
from collections import defaultdict

def urlify_pythonic(text: str, length: int) -> str:
    return text[:length].replace(" ", "%20")

def urlify_algo(text: str, length: int) -> str:
    """
    Replaces the white spaces with "%20" and removes trailing spaces.
    """
    cropped_txt = text[:length]
    replaced_txt = str()
    for char in cropped_txt:
        if char == " ":
            replaced_txt += "%20"
        else:
            replaced_txt += char
    return replaced_txt

class Test_urlify(unittest.TestCase):
    test_cases = {
        ("much ado about nothing      ", 22): "much%20ado%20about%20nothing",
        ("Mr John Smith       ", 13): "Mr%20John%20Smith",
        (" a b    ", 4): "%20a%20b",
        (" a b       ", 5): "%20a%20b%20",
        (" a b ", 4): "%20a%20b"
    }

    test_functions = [
        urlify_pythonic,
        urlify_algo
    ]

    def test_urlify(self):
        number_runs = 1000
        runtimes = defaultdict(float)
        for _ in range(number_runs):
            for fctn in self.test_functions:
                for args, output in self.test_cases.items():
                    start = time.perf_counter()
                    res = fctn(*args)
                    self.assertEqual(res, output)
                    runtimes[fctn.__name__] += (time.perf_counter() - start) * 1000

        print(f"{number_runs} runs")
        for fctn_name, run_time in runtimes.items():
            print(f"{fctn_name}: {run_time: .1f}ms")

if __name__ == '__main__':
    unittest.main()


'''
Assignment
We represent a serial number either as a strictly positive natural number (int) or
as a string (str) that exists of one or more digits that are not all equal to zero.
The number of digits in a serial number is not fixed, but there's always at least one digit.
In converting a serial number into its string representation (str),
leading zeros are added until the string representation has at least eight digits.

Your task:

- Write a function serial_number that takes a serial number.
The function must return the string representation (str) of the given serial number,
where leading zeros may have been added such that the string representation has at least eight digits.

- Write a function solid that takes a serial number.
The function must return a Boolean value (bool) that indicates whether the string representation of the given serial number is a solid.

- Write a function radar that takes a serial number.
The function must return a Boolean value (bool) that indicates whether the string representation of the given serial number is a radar.
By definition, solids are never a radar.

- Write a function repeater that takes a serial number.
The function must return a Boolean value (bool) that indicates whether the string representation of the given serial number is a repeater.
By definition, solids are never a repeater.

- Write a function radar_repeater that takes a serial number.
The function must return a Boolean value (bool) that indicates whether the string representation of the given serial number is a radar repeater.

- Write a function numismatist that takes a sequence (list or tuple) of serial numbers.
The function must return a new list containing the serial numbers from the given sequence that belong to some kind.
By default we are looking for solids, but the function numismatist also has a second optional parameter kind.
This parameter may take a function that itself takes a serial number as an argument,
and returns a Boolean value (bool) that indicates whether the given serial number belongs to the kind we are looking for.

All these function must raise an AssertionError with the message invalid serial number if an argument is passed that is not a valid serial number.
'''

def serialNumber(number):
    number = str(number)
    assert number.isdigit(), 'invalid serial number'
    if '0' in set(number):
        assert len(set(number)) != 1, 'invalid serial number'
    new_string = ''
    Len = int(len(number)) #length of number
    if Len >= 8:
        return number
    if Len <= 8:
        new_string += '0'*(8-Len)
        new_string += number
        return new_string

def solid(number):
    number = serialNumber(number)
    return  len(set(number)) == 1

def radar(number):
    number = serialNumber(number)
    radar_len = int(len(number)/2) #half length of number
    first_four = number[:radar_len] #first_four digits
    if len(set(number)) == 1:
        return False
    return first_four+first_four[::-1] == number

def repeater(number):
    number = serialNumber(number)
    if len(set(number)) == 1:
        return False
    repeater_len = int(len(number)/2)
    half = number[:repeater_len]
    return half+half == number
def radarRepeater(number):
    number = str(number)
    if radar(number) and repeater(number) == True:
        return True
    else:
        return False
def numismatist(number, kind = None):
    list = [] #empty_list
    for element in number:
        if kind == None:
            if solid(element) == True:
                list.append(element)
        else:
            if kind(element) == True:
                list.append(element)
    return list
if __name__ == '__main__':
    import doctest
    doctest.testmod()

'''
Example
>>> serial_number(834783)
'00834783'
>>> serial_number('47839')
'00047839'
>>> serial_number(834783244839184)
'834783244839184'
>>> serial_number('4783926132432*')
Traceback (most recent call last):
AssertionError: invalid serial number

>>> solid(44444444)
True
>>> solid('44544444')
False

>>> radar(1133110)
True
>>> radar('83289439')
False

>>> repeater(20012001)
True
>>> repeater('83289439')
False

>>> radar_repeater('12211221')
True
>>> radar_repeater('83289439')
False

>>> numismatist([33333333, 1133110, '77777777', '12211221'])
[33333333, '77777777']
>>> numismatist([33333333, 1133110, '77777777', '12211221'], radar)
[1133110, '12211221']
>>> numismatist([33333333, 1133110, '77777777', '12211221'], kind=repeater)
['12211221']
'''

informatics/7-advanced functions and modules [advanced]/Cool serial numbers.py


from __future__ import print_function
import sys

import Pyro4.utils.flame

class Rmi():
    # pola

    src_file = None
    dest_name = None

    # funkcje

    def __init__(self):
        self.src_file = ''
        self.dest_name = ''

    def numberoflines(self, filename):
        number = 0
        with open(filename, 'r') as f:
            for line in f:
                number += 1
        return number

    def zmienne(self, source_file=None, dest_name=None):
        if source_file is None:
            self.src_file = input("Wprowadz nazwe pliku zrodlowego: ")
        else:
            self.src_file = source_file

        if dest_name is None:
            self.dest_name = input("Wprowadz nazwe pliku docelowego: ")
        else:
            self.dest_name = dest_name

    def wysylanie(self, server):

        filesource = open(self.src_file, 'rb').read()  # open ('function.py').read()
        # wylacznie zaladowanie kodu na serwer

        server.sendfile(self.dest_name, filesource)

        return 1

    def moduly(self):
        self.zmienne()
        modulesource = open(self.src_file).read()
        # wyslanie kodu JAKO MODUL (import):
        flame.sendmodule(self.dest_name, modulesource)
        # uruchomienie wyslanego kodu z podanymi argumentami
        result = flame.module(self.dest_name).doSomething("Marcin", 42)
        print("\nresult from uploaded module:", result)

    def poprawnosc(self):
        self.zmienne()    #czekaj wrzuc tutaj tylko zmiernna self.src_ zrodlowego...									##czekaj.... wez w tym file source daj plik 4funkcja,py
        filesource = open(self.src_file, 'rb').read()
        root = flame.evaluate(filesource)
        print("result=", root)
        return root

    def funkcja(self, server, filename):
        modulesource = open('function.py').read()
        # wyslanie kodu JAKO MODUL (import):
        server.sendmodule('flameexample.function', modulesource)
        # uruchomienie wyslanego kodu z podanymi argumentami
        result = server.module('flameexample.function').numberoflines(filename)
        print("\nWybrany program ma:", result, "lini kodu")
        r = self.numberoflines(filename)
        print('kod żródłowy zawiera:', r, 'linii')
        if r == result:
            print("Ilość linii kodu programu przesłanego na serwer jest równa ilości linii kodu źródłowego programu,",
                  result, '=', r)

        return result

def main():
    wybor = 0
    print("Witaj programie klienta")
    while wybor != 1 and wybor != 2 and wybor != 3 and wybor != 4 and wybor != 5:
        print("Aby uruchomic funkcje klienta wybierz jedna z wybranych opcji: ")

        print("1 - Wysylanie pliku do serwera")
        print("2 - Sprawdzenie poprawnosci kodu i kompilacja  ")
        print("3 - Wykonanie programu")
        print("4 - 4 funkcja")
        print("5 - Wyjscie z programu")

        wybor = int(input("Podaj numer: "))
        if wybor != 1 and wybor != 2 and wybor != 3 and wybor != 5 and wybor != 4:
            print("Niewłaściwy wybór. Spróbuj jeszcze raz")
    # wyslanie pliku na serwer

    if wybor == 1:
        file.zmienne()
        file.wysylanie(flame)

    # wysylanie kodu jako modul na serwer
    if wybor == 2:
        file.moduly()

    # sprawdzenie poprawnosci
    if wybor == 3:
        file.zmienne()
        file.poprawnosc()
    # sprawdzenie poprawnosci
    if wybor == 4:
        file_name = input("podaj nazwe pliku znajdującego się na serwerze")
        file.funkcja(flame, file_name)
    # wyjscie z menu
    if wybor == 5:
        print("Dziękujemy za skorzystanie z naszych usług. Do zoboczenia!")
        sys.exit(0)

def koniec():
    print("Dziękujemy za skorzystanie z naszych usług. Do widzenia! ")
    return False

if __name__ == '__main__':
    if sys.version_info < (3, 0):
        input = raw_input

    Pyro4.config.SERIALIZER = "pickle"  # flame requires pickle serializer

    print("Start a Pyro Flame server somewhere.")
    # location = input("what is the location of the flame server, hostname:portnumber? ")
    location = "localhost:43863"
    print()

    # connect!
    flame = Pyro4.utils.flame.connect(location)

    # basic stuff
    socketmodule = flame.module("socket")
    osmodule = flame.module("os")
    print("remote host name=", socketmodule.gethostname())
    print("remote server current directory=", osmodule.getcwd())

    file = Rmi()
    # Informacje na serwerze gdy polaczy sie z klientem
    flame.builtin("print")("Serwer polaczyl sie z klientem")

    while True:
        main()
        a = input("Czy chciałbyś jeszcze skorzystać z któreś usługi? (y/n)")
        if a == 'y':
            main()
        elif a != 'y' and a != 'n':
            print("Wybrano niewłaściwy klawisz")
            a = input("Czy chciałbyś jeszcze skorzystać z któreś usługi? (y/n)")

        if a == "n":
            print("Dziękujemy za skorzystanie z naszych usług. Do zoboczenia!")
            sys.exit(0)


from simcity import *

class Factories(dict):
    STYLE = 4
    FG = 38
    BG = 48
    STYLE_MAP = {}

    STARS_SPEED_UP = {
        0: 1.0,
        1: 0.9,
        2: 0.85,  # not confirm
        3: 0.8    # not confirm
    }

    def __hash__(self) -> int:
        return self['cn_name'].__hash__()

    def __init__(self, city, cn_name='工厂', slot=2, **kwargs):
        self._city = city
        self.update(kwargs)
        self['cn_name'] = cn_name
        self['slot'] = slot
        self._idle = slot
        self.__producting = {}
        if cn_name not in Factories.STYLE_MAP:
            self._fg = Factories.FG
            self._bg = Factories.BG
            if Factories.BG == 48:
                Factories.FG -= 1
                if Factories.FG < 31:
                    Factories.BG = 47
            else:
                Factories.FG += 1
            _fmt = ';'.join([str(Factories.STYLE), str(self._fg), str(self._bg)])
            Factories.STYLE_MAP[cn_name] = _fmt
        else:
            _fmt = Factories.STYLE_MAP[cn_name]
        self._cn_name = '\x1b[%sm%s\x1b[0m' % (_fmt, self['cn_name'])

        self.setdefault('_factory', [])
        if len(self._factory) < self.slot:
            for i in range(len(self._factory), self.slot):
                self._factory.append(None)

        self.setdefault('_waiting', [])

        for k in self._factory:
            if k is not None:
                self._idle -= 1

        logging.debug('%s[%d/%d] -> %s ... %s', self._cn_name, self._idle, self.slot, self._factory, self._waiting)

    def __str__(self):
        return "%s#%d" % (self.cn_name, self.slot)

    def _producting(self, pid, raise_on_nf=True, pop=False):
        pid, _pid = abs(pid), pid
        if pid in self.__producting:
            return self.__producting[pid]
        p = self._city.get_product(pid)
        if p is None and raise_on_nf and not pop:
            raise Exception("没有找到 %d 的生产" % _pid)
        if p is not None and not pop:
            self.__producting[pid] = p
        if pop:
            self.__producting.pop(pid, None)
        return pid if p is None else p

    def _factory_get(self, i: int) -> Product:
        if i >= self.slot or self._factory[i] is None or self._factory[i] < 0:
            return None
        pid = self._factory[i]
        return self._producting(pid)

    def factory_get(self, i: int) -> Product:
        return self._factory_get(i)

    def _factory_set(self, i, p: Product):
        self.__producting[p.pid] = p
        self._factory[i], old_pid = p.pid, self._factory[i]
        if old_pid is None:
            self._idle -= 1

    def _factory_del(self, i) -> Product:
        if i >= self.slot or self._factory[i] is None:
            return None
        pid, self._factory[i] = self._factory[i], None
        p = self._producting(pid, pop=True)
        self._idle += 1
        return p

    def waiting_delete(self, product: Product):
        if self._waiting_del(product.pid):
            product.product_done()
            return True
        return False

    def _waiting_del(self, pid) -> bool:
        for i in range(0, len(self._waiting)):
            if self._waiting[i] == pid:
                del self._waiting[i]
                return True
        return False

    def waiting_push(self, p: Product):
        self.__producting[p.pid] = p
        self._waiting.append(p.pid)

    def extend_slot(self, ext_slot: int):
        self['slot'] += ext_slot
        self._idle += ext_slot
        for i in range(len(self._factory), self.slot):
            self._factory.append(None)

    def speed_up_timing(self, time_consuming, start_timing):
        _time_consuming = time_consuming
        if 0 <= start_timing < self.speed_up_end_timing and (self.speed_up_start_timing - start_timing) < _time_consuming:
            if start_timing < self.speed_up_start_timing:
                _normal_time = self.speed_up_start_timing - start_timing
                _speed_up_duration = (_time_consuming - _normal_time) / self.speed_up_times
                if _speed_up_duration > (self.speed_up_end_timing - self.speed_up_start_timing):
                    _speed_up_duration = self.speed_up_end_timing - self.speed_up_start_timing
                _time_consuming -= _speed_up_duration * (self.speed_up_times - 1)
            else:
                _speed_up_duration = self.speed_up_end_timing - start_timing
                if _time_consuming > (_speed_up_duration * self.speed_up_times):
                    _time_consuming -= _speed_up_duration * (self.speed_up_times - 1)
                else:
                    _time_consuming /= self.speed_up_times
        return time_consuming - round(_time_consuming)

    def set_times_speed_up(self, times=2, duration=3600, start_timing=None):
        if self.speed_up_end_timing >= (start_timing if start_timing is not None else self._city.city_timing):
            return False, self.speed_up_start_timing, self.speed_up_end_timing, self.speed_up_times
        self['speed_up_start_timing'] = start_timing if start_timing is not None else self._city.city_timing
        self['speed_up_end_timing'] = self.speed_up_start_timing + duration
        self['speed_up_times'] = times
        if start_timing is None:
            self._city.cprint('%s 的加速币(x%d) 开始生效, 持续时间 %s 直至 %s', format_cn(self.cn_name, 8, left_align=True), self.speed_up_times,
                              fmt_time_delta(duration), fmt_city_timing(self.speed_up_end_timing))
        return True, self.speed_up_start_timing, self.speed_up_end_timing, self.speed_up_times

    @property
    def speed_up_start_timing(self):
        return self['speed_up_start_timing'] if 'speed_up_start_timing' in self else -1

    @property
    def speed_up_end_timing(self):
        return self['speed_up_end_timing'] if 'speed_up_end_timing' in self else -1

    @property
    def speed_up_times(self):
        return abs(self['speed_up_times']) if 'speed_up_times' in self else 1

    def reset_speed_up(self):
        self.pop('speed_up_start_timing', None)
        self.pop('speed_up_end_timing', None)
        self.pop('speed_up_times', None)

    @property
    def stars(self):
        return self['stars'] if 'stars' in self else 0

    @stars.setter
    def stars(self, s):
        self['stars'] = s

    @property
    def stars_speed_up(self):
        return Factories.STARS_SPEED_UP.get(self.stars, 1.0)

    @property
    def cn_name(self):
        return self._cn_name

    @property
    def city(self):
        return self._city

    @property
    def _factory(self):
        return self['_factory']

    @property
    def _waiting(self):
        return self['_waiting']

    @property
    def slot(self):
        return self['slot']

    @property
    def idle_slot(self):
        return self._idle

    @property
    def available_slot(self):
        n = 0
        for k in self._factory:
            n += 1 if k is None or k < 0 else 0
        return n

    def get_schedule(self, initial=False) -> Schedule:
        return FactoriesSchedule(city_timing=0 if initial else self._city.city_timing, fact=self, initial=initial)

    @property
    def is_idle(self):
        n = 0
        for k in self._factory:
            n += 1 if k is not None and k > 0 else 0
        n += len(self._waiting)
        return n == 0

    def is_manufacturing_brother(self, bro):
        manufacturing = MaterialList()
        for pid in self._factory:
            if pid is None:
                continue
            manufacturing.append(self._producting(pid))
        return manufacturing.contain_brother(bro)

    def _manufacturing_count(self, batch_id=0):
        return self.__count(self._factory, batch_id)

    def _waiting_count(self, batch_id=0):
        return self.__count(self._waiting, batch_id)

    def __count(self, iterable, batch_id):
        n = 0
        for pid in iterable:
            if pid is None or pid < 0:
                continue
            p = self._producting(pid)
            if batch_id == 0 or p.batch_id == batch_id:
                n += 1
        return n

    def is_busying(self):
        return False

    def start_products(self, *products, force=False):
        _sth_started = MaterialList()
        for product in products:
            if self._idle <= 0:
                self._city.cprint('排产[%s]失败: 没有空闲生产位', product)
                break
            reason = self._product(product, force=force)
            if reason is not None:
                self._city.cprint('排产[%s]失败: %s', product, reason)
                continue
            if not self.is_busying():
                product.start_timing = self._city.city_timing
                self._city.cprint('  %s 开始生产 %s\x1b[1;38;48m%s\x1b[0m%s.%d, 预计耗时 %s', self.cn_name, '' if product.depth > 0 else '\x1b[4;38;48m',
                                  product, product.consumed_info, self._city.warehouse.capacity, fmt_time_delta(product.time_consuming))
            else:
                product.start_timing = -1
                self._city.cprint('  %s 准备生产 %s\x1b[1;38;48m%s\x1b[0m%s.%d', self.cn_name, '' if product.depth > 0 else '\x1b[4;38;48m',
                                  repr(product), product.consumed_info, self._city.warehouse.capacity)
            self._waiting_del(product.pid)
        return _sth_started

    def _product(self, material: Product, force=False):
        '''尝试排产（如果有空闲生产位的话）'''
        for idx in range(0, self.slot):
            if self._factory[idx] is None:
                self._factory_set(idx, material)
                logging.debug('  %s[%d] 等待生产 "%s"', self.cn_name, idx, material)
                return None
        return '没有找到空闲生产位'

    def _check_slot_done(self, i):
        m = self._factory_get(i)
        if m is not None and not m.is_done() and m.time_to_done <= 1:
            m.product_done()
            self._city.warehouse.append(m)
            self._factory[i] = -self._factory[i]
            self._city.cprint('  %s%s 完成生产 %s\x1b[1;38;42m%s\x1b[0m, 历时 %s/%s', self.cn_name, ('[%02d]' % i) if m.is_factory_material else '',
                              '' if m.depth > 0 else '\x1b[4;38;48m', repr(m),
                              fmt_time_delta(self._city.city_timing - m.start_timing),
                              fmt_time_delta(self._city.city_timing - m.arrange_timing))
            if m.depth == 0 and self._city.auto_into_warehouse:
                self.move_to_warehouse(m, i)
            return m
        else:
            return None

    def move_to_warehouse(self, product: Product, i=-1):
        if i < 0 or product is None:
            for idx in range(0, self.slot):
                pid = self._factory[idx]
                if pid is None or pid > 0:
                    continue
                if product is not None:
                    if -pid == product.pid:
                        i = idx
                        break
                else:
                    self._move_fact_to_ware(idx, product)
        if i >= 0 and product is not None:
            self._move_fact_to_ware(i, product)
            return True
        else:
            return product is None

    def _move_fact_to_ware(self, slot_idx: int, product: Product):
        p = self._factory_del(slot_idx)
        if isinstance(p, int) and product is not None and product.pid == p:
            p = product
        if isinstance(p, Product):
            p.in_warehouse = True
        self._city.cprint('    \x1b[1;38;44m%s 进入仓库\x1b[0m.%d', repr(p), self._city.warehouse.capacity)

    def producting_list(self, include_pending=False, include_done=False):
        _ps = []
        for pid in self._factory:  # 外面还会再排序，此处不需要排序
            if pid is None or pid < 0:
                continue
            p = self._producting(pid)
            if p.is_done() and not include_done:
                continue
            if p.start_timing >= 0 or include_pending:
                _ps.append(p)
        return _ps

    def _compose_factory_arrange(self, batch_id, detail=False):
        if detail:
            return self._compose_factory_arrange_detail()
        _fact = MaterialList()
        _done = MaterialList()
        for pid in self._factory:
            if pid is None:
                continue
            p = self._producting(pid)
            if batch_id == 0 or p.batch_id == batch_id:
                if pid < 0:
                    _done.append(p)
                else:
                    _fact.append(p)
        if len(_done) > 0:
            return '%s, \x1b[0;34;46m%s\x1b[0m]' % (MaterialList.to_str(_fact, suffix=''), MaterialList.to_str(_done, prefix='', suffix=''))
        else:
            return '%s' % _fact

    def _manufacture_order(self, pid1, pid2):
        if pid1 is None:
            return -1
        if pid2 is None:
            return 1
        return self._city.manufacture_order(self._producting(pid1), self._producting(pid2))

    def _compose_factory_arrange_detail(self):
        _arrange_str = '['
        for pid in sorted(self._factory, key=lambda _pid: 0 if _pid is None else self._producting(_pid).latest_product_timing):
            if pid is None:
                continue
            if _arrange_str != '[':
                _arrange_str += '|'
            p = self._producting(pid, raise_on_nf=False)
            if p is None:
                _arrange_str += '\x1b[0;34;46m%d\x1b[0m' % (abs(pid) % 1000)
            else:
                if pid < 0:
                    _arrange_str += '\x1b[0;34;46m'
                if p.depth == 0:
                    _arrange_str += '\x1b[4;38;48m'
                _arrange_str += '%s' % repr(p)
                if p.depth == 0 or pid < 0:
                    _arrange_str += '\x1b[0m'
        if self.idle_slot > 0:
            if _arrange_str != '[':
                _arrange_str += '|'
            _arrange_str += '..x%d' % self.idle_slot
        _arrange_str += ']'
        return _arrange_str

    def print_arrangement(self, batch_id=0, print_idle=False):
        '''输出工厂的生产安排'''
        if self.is_idle or (self._manufacturing_count(batch_id) + self._waiting_count(batch_id) == 0):
            if not print_idle:
                return 'zzz...'
        _arrange_str = self._compose_factory_arrange(batch_id, detail=print_idle) if batch_id == 0 or self._manufacturing_count(batch_id) > 0 else '[ ]'
        if self._waiting_count(batch_id) > 0:
            self._waiting.sort(key=functools.cmp_to_key(self._manufacture_order))
            _w_str = ''
            for i in range(0, len(self._waiting)):
                w = self._producting(self._waiting[i])
                if batch_id == 0 or w.batch_id == batch_id:
                    if _w_str != '':
                        _w_str += '->'
                    if w.depth == 0:
                        _w_str += '\x1b[4;38;48m'
                    if batch_id == 0 and w.start_timing == -2:
                        _w_str += '\x1b[5;38;48m%s\x1b[0m' % (w if batch_id != 0 else repr(w))
                    else:
                        _w_str += '%s' % (w if batch_id != 0 else repr(w))
                        if w.depth == 0:
                            _w_str += '\x1b[0m'
            _arrange_str += ' ...... [%s]' % _w_str
        if batch_id > 0:
            self._city.cprint('     %s 生产 %s', format_cn(self.cn_name, 8), _arrange_str)
        return _arrange_str

    def check_products_done(self):
        '''时光飞逝，让我们看看生产完成了没有'''
        if 0 <= self.speed_up_end_timing < self._city.city_timing and self['speed_up_times'] > 0:
            self._city.cprint('%s 的加速币[x%d] 已经于 %s 失效', self.cn_name, self.speed_up_times, fmt_city_timing(self.speed_up_end_timing))
            self['speed_up_times'] = -self.speed_up_times
        sth_done = False
        for i in range(0, self.slot):
            if self._factory_get(i) is None:
                continue
            if self._check_slot_done(i) is not None:
                sth_done = True
        return sth_done

class _Arrangement(list):

    def __init__(self, product, start, end):
        super().__init__([start, end, product, start, end])

    @property
    def start(self):
        return self[0]

    @property
    def end(self):
        return self[1]

    @property
    def product(self):
        return self[2]

    @property
    def latest_start(self):
        return self[3]

    @latest_start.setter
    def latest_start(self, t):
        self[3] = t

    @property
    def latest_end(self):
        return self[4]

    @latest_end.setter
    def latest_end(self, t):
        self[4] = t

    def __repr__(self):
        return '%s%s(%s/%s-%s/%s)' % ('+'*self.product.depth, self.product, fmt_time(self.start), fmt_time(self.end), fmt_time(self.latest_start + self.product.time_consuming), fmt_time(self.latest_end))

class FactoriesSchedule(Schedule):

    def __init__(self, city_timing, fact: Factories, initial=False):
        self._city_timing = city_timing
        self._fact = fact
        self._schedule = []
        for i in range(self._fact.slot):
            p = None if initial else self._fact.factory_get(i)
            _sch = FactorySchedule(self._city_timing, self._fact, pending_timing=p.time_to_done if p is not None else 0, fact_slot=i)
            self._schedule.append(_sch)

    def schedule_earliest(self, product: Product):
        # 找一个最早可用的fact_slot
        _sch = min(self._schedule, key=lambda _s: _s.available_timing)
        start = _sch.schedule_earliest(product=product)
        return start

    def schedule_latest(self, product: Product, latest: int):
        # 找到所在的fact_slot
        for _sch in self._schedule:
            if product in _sch:
                _sch.schedule_latest(product, latest)
                break

    def log(self, stdout=False, fact_name=None):
        for _sch in self._schedule:
            _sch.log(stdout=stdout, fact_name=fact_name)

class FactorySchedule(Schedule):

    def __init__(self, city_timing: int, fact: Factories, pending_timing=0, fact_slot=None):
        self._city_timing = city_timing
        self._fact = fact
        self._fact_slot = fact_slot
        self._schedule = []
        self._pending_timing = pending_timing

    @property
    def available_timing(self):
        if len(self._schedule) == 0:
            return self._city_timing + self._pending_timing
        _sch = self._schedule[len(self._schedule) - 1]
        return _sch.end

    def __contains__(self, item):
        if isinstance(item, Product):
            for _sch in self._schedule:
                if _sch.product.pid == item.pid:
                    return True
        return False

    def schedule_earliest(self, product: Product):
        return self.schedule_earliest_impl(expect_start=product.waiting_time, duration=product.time_consuming, product=product)

    def schedule_earliest_impl(self, expect_start=None, duration=None, product=None):
        if product is not None and isinstance(product, Product):
            expect_start = product.waiting_time
            duration = product.time_consuming
        if isinstance(expect_start, str):
            expect_start = str2time(expect_start)
        if isinstance(duration, str):
            duration = str2time(duration)
        start = self._city_timing + max(self._pending_timing, expect_start)
        for _sch in self._schedule:
            if _sch.end < start:
                continue
            if start + duration < _sch.start:
                break
            start = _sch.end
        if product is not None and isinstance(product, Product):
            product.all_product_time_consuming = start + product.time_consuming - self._city_timing
        self._schedule.append(_Arrangement(product, start, start + duration))
        self._schedule.sort(key=lambda _s: _s.start)
        return start

    def schedule_latest(self, product: Product, latest: int):
        latest_end = latest
        for _sch in sorted(self._schedule, key=lambda _s: _s.latest_start, reverse=True):
            if _sch.product.pid == product.pid:
                if _sch.latest_start + product.time_consuming < latest_end:
                    _sch.latest_start = latest_end - product.time_consuming
                product.latest_product_timing = _sch.latest_start
                _sch.latest_end = latest
                break
            if latest_end > _sch.latest_start:
                latest_end = _sch.latest_start

    def log(self, stdout=False, fact_name=None):
        if len(self._schedule) == 0:
            return
        _name = '%s' % fact_name if fact_name is not None else self._fact.cn_name
        if self._fact_slot is not None:
            _name += '[%d]' % self._fact_slot
        print_func = logging.debug if not stdout else print if self._fact is None else self._fact.city.cprint
        print_func('%s[%s]:' % (_name, fmt_time(self._pending_timing)))
        for _sch in self._schedule:
            print_func(_sch)


import sys
import random
from itertools import combinations
from typing import List

N = random.randint(99990, 100000)
points = [(random.randint(-100000, 100000), random.randint(-100000, 100000))
           for _ in range(N)]

# N = 4
# points = [(-6, 1), (-6, -2), (2, -3), (3, 2)]
points.sort(key=lambda x: x[0])

# print(N)
# print(points)

def len_point_square(point1, point2: tuple) -> float:
    return (point2[0] - point1[0])**2 + (point2[1] - point1[1])**2

def find_nearest_pair_len(N: int, points: List) -> int:
    if len(points) < 4:
        nearest_len_square = 800000000
        for point1, point2 in combinations(points, 2):
            tmp = len_point_square(point1, point2)
            if nearest_len_square > tmp:
                nearest_len_square = tmp

        return nearest_len_square

    nearest_len_square = min(find_nearest_pair_len(N//2, points[:N//2]), find_nearest_pair_len(N-N//2, points[N//2:]))
    nearest_len = nearest_len_square**(1/2)

    x_middle = (points[N//2-1][0] + points[N//2][0])/2
    left_points, right_points = [], []
    for point in reversed(points[:N//2]):
        if x_middle - point[0] < nearest_len:
            left_points.append(point)
        else:
            break

    if left_points:
        left_y_min = min(left_points, key=lambda x:x[1])
        left_y_max = max(left_points, key=lambda x:x[1])
    else:
        left_y_min = 800000000
        left_y_max = 0

    for point in points[N//2:]:
        if point[0] - x_middle < nearest_len and (abs(point[1] - left_y_min[1]) < nearest_len or abs(point[1] - left_y_max[1]) < nearest_len):
            right_points.append(point)
        else:
            break

    for lpoint in left_points:
        for rpoint in right_points:
            tmp = len_point_square(lpoint, rpoint)
            if nearest_len_square > tmp:
                nearest_len_square = tmp
    return nearest_len_square

print(find_nearest_pair_len(N, points))


import os, sys

import numpy as np
# from garl_gym.scenarios.simple_population_dynamics_rule_base import *
from tqdm import tqdm

def run_rulebase(env, args):
    model_dir = os.path.join('results', 'exp_{:d}'.format(args.experiment_id), 'models')
    bar = tqdm(range(args.test_step))
    try:
        os.makedirs(model_dir)
    except:
        shutil.rmtree(model_dir)
        os.makedirs(model_dir)
    rounds = 0

    img_dir = os.path.join('results', 'exp_{:d}'.format(args.experiment_id), 'images',str(rounds))
    log_dir = os.path.join('results', 'exp_{:d}'.format(args.experiment_id), 'logs', str(rounds))
    try:
        os.makedirs(img_dir)
    except:
        shutil.rmtree(img_dir)
        os.makedirs(img_dir)
    try:
        os.makedirs(log_dir)
    except:
        shutil.rmtree(log_dir)
        os.makedirs(log_dir)
    log = open(os.path.join(log_dir, 'log.txt'), 'w')

    for i in range(args.test_step):
        obs = get_obs(env, only_view=True)
        if args.video_flag:
            env.dump_image(os.path.join(img_dir, '{:d}.png'.format(i)))
        env.take_actions()

        next_view_batches, rewards, killed = get_obs(env)
        env.killed = killed
        episode_reward = np.sum(list(rewards.values()))

        killed = env.remove_dead_agents()

        view_batches = next_view_batches
        msg = "episode {:03d} episode step {:03d} reward:{:5.3f}".format(0, i, episode_reward/len(obs))
        bar.set_description(msg)
        bar.update(1)

        info = "Episode\t{:03d}\tStep\t{:03d}\tReward\t{:5.3f}\tnum_agents\t{:d}\tnum_preys\t{:d}\tnum_predators\t{:d}".format(0, i, episode_reward/len(obs), len(env.agents), len(self.env.preys), len(self.env.predators))
        log.write(info+'\n')
        log.flush()

        env.crossover_prey(self.args.crossover_scope, crossover_rate=self.args.prey_increase_prob)
        #env.increase_prey(self.args.prey_increase_prob)
        env.crossover_predator(self.args.crossover_scope, crossover_rate=self.args.predator_increase_prob)

    log.close()


from typing import List, Optional, Tuple, TypedDict

import torch, torchvision
from torch.utils.data import DataLoader
from torchvision.transforms import ToTensor

from torchvision.models.detection.generalized_rcnn import GeneralizedRCNN
from torchvision.models.detection.faster_rcnn import FastRCNNPredictor
from torchvision.models.detection import MaskRCNN
from torchvision.models.detection.mask_rcnn import MaskRCNNPredictor

from vision.dataset import AbstractCocoDataset

from vision.dataset_factory import create_dataset

from vision.helpers.engine import train_one_epoch, evaluate
from vision.helpers import utils

from os import path
from constants import ROOT_DIR

from PIL import Image
from datetime import datetime

SAVED_MODELS_DIR = path.join(ROOT_DIR, "vision/saved_models/")

class EvalutationResult(TypedDict):
    categories: List[str]
    supercategories: List[str]
    scores: List[int]
    boxes: List[Tuple[float, float, float, float]]

class AbstractLitgoModel:
    model: GeneralizedRCNN
    dataset: AbstractCocoDataset

    def __init__(self, name: str) -> None:
        self.name

    def train(self, num_epochs: int, batch_size: int, test_batch_size: int):
        raise NotImplementedError

    def evaluate(self, image: Image, score_threshold: float) -> EvalutationResult:
        raise NotImplementedError

    def name(self):
        return self.name

    def save(self):
        timestamp = int(datetime.today().timestamp())
        state_filename = f"{self.dataset.get_name()}-{self.name()}-state-{timestamp}.pt"
        torch.save(self.model.state_dict(), path.join(SAVED_MODELS_DIR, state_filename))

    def load_saved_state(self, saved_model_filename: str):
        if not path.isfile(path.join(SAVED_MODELS_DIR, saved_model_filename)):
            raise Exception(f"Could not find {saved_model_filename} in saved_models directory")

        if torch.cuda.is_available():
            self.model.load_state_dict(torch.load(path.join(SAVED_MODELS_DIR, saved_model_filename)))
        else:
            self.model.load_state_dict(torch.load(
                path.join(SAVED_MODELS_DIR, saved_model_filename),
                map_location=torch.device('cpu')
            ))

class LitgoModel(AbstractLitgoModel):
    model: MaskRCNN

    def train(self, num_epochs: int, batch_size: int, test_batch_size: int):
        # get data loaders
        data_loader, data_loader_test = self._get_data_loaders(batch_size, test_batch_size)

        # train on the GPU or on the CPU, if a GPU is not available
        device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')

        # move model to the right device
        self.model.to(device)

        # construct an optimizer
        params = [p for p in self.model.parameters() if p.requires_grad]
        optimizer = torch.optim.SGD(params, lr=0.005,
                                    momentum=0.9, weight_decay=0.0005)
        # and a learning rate scheduler
        lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
                                                        step_size=3,
                                                        gamma=0.1)

        print("Starting Training...")
        for epoch in range(num_epochs):
            # train for one epoch, printing every 10 iterations
            train_one_epoch(self.model, optimizer, data_loader, device, epoch, print_freq=10)
            # update the learning rate
            lr_scheduler.step()
            # evaluate on the test dataset
            evaluate(self.model, data_loader_test, device=device)
        print("Done Training!")

    def evaluate(self, image: Image, score_threshold: float):
        self.model.eval()
        image_tensor = ToTensor()(image)

        with torch.no_grad():
            prediction = self.model([image_tensor])[0]

            # scores are guanteed to be in descending order so find left sublist that meets threshold
            cutoff = 0
            while prediction["scores"][cutoff] >= score_threshold: cutoff += 1 # TODO: Do binary search instead of linear
            labels = prediction["labels"][:cutoff].tolist()

            return EvalutationResult(
                categories=list(map(lambda l: self.dataset.get_category_from_label(l)[0],labels)),
                supercategories=list(map(lambda l: self.dataset.get_category_from_label(l)[1],labels)),
                scores= prediction["scores"][:cutoff].tolist(),
                boxes=prediction["boxes"][:cutoff].tolist()
            )

    def _get_data_loaders(self,  batch_size: int, test_batch_size: int) -> Tuple[DataLoader, DataLoader]:

        # split the dataset into train and validation sets
        indices = torch.randperm(len(self.dataset)).tolist()
        training_dataset = torch.utils.data.Subset(self.dataset, indices[:-50])
        validation_dataset = torch.utils.data.Subset(
            create_dataset(self.dataset.get_name(), train=True), indices[-50:]
        )

        # define training and validation data loaders
        data_loader = torch.utils.data.DataLoader(
            training_dataset, batch_size=batch_size, shuffle=True, num_workers=2,
            collate_fn=utils.collate_fn)

        data_loader_test = torch.utils.data.DataLoader(
            validation_dataset, batch_size=test_batch_size, shuffle=False, num_workers=2,
            collate_fn=utils.collate_fn)

        return data_loader, data_loader_test


# pylint: disable=invalid-name, missing-docstring

from __future__ import print_function

import sys

with open(sys.argv[1], 'r') as fh:
    for line in fh:
        numbers, indices = line.split(':')
        numbers = numbers.strip().split(' ')
        indices = [i.strip().split('-') for i in indices.strip().split(',')]

        for first, second in indices:
            first, second = int(first), int(second)
            numbers[first], numbers[second] = numbers[second], numbers[first]

        print(' '.join(numbers))


# -*- encoding: utf-8 -*-
import os

from mongoengine import *
import models

class Content(object):

	def getMainDocument(self):
		query = {'document_id' : 'main_content'}
		results = models.Content.objects.get_or_create(**query)
		#get the document
		mainContent = results[0]

		if results[1]:
			#new document
			mainContent.content = {}
			mainContent.save()

		return mainContent

	def getAllText(self, language='en'):
		if language is None:
			language = 'de'

		mainDocument = self.getMainDocument()

		localizedTexts = {}
		for text_id, languageDict in mainDocument.content.iteritems():
			localizedTexts[text_id] = languageDict.get(language)

		return localizedTexts


import csv
import boto3
from dynamodb_json import json_util as json
from decimal import Decimal

table_name = 'RipleyFitbark_Activity_Daily'

db_data = []
'''
"date (S)","time (S)","activity_goal (N)","activity_value (N)","distance_in_miles (N)","kcalories (N)","min_active (N)","min_play (N)","min_rest (N)"
"2019-01-18","02:00:00","23100","102","0.01","24","6","0","54"

"date (S)","activity_average (N)","activity_value (N)","daily_target (N)","has_trophy (N)","min_active (N)","min_play (N)","min_rest (N)"
"2019-01-12","46766","5214","4200","0","389","58","352"
'''

with open('RipleyFitbark_Activity_Daily.csv', 'r') as f:
    reader = csv.reader(f)
    count = 0
    for row in reader:
        count = count + 1
        if count == 1:
            continue
        put_item = {}

        # put_item['date'] = row[0]
        # put_item['time'] = row[1]
        # put_item['activity_goal'] = int(row[2])
        # put_item['activity_value'] = int(row[3])
        # put_item['distance_in_miles'] = Decimal(row[4])
        # put_item['kcalories'] = int(row[5])
        # put_item['min_active'] = int(row[6])
        # put_item['min_play'] = int(row[7])
        # put_item['min_rest'] = int(row[8])

        put_item['date'] = row[0]
        put_item['activity_average'] = int(row[1])
        put_item['activity_value'] = int(row[2])
        put_item['daily_target'] = int(row[3])
        put_item['has_trophy'] = int(row[4])
        put_item['min_active'] = int(row[5])
        put_item['min_play'] = int(row[6])
        put_item['min_rest'] = int(row[7])
        db_data.append(put_item)

session = boto3.session.Session(profile_name='ripley_api', region_name='us-west-2')
db = session.resource('dynamodb')
table = db.Table('RipleyFitbark_Activity_Daily')

with table.batch_writer() as batch:
    for item in db_data:
        batch.put_item(
            Item=item
        )


import glob
import json
import argparse
import matplotlib.pyplot as plt
import numpy as np
import logging
from src.shared.logger_factory import LoggerFactory
from src.dependencies.injector import Injector
from src.shared.utils import get_project_root

log = LoggerFactory.logger(__name__)
DEFAULT_PATH = str(get_project_root()) + "/src/scripts/config/default_config.yaml"

def produce_plots(user_name, path = DEFAULT_PATH):
    #series = ['5', '10', '15']
    # series = ['0', '200', '400', '600', '800', '1000', '1200', '1400', '1600', '1800', '2000']
    labels = []
    series_means = {}
    injector = Injector.get_injector_from_file(path)
    process_module = injector.get_process_module()
    dao_module = injector.get_dao_module()

    user_friend_getter = dao_module.get_user_friend_getter()
    friends_cleaner = process_module.get_extended_friends_cleaner()
    user_getter = dao_module.get_user_getter()

    seed_id = user_getter.get_user_by_screen_name(user_name).id
    # Full user friend list
    init_user_friends = user_friend_getter.get_user_friends_ids(seed_id)
    # tweet_processor.process_tweets_by_user_list(init_user_friends)
    global_clean = friends_cleaner.clean_friends_global(seed_id, init_user_friends, tweet_threshold=50,
                                                        follower_threshold=50, bot_threshold=0)
    clean_list10, removed_list10 = friends_cleaner.clean_friends_local(seed_id, global_clean, local_following=10)
    clean_list15, removed_list15 = friends_cleaner.clean_friends_local(seed_id, global_clean, local_following=15)

    lst10 = [str(user) for user in clean_list10]
    lst15 = [str(user) for user in clean_list15]
    user_difference = list(set(lst10) - set(lst15))

# fig = plt.figure()

    fig, axes = plt.subplots(1, 3)
    fig.suptitle('Proportion of Users Removed from Cluster that are Actually Cleaned out for ' + str(user_name) + " with Global Threshold 50 and Comparing Local Threshold 10 to 15")

    titles = ['Cluster 1', 'Cluster 2', 'Cluster 3']
    type = 'local_and_global'
    prefix = "./dc2_exp/"

    d_repr = './dc2_exp/local_and_global/clusters_local_10.0_global_50/david_madras_clusters_0.json'
    d2_repr = './dc2_exp/default/clusters_80/david_madras_clusters_8.json'

    repr_lst = []
    with open(d_repr) as file:
        user_lists = json.load(file)
        #assert len(user_lists) == 3, "Nope!"

        repr1 = user_lists[0]
        repr2 = user_lists[1]
        repr3 = user_lists[2]

    repr3_seed_removed = repr3[:]
    repr3_seed_removed.remove(str(seed_id))

    title1 = titles[0]
    title2 = titles[1]
    title3 = titles[2]

    filename_list = glob.glob(prefix + str(type) + '/clusters_local_15.0_global_50' '/' + str(user_name) + '_clusters_*.json')
    counts1 = []
    counts2 = []
    counts3 = []

    delete_counts1 = []
    delete_counts2 = []
    delete_counts3 = []

    subset_counts1 = []
    subset_counts2 = []

    iterations = []

    ax1 = axes[0]
    ax2 = axes[1]
    ax3 = axes[2]
    j = 0
    filename_start = prefix + str(type) + '/clusters_local_15.0_global_50' '/' + str(user_name) + '_clusters_'
    for k in range(20):
        filename = filename_start + str(k) + '.json'
        with open(filename, 'r') as file:
            user_lists = json.load(file)
            count = len(user_lists)
            sims1 = []
            sims2 = []
            sims3 = []
            for i in range(count):
                sims1.append(jaccard_similarity(user_lists[i], repr1))
                sims2.append(jaccard_similarity(user_lists[i], repr2))
                sims3.append(jaccard_similarity(user_lists[i], repr3))

            index1 = sims1.index(max(sims1))
            index2 = sims2.index(max(sims2))
            index3 = sims3.index(max(sims3))

            #if index1 == index2 and index2 == index3 and index3 == index1:
            if index1 == index2 or count == 1:
                log.info('does not work for ' + filename + ', ' + str(j))
            else:
                max_sim = [max(sims1), max(sims2), max(sims3)]
                max_sim.sort(reverse=True)

                cluster1 = user_lists[index1]
                cluster2 = user_lists[index2]
                cluster3 = user_lists[index3]

                d1 = jaccard_similarity(repr1, cluster1)
                difference1 = list(set(repr1) - set(cluster1))
                delete_counts1.append(overlap(difference1, user_difference))
                subset_counts1.append(overlap(cluster1, repr1))

                d2 = jaccard_similarity(repr2, cluster2)
                difference2 = list(set(repr2) - set(cluster2))
                delete_counts2.append(overlap(difference2, user_difference))
                subset_counts2.append(overlap(cluster2, repr2))

                d3 = jaccard_similarity(repr3, cluster3)
                delete_counts3.append(overlap(repr3_seed_removed, user_difference))

                remaining3 = list(set(repr3_seed_removed)-set(user_difference))
                log.info(remaining3)

                counts1.append(d1)
                counts2.append(d2)
                counts3.append(d3)
                iterations.append(j)
        j += 1

    ax1.bar(iterations, delete_counts1)
    ax2.bar(iterations, delete_counts2)
    ax3.bar(iterations, delete_counts3)

    # Add some text for labels, title and custom x-axis tick labels, etc.

    for ax in [ax1, ax2, ax3]:
        #for ax in [ax2, ax2]:
        ax.set_ylabel('Overlap Similarity')
        ax.set_xlabel('Iteration Number')

    ax1.set_title(title1, fontsize=10)
    ax2.set_title(title2, fontsize=10)
    ax3.set_title(title3, fontsize=10)

    plt.show()

    fig, axes = plt.subplots(1, 2)
    fig.suptitle('Proportion of Users in Remaining Cluster that were in Previous Cluster ' + str(user_name) + " with Global Threshold 50 and Comparing Local Threshold 10 to 15")
    ax1 = axes[0]
    ax2 = axes[1]
    ax1.bar(iterations, subset_counts1)
    ax2.bar(iterations, subset_counts2)

    for ax in [ax1, ax2]:
        #for ax in [ax2, ax2]:
        ax.set_ylabel('Overlap Similarity')
        ax.set_xlabel('Iteration Number')

    ax1.set_title(title1, fontsize=10)
    ax2.set_title(title2, fontsize=10)
    ax3.set_title(title3, fontsize=10)
    plt.show()

    plot_removed(user_difference, repr3, user_name)

def plot_removed(users, rep, user_name):
    sim = jaccard_similarity(users, rep)
    log.info(sim)
    counts = []
    iterations = []
    for i in range(20):
        filename_start = './dc2_exp/local_and_global/clusters_local_10.0_global_50' '/' + str(user_name) + '_clusters_'
        filename = filename_start + str(i) + '.json'
        with open(filename, 'r') as file:
            user_lists = json.load(file)
            count = len(user_lists)
            if count == 3:
                counts.append(sim)
                iterations.append(i)
    plt.ylabel('Jaccard Similarity')
    plt.xlabel('Iteration Number')
    plt.title('Jaccard Similarity of the Removed Users from Local Threshold 10-15 to Cluster 3 for ' + str(user_name))
    plt.bar(iterations, counts)
    plt.show()

def jaccard_similarity(user_list1, user_list2):
    intersection = len(list(set(user_list1).intersection(user_list2)))
    union = (len(user_list1) + len(user_list2)) - intersection

    return float(intersection) / union

def overlap(user_list1, user_list2):
    intersection = len(list(set(user_list1).intersection(user_list2)))

    return float(intersection) / len(user_list1)

if __name__ == "__main__":
    """
    Short script to produce scatter plots
    """
    parser = argparse.ArgumentParser(description='Short script to produce scatter plots of utility')
    parser.add_argument('-n', '--screen_name', dest='name',
                        help="The screen name of the user to download", required=True)

    args = parser.parse_args()

    produce_plots(args.name)


import nuke
float2 = nuke.math.Vector2
float3 = nuke.math.Vector3
float3x3 = nuke.math.Matrix3
float4x4 = nuke.math.Matrix4

def get_primaries(node, dst=False):
    # get xy coordinates from node
    dst_pri = ['drxy', 'dgxy', 'dbxy', 'dwxy']
    pri = ['rxy', 'gxy', 'bxy', 'wxy']
    d = {}
    for i, p in enumerate(pri):
        if dst:
            d[p] = float2(node[dst_pri[i]].getValue()[0], node[dst_pri[i]].getValue()[1])
        else:
            d[p] = float2(node[p].getValue()[0], node[p].getValue()[1])
    return d

def RGBtoXYZ(xy, Y=1.0, inverse=False):
    # given r g b chromaticities and whitepoint
    # convert RGB colors to XYZ
    # based on CtlColorSpace.cpp from the CTL source code
    r = xy['rxy']
    g = xy['gxy']
    b = xy['bxy']
    w = xy['wxy']

    X = w.x * Y / w.y
    Z = (1 - w.x - w.y) * Y / w.y

    # Scale factors for matrix rows
    d = r.x * (b.y - g.y) + b.x * (g.y - r.y) + g.x * (r.y - b.y)

    Sr =    (X * (b.y - g.y) -      \
            g.x * (Y * (b.y - 1) +  \
            b.y  * (X + Z)) +       \
            b.x  * (Y * (g.y - 1) + \
            g.y * (X + Z))) / d

    Sg =    (X * (r.y - b.y) +      \
            r.x * (Y * (b.y - 1) +  \
            b.y * (X + Z)) -        \
            b.x * (Y * (r.y - 1) +  \
            r.y * (X + Z))) / d

    Sb =    (X * (g.y - r.y) -      \
            r.x * (Y * (g.y - 1) +  \
            g.y * (X + Z)) +        \
            g.x * (Y * (r.y - 1) +  \
            r.y * (X + Z))) / d

    # Assemble the matrix
    M = float3x3()
    M.set(  Sr * r.x, Sr * r.y, Sr * (1 - r.x - r.y),
            Sg * g.x, Sg * g.y, Sg * (1 - g.x - g.y),
            Sb * b.x, Sb * b.y, Sb * (1 - b.x - b.y))
    if inverse:
        return M.inverse()
    else:
        return M

def start():
    node = nuke.thisNode()

    invert = node['invert'].getValue()
    label = node['label']

    if invert:
        # Convert from XYZ colorspace to Src RGB
        mtx = RGBtoXYZ(get_primaries(node), inverse=True)
        label.setValue('XYZ to {0}'.format(node['src_colorspace_name'].getValue()))
    else:
        # Convert from Src RGB colorspace to XYZ
        mtx = RGBtoXYZ(get_primaries(node))
        label.setValue('{0} to XYZ'.format(node['src_colorspace_name'].getValue()))

    node['matrix'].setValue((mtx))

if __name__=='__main__':
    start()


# tcpsocket.py
# author: Jacob Schwartz (schwartzj1)

# Library imports
import re
import socket
from request import Request
from urlparser import URLparser

# Constants
BUFFER = 2048
TIMEOUT = 25

# Class definition
class TCPsocket:

    status2xx = 0
    status3xx = 0
    status4xx = 0
    status5xx = 0
    statusOther = 0

    def __init__(self):
        self.sock = None
        self.host = ""

    # Create Socket
    def createSocket(self):
        try:
            self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        except socket.error as error:
            # print("Failed to create a TCP socket {}".format(error))
            self.sock = None

    # Get IP from Hostname
    def getIP(self, url):
        self.host = url
        try:
            return socket.gethostbyname(url)
        except socket.gaierror:
            # print("Failed to gethostbyname\n")
            return None
        except UnicodeError:
            return None

    # Function for debugging
    def connect(self, ip, port):

        if self.sock is None or ip is None:
            if self.sock is None:
                print("Sock is empty")

            if ip is None:
                print("IP is empty\n")
            return
        try:
            self.sock.connect((ip, port))
            print("Connection Successful: ", ip + "\n")
        except socket.error as error:
            print("Connection Failed {}".format(error))
            self.sock.close()
            self.sock = None

    def crawl(self, host, port, request):

        ip = self.getIP(host)
        found = False
        status = 0
        href = 0
        size = 0
        # pagelinks = list()

        if self.sock is None or ip is None or request is None:
            self.closeSocket()
            return found, status, href, size

        try:
            self.sock.settimeout(5.0)
            self.sock.connect((ip, port))
            # print("GET request response:")
            self.sock.send(request)
            response = self.sock.recv(4096)
            getList = []
            while len(response) > 0:
                getList.append(response.decode("utf-8", "ignore"))
                response = self.sock.recv(4096)

            getResponse = "".join(getList)
            # print(getResponse)

            size = len(getResponse.encode('utf-8'))

            if "HTTP/1.0 2" in getResponse or "HTTP/1.1 2" in getResponse:
                # regex = re.compile('(?<=href=").*?(?=")')
                found = True
                # pagelinks = regex.findall(getResponse)
                # href += len(pagelinks)
                href += getResponse.count("href=")
                status = 2
            elif "HTTP/1.0 3" in getResponse or "HTTP/1.1 3" in getResponse:
                status = 3
            elif "HTTP/1.0 4" in getResponse or "HTTP/1.1 4" in getResponse:
                status = 4
            elif "HTTP/1.0 5" in getResponse or "HTTP/1.1 5" in getResponse:
                status = 5
        except socket.error as error:
            found = False
        finally:
            self.closeSocket()

        return found, status, href, size

    def robots(self, host, port, request):

        ip = self.getIP(host)
        found = False
        info = "\tLoading... failed\n"

        if self.sock is None or ip is None or request is None:
            self.closeSocket()
            return found, info

        info = "\tLoading... done\n"

        try:
            self.sock.settimeout(5.0)
            self.sock.connect((ip, port))
            self.sock.send(request)
            response = self.sock.recv(4096)
            headList = []
            while len(response) > 0:
                headList.append(response.decode("utf-8", "ignore"))
                response = self.sock.recv(4096)
            headResponse = "".join(headList)

            # print(headResponse)

            if "200 OK" in headResponse:
                found = True
        except socket.error as error:
            info = "\tLoading... failed\n"
        finally:
            self.closeSocket()

        return found, info

    # Used for testing how many extracted links return with status 200 OK. Not used in final project
    def linkStatus(self, url):

        urlParse = URLparser()
        r = Request()

        host, port, path, file = urlParse.parse(url)

        request = r.createGETReq(host, path, file)
        ip = self.getIP(host)
        found = False

        self.createSocket()

        if self.sock is None or ip is None or request is None:
            self.closeSocket()
            return found

        gReq = str.encode(request)

        try:
            self.sock.settimeout(5.0)
            self.sock.connect((ip, port))
            self.sock.send(gReq)
            response = self.sock.recv(4096)
            getList = []
            while len(response) > 0:
                getList.append(response.decode("utf-8", "ignore"))
                response = self.sock.recv(4096)

            getResponse = "".join(getList)
            # print(getResponse)

            if "HTTP/1.0 2" in getResponse or "HTTP/1.1 2" in getResponse:
                found = True
        except socket.error as error:
            found = False
        finally:
            self.closeSocket()

        return found

    def closeSocket(self):
        self.sock.close()
        self.sock = None


# Sum of Cubes
# https://www.codechef.com/problems/SUMCUBE

# Editorial's approach
# https://discuss.codechef.com/questions/108250/sumcube-editorial
# O(m ^ 1.5)

MOD = 10**9 + 7

def pow2(n):
    return 0 if n<0 else pow(2, n, MOD)

def getB():
    gs = [set(g_u) for g_u in g]
    es = set(e)
    B = [0]*(n+1)
    lim = m**0.5
    for u in range(1, n+1):
        if deg[u] < lim:
            B[u] = sum((v, w) in es for v in g[u] for w in g[u])
        else:
            B[u] = sum(v in gs[u] and w in gs[u] for (v, w) in e)
    return B

def solve3():
    a23, a24 = solve2(get_a23_a24=True)
    B = getB()
    tri = sum(B)
    claw = sum(deg_u*(deg_u-1)*(deg_u-2) for deg_u in deg)
    P4 = 3*sum((deg[u]-1)*(deg[v]-1) for u, v in e) - 3*tri

    a32 = m
    a33 = 3*a23 + tri
    a34 = 3*a24 + claw + P4
    a35 = 3*sum(deg[u]*(deg[u]-1)*(m-deg[u]+2)+B[u] - 2*(deg[u]-1)*sum(deg[v] for v in g[u]) for u in range(1, n+1))
    a36 = m*m*m - a32 - a33 - a34 - a35
    return (a32*pow2(n-2) + a33*pow2(n-3) + a34*pow2(n-4) + a35*pow2(n-5) + a36*pow2(n-6)) % MOD

def solve2(get_a23_a24=False):
    a22 = m
    a23 = sum(deg_u*(deg_u-1) for deg_u in deg)
    a24 = m*m - a22 - a23
    if get_a23_a24:
        return a23, a24
    return (a22*pow2(n-2) + a23*pow2(n-3) + a24*pow2(n-4)) % MOD

def solve1():
    return (m * pow2(n-2)) % MOD

T = int(input())
for t in range(T):
    n, m, k = map(int, input().split())
    e = []
    g = [[] for i in range(n+1)]
    for i in range(m):
        u, v = map(int, input().split())
        e.append((u, v))
        e.append((v, u))
        g[u].append(v)
        g[v].append(u)
    deg = [len(g_u) for g_u in g]

    ans = solve1() if k==1 else solve2() if k==2 else solve3()
    print(ans)


#!/usr/bin/eny python
# -*- coding:utf-8 -*-
import urllib.request
import urllib.parse
from bs4 import BeautifulSoup
import re

def askURL(url):
    html = ""
    head = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36 Edg/87.0.664.75"}
    request = urllib.request.Request(url,headers=head);
    # print (reponse.read().decode("utf-8"))
    try:
        reponse = urllib.request.urlopen(request)
        html = reponse.read()
    except urllib.error.URLError as e:
        print("error")

    return html

def getData():
    url = "https://www.baidu.com/s?ie=utf-8&medium=1&rtt=4&bsst=1&rsv_dl=news_b_pn&cl=2&wd=%E7%96%AB%E6%83%85&tn=news&rsv_bp=1&tfflag=0&x_bfe_rqs=03E80000001&x_bfe_tjscore=0.000000&tngroupname=organic_news&newVideo=12&pn=520"
    html = askURL(url)
    bs = BeautifulSoup(html,"lxml")
    # print(bs.title.string)

    for item in bs.find_all("div",class_='result-op c-container xpath-log new-pmd'):
        print(item.find_all("h3"))

getData()