import os
import pickle
import argparse
import torch
import torch.optim as optim
from torch.utils.data import DataLoader
from Model.INPLIM import Doctor
from data_utils import CodeLevelDataset
from utils import train_eval

def args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--data_root', type=str, default='',
                        help='Set the root path of the dataset')
    parser.add_argument('--devices', type=str, default='cpu',
                        help='Setting the IDs of GPU devices.')
    parser.add_argument('--epochs', type=int, default=15,
                        help='Setting the number of epochs to run.')
    parser.add_argument('--batch_size', type=int, default=512,
                        help='Setting the mini-batch size.')
    parser.add_argument('--weight_decay', type=float, default=1e-3,
                        help='Setting weight decay')
    parser.add_argument('--lr', type=float, default=1e-3,
                        help='Setting the learning rate.')
    parser.add_argument('--dim', type=int, default=128,
                        help='Setting the inner dim of the model.')
    parser.add_argument('--max_len', type=int, default=200,
                        help='Setting the maximum number of code to use for a patient.')
    parser.add_argument('--drop_context', type=float, default=0.3,
                        help='Setting drop rate of the context-aware branch.')
    parser.add_argument('--drop_time', type=float, default=0.3,
                        help='Setting drop rate of the time-aware branch.')
    parser.add_argument('--save_model', action='store_true',
                        help='Whether to save the parameters of the trained model.',
                        default=True)
    parser.add_argument('--save_dir', type=str, default='./saved_models',
                        help='Setting the dir of saving trained model.')
    return parser

def main(opts):
    if opts.devices != 'cpu':
        os.environ['CUDA_VISIBLE_DEVICES'] = opts.devices

    dataset = pickle.load(open(opts.data_root, 'rb'))
    train_set = CodeLevelDataset(dataset=dataset, max_len=opts.max_len, phase='train')
    valid_set = CodeLevelDataset(dataset=dataset, max_len=opts.max_len, phase='val')
    test_set = CodeLevelDataset(dataset=dataset, max_len=opts.max_len, phase='test')

    train_loader = DataLoader(train_set, batch_size=opts.batch_size, num_workers=2, shuffle=True)
    val_loader = DataLoader(valid_set, batch_size=1, num_workers=1, shuffle=False)
    test_loader = DataLoader(test_set, batch_size=1, num_workers=1, shuffle=False)

    net = Doctor(features=dataset['features'], out_dim=2, emb_dim=opts.dim, dropout_context=opts.drop_context,
                 dropout_time=opts.drop_time)
    criterion = torch.nn.CrossEntropyLoss()
    optimizer = optim.Adam(net.parameters(), lr=opts.lr, weight_decay=opts.weight_decay, eps=0)

    if opts.devices != 'cpu':
        net = torch.nn.DataParallel(net).cuda()
        criterion = criterion.cuda()

    best_auc, best_epoch, best_test_nll, best_test_auc_roc = 0, 0, 0, 0
    model_dict = {}
    for epoch in range(opts.epochs):
        train_eval(opts, net, criterion, optimizer, train_loader, epoch, phase='Train')
        _, auc = train_eval(opts, net, criterion, optimizer, val_loader, epoch, phase='Valid')

        if auc > best_auc:
            best_auc, best_epoch = auc, epoch
            best_test_nll, best_test_auc_roc = train_eval(opts, net, criterion, optimizer, test_loader, epoch,
                                                          phase='Test')
            model_dict['opts'] = opts
            model_dict['states'] = net.state_dict()
    print('Best Test NLL:{:.4f}\t Best AUROC:{:.4f}'.format(best_test_nll, best_test_auc_roc))

    if not os.path.exists(opts.save_dir):
        os.makedirs(opts.save_dir)

    if opts.save_model:
        torch.save(model_dict, os.path.join(opts.save_dir, 'Model-AUC-{:.4f}.pth'.format(best_test_auc_roc)))

if __name__ == '__main__':
    opts = args().parse_args()
    main(opts)


from fastai import vision, metrics
from fastai.callback import hooks
from fastai.utils import mem
import numpy as np
from os import path
import torch

vision.defaults.device = vision.defaults.device if torch.cuda.is_available() else torch.device('cpu')

# Download data and get path
fastai_path = vision.untar_data(vision.URLs.CAMVID)
PATH = str(fastai_path)
print('CAMVID paths:')
print(fastai_path.ls())
BATCH_SIZE = 64
WD = 1e-2
LR = 1e-4
PCT_START_FINETUNE = 0.9 # given the default of 0.3, it means that your LR is going up for 30% of your iterations and then decreasing over the last 70%
PCT_START = 0.8
EPOCHS_FINETUNE = 12
EPOCHS = 12

# Define images and label path
LABEL_PATH = path.sep.join([PATH, 'labels'])
IMAGE_PATH = path.sep.join([PATH, 'images'])

# Define paths of image and label
image_paths = vision.get_image_files(IMAGE_PATH)
label_paths = vision.get_image_files(LABEL_PATH)

# Load some samples to see what's inside
rand_indx = np.random.randint(0, len(image_paths))
sample_image_path = image_paths[rand_indx]
sample_image = vision.open_image(sample_image_path)
sample_image.show(figsize=(6, 6))
# Function to match between image and its label path. E.g. image path: /root/.fastai/data/camvid/images/0006R0_f02910.png; label path: /root/.fastai/data/camvid/labels/0006R0_f02910_P.png
segment_name_fn = lambda image_path: path.sep.join([LABEL_PATH, f'{image_path.stem}_P{image_path.suffix}'])
# Load image segmentation by defaults (segment image given in dataset) and vision.open_mask()
sample_label_path = segment_name_fn(sample_image_path)
sample_label = vision.open_image(sample_label_path)
sample_label.show(figsize=(6, 6))
# Note sample segment after preprocess based on vision.open_mask just has 1 depth instead of 3 depth as origin segment
sample_label_preprocessed = vision.open_mask(sample_label_path)
sample_label_preprocessed.show(figsize=(6, 6))
print(sample_label_preprocessed.data) # sample_label_preprocessed is also fastai tensor

# get image dimension (height and width)
image_size = np.array(sample_label_preprocessed.shape[1:])
data_size = image_size//2
objects_in_image = np.loadtxt(path.sep.join([PATH, 'codes.txt']), dtype=str)

# Determine batch size by gpu free memory to avoid CUDA out pf memory
if torch.cuda.is_available():
    free = mem.gpu_mem_get_free_no_cache()
    if free > 8200:
        BATCH_SIZE = 8
    else:
        BATCH_SIZE = 4

    print(f'Using batch size of {BATCH_SIZE}, have {free}MB of GPU RAM free')

origin_data = vision.SegmentationItemList.from_folder(IMAGE_PATH).split_by_fname_file(path.sep.join([PATH, 'valid.txt'])).label_from_func(segment_name_fn, classes=objects_in_image)
data = origin_data.transform(vision.get_transforms(), size=data_size, tfm_y=True).databunch(bs=BATCH_SIZE).normalize(vision.imagenet_stats) # tfm_y=True means apply transform to label
print(data.show_batch(2, figsize=(10, 7)))
print(data.show_batch(2, figsize=(10, 7), ds_type=vision.DatasetType.Valid))

# Define accuracy
object2id = {value: key for key, value in enumerate(objects_in_image)}
void_index = object2id['Void']

def camvid_accuracy(inputs, target):
    target = target.squeeze(1)
    mask = target != void_index
    return (inputs.argmax(dim=1)[mask] == target[mask]).float().mean()

# Define model
learner = vision.unet_learner(data, vision.models.resnet34, metrics=camvid_accuracy, wd=WD)
# Find good LR
learner.lr_find()
learner.recorder.plot()
learner.fit_one_cycle(EPOCHS_FINETUNE, max_lr=slice(LR), pct_start=PCT_START_FINETUNE)
learner.save('stage-1-34-unet')
# Show results
learner.show_results(rows=3, figsize=(8, 9))

# After warming up, start to train all network
learner.unfreeze()
learner.fit_one_cycle(EPOCHS, max_lr=slice(LR/400, LR/4), pct_start=PCT_START)
learner.save('stage-2-34-unet')


import functools
from enum import Enum
import json
import re

class SCJDEntry:
	def __init__(self):
		self.data = {}

	def set_title(self, title):
		self.data['title'] = title

	def set_id(self, idx):
		self.data['idx'] = idx

	def glue_pronounce(self, prn):
		if 'prn' in self.data:
			self.data['prn'] += prn
		else:
			self.data['prn'] = prn

	def push_kanji(self, kanji):
		if 'kanjis' in self.data:
			self.data['kanjis'].append(kanji)
		else:
			self.data['kanjis'] = [kanji]

	def push_void_definition(self):
		if 'defs' in self.data:
			self.data['defs'].append({})
		else:
			self.data['defs'] = [{}]

	def set_definition_order(self, order):
		self.data['defs'][-1]['order'] = order

	def glue_definition_indicator(self, ind):
		if 'defs' in self.data:
			if 'ind' in self.data['defs'][-1]:
				self.data['defs'][-1]['ind'] += ind
			else:
				self.data['defs'][-1]['ind'] = ind

	def push_definition_chinese_translation(self, trans):
		if 'chi_transs' in self.data['defs'][-1]:
			self.data['defs'][-1]['chi_transs'].append(trans)
		else:
			self.data['defs'][-1]['chi_transs'] = [trans]

	def push_definition_english_translation(self, trans):
		if 'eng_transs' in self.data['defs'][-1]:
			self.data['defs'][-1]['eng_transs'].append(trans)
		else:
			self.data['defs'][-1]['eng_transs'] = [trans]

	def push_void_definition_sentence_example(self):
		if 'sent_exs' in self.data['defs'][-1]:
			self.data['defs'][-1]['sent_exs'].append({})
		else:
			self.data['defs'][-1]['sent_exs'] = [{}]

	def glue_definition_sentence_example(self, jpn_sent, chi_sent):
		if jpn_sent is not None:
			if 'jpn_sent' in self.data['defs'][-1]['sent_exs'][-1]:
				self.data['defs'][-1]['sent_exs'][-1]['jpn_sent'] += jpn_sent
			else:
				self.data['defs'][-1]['sent_exs'][-1]['jpn_sent'] = jpn_sent

		if chi_sent is not None:
			if 'chi_sent' in self.data['defs'][-1]['sent_exs'][-1]:
				self.data['defs'][-1]['sent_exs'][-1]['chi_sent'] += chi_sent
			else:
				self.data['defs'][-1]['sent_exs'][-1]['chi_sent'] = chi_sent

	def push_void_phrase(self):
		if 'phrs' in self.data:
			self.data['phrs'].append({})
		else:
			self.data['phrs'] = [{}]

	def set_phrase_idx(self, idx):
		self.data['phrs'][-1]['idx'] = idx

	def glue_phrase_first_title(self, title):
		if 'titles' in self.data['phrs'][-1]:
			self.data['phrs'][-1]['titles'][0] += title
		else:
			self.data['phrs'][-1]['titles'] = [title]

	def push_phrase_title(self, title):
		self.data['phrs'][-1]['titles'].append(title)

	def push_void_phrase_section(self):
		if 'secs' in self.data['phrs'][-1]:
			self.data['phrs'][-1]['secs'].append({})
		else:
			self.data['phrs'][-1]['secs'] = [{}]

	def push_phrase_section_translation(self, trans):
		if 'transs' in self.data['phrs'][-1]['secs'][-1]:
			self.data['phrs'][-1]['secs'][-1]['transs'].append(trans)
		else:
			self.data['phrs'][-1]['secs'][-1]['transs'] = [trans]

	def push_void_phrase_section_sentence_example(self):
		if 'sent_exs' in self.data['phrs'][-1]['secs'][-1]:
			self.data['phrs'][-1]['secs'][-1]['sent_exs'].append({})
		else:
			self.data['phrs'][-1]['secs'][-1]['sent_exs'] = [{}]

	def glue_phrase_section_sentence_example(self, jpn_sent, chi_sent):
		if jpn_sent is not None:
			if 'jpn_sent' in self.data['phrs'][-1]['secs'][-1]['sent_exs'][-1]:
				self.data['phrs'][-1]['secs'][-1]['sent_exs'][-1]['jpn_sent'] += jpn_sent
			else:
				self.data['phrs'][-1]['secs'][-1]['sent_exs'][-1]['jpn_sent'] = jpn_sent

		if chi_sent is not None:
			if 'chi_sent' in self.data['phrs'][-1]['secs'][-1]['sent_exs'][-1]:
				self.data['phrs'][-1]['secs'][-1]['sent_exs'][-1]['chi_sent'] += chi_sent
			else:
				self.data['phrs'][-1]['secs'][-1]['sent_exs'][-1]['chi_sent'] = chi_sent

	def get_data(self):
		return self.data

class SCJDStateMachine:
	class Node(Enum):
		IGNORE = 0
		ROOT = 1
		D_ENTRY = 2
		D_PRN = 3
		D_DEF = 4
		A = 5

		HWG_SPAN = 10
		HW_SPAN = 11
		HV_SPAN = 12
		GRAMB_SPAN = 13
		SEMB_SPAN = 14
		TRG_SPAN = 15
		OUP_LABEL_SPAN = 16
		TRANS_SPAN = 17
		IDMB_SPAN = 18
		IDMSEC_SPAN = 19
		IDM_SPAN = 20
		EXG_SPAN = 21
		EX_SPAN = 22
		IND_SPAN = 23
		CB_SPAN = 24
		CSEC_SPAN = 25
		CW_SPAN = 26
		CV_SPAN = 27

	class State(Enum):
		NUMBNESS = 0
		BEGIN = 1

		GRAMB = 10
		GRAMB_SEMB = 11
		GRAMB_OUP_LABEL = 12
		GRAMB_TRG_AFTER_OUP_LABEL = 13
		GRAMB_EXG = 14

		IDMB = 20
		IDMB_FIRST_IDM = 21
		IDMB_NOT_FIRST_IDM = 22
		IDMB_SEMB = 23
		IDMB_EXG = 24

		CB = 30
		CB_GRAMB = 31
		CB_SEMB = 32
		CB_EXG = 33

		def get_gramb_cluster():
			return [SCJDStateMachine.State.GRAMB, SCJDStateMachine.State.GRAMB_SEMB, SCJDStateMachine.State.GRAMB_OUP_LABEL, SCJDStateMachine.State.GRAMB_EXG, SCJDStateMachine.State.GRAMB_TRG_AFTER_OUP_LABEL]

		def get_idmb_cluster():
			return [SCJDStateMachine.State.IDMB, SCJDStateMachine.State.IDMB_FIRST_IDM, SCJDStateMachine.State.IDMB_NOT_FIRST_IDM, SCJDStateMachine.State.IDMB_SEMB, SCJDStateMachine.State.IDMB_EXG]

		def get_cb_cluster():
			return [SCJDStateMachine.State.CB, SCJDStateMachine.State.CB_GRAMB, SCJDStateMachine.State.CB_SEMB, SCJDStateMachine.State.CB_EXG]

	IGNORE_SPAN = {'hvg', 'gp', 'x_xoh', 'ty_pinyin', 'x_xdh', 'sn', 'gl', 'cwg', 'cvg', 'tail', 'ty_日中比較', 'x_xopt', 'pr', 'ty_参考', 'ty_参考参照', 'ty_項目参照', 'ty_注意', 'gr', 'ty_文化', 'ph', 'xr', 'xrlabelGroup', 'xrlabel', 'underline'}
	INHERIT_SPAN = {'rf', 'tg_ind', 't_fld', 'subEnt'}

	def __init__(self):
		self.reinit()

	def reinit(self):
		self.stk = [SCJDStateMachine.Node.ROOT]
		self.sta = SCJDStateMachine.State.BEGIN

	def get_node(self):
		return self.stk[-1]

	def get_state(self):
		return self.sta

	def push_node(self, node):
		self.stk.append(node)

	def pop_node(self):
		return self.stk.pop()

	def numb(self):
		self.sta = SCJDStateMachine.State.NUMBNESS

	def is_numb(self):
		return self.get_state() == SCJDStateMachine.State.NUMBNESS

	def startelement_move(self, tag, attrs):

		if tag == 'd:entry':
			self.push_node(SCJDStateMachine.Node.D_ENTRY)

		elif tag == 'span':
			attrs_keys = attrs.getQNames()
			if 'class' in attrs_keys:
				attrs_class_values = attrs['class'].split(' ')

				if not SCJDStateMachine.INHERIT_SPAN.isdisjoint(attrs_class_values):
					self.push_node(self.get_node())

				elif not SCJDStateMachine.IGNORE_SPAN.isdisjoint(attrs_class_values):
					self.push_node(SCJDStateMachine.Node.IGNORE)

				elif 'hwg' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.HWG_SPAN)

				elif 'hw' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.HW_SPAN)

				elif 'hv' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.HV_SPAN)

				elif 'gramb' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.GRAMB_SPAN)
					if self.get_state() == SCJDStateMachine.State.BEGIN:
						self.sta = SCJDStateMachine.State.GRAMB

					elif self.get_state() in SCJDStateMachine.State.get_cb_cluster():
						self.sta = SCJDStateMachine.State.CB_GRAMB

					else:
						raise RuntimeError(f'{self.get_state()}')

				elif 'semb' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.SEMB_SPAN)

					if self.get_state() in SCJDStateMachine.State.get_gramb_cluster():
						self.sta = SCJDStateMachine.State.GRAMB_SEMB

					elif self.get_state() in SCJDStateMachine.State.get_idmb_cluster():
						self.sta = SCJDStateMachine.State.IDMB_SEMB

					elif self.get_state() in SCJDStateMachine.State.get_cb_cluster():
						self.sta = SCJDStateMachine.State.CB_SEMB

					else:
						raise RuntimeError(f'{self.get_state()}')

				elif 'trg' in attrs_class_values:
					"""
					Generally, there is only one "oup_label" span in "gramb-semb" span, except in some rare cases.
					This happens when there is more than one kinds of translation in one "gramb-semb" span.
					An example is where id = j_CRJC000115, and title = 相手役.
					That "[芝居など] 配角" and "[ダンス] 舞伴" show up here is weird.
					And this is the reason why I put "tg_ind" span into INHERIT_SPAN instead of IGNORE_SPAN.
					Otherwise, key of ind in entry will become "芝居などダンス" instead of "[芝居など][ダンス]".
					"""
					if 'x_xd2' in attrs_class_values:
						if self.get_state() == SCJDStateMachine.State.GRAMB_OUP_LABEL:
							self.sta = SCJDStateMachine.State.GRAMB_TRG_AFTER_OUP_LABEL

						elif self.get_state() == SCJDStateMachine.State.GRAMB_EXG:
							pass

						elif self.get_state() == SCJDStateMachine.State.GRAMB_SEMB:
							pass

						elif self.get_state() == SCJDStateMachine.State.GRAMB_TRG_AFTER_OUP_LABEL:
							pass

						else:
							raise RuntimeError(f'{self.get_state()}')

					self.push_node(SCJDStateMachine.Node.TRG_SPAN)

				elif 'oup_label' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.OUP_LABEL_SPAN)

					if self.get_state() == SCJDStateMachine.State.GRAMB_SEMB:
						self.sta = SCJDStateMachine.State.GRAMB_OUP_LABEL

					elif self.get_state() == SCJDStateMachine.State.GRAMB_TRG_AFTER_OUP_LABEL:
						self.sta = SCJDStateMachine.State.GRAMB_OUP_LABEL

						"""
						Generally, there is no "oup_label" span in "idmb" span, except in some rare cases.
						An example is where id = j_CRJC010600, and title = 塞翁が馬.
						"""
					elif self.get_state() == SCJDStateMachine.State.IDMB_SEMB:
						pass

					else:
						raise RuntimeError(f'{self.get_state()}')

				elif 'trans' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.TRANS_SPAN)

				elif 'idmb' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.IDMB_SPAN)
					self.sta = SCJDStateMachine.State.IDMB

				elif 'idmsec' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.IDMSEC_SPAN)

				elif 'idm' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.IDM_SPAN)

					if self.get_state() == SCJDStateMachine.State.IDMB or self.get_state() == SCJDStateMachine.State.IDMB_EXG or self.get_state() == SCJDStateMachine.State.IDMB_SEMB:
						self.sta = SCJDStateMachine.State.IDMB_FIRST_IDM

					elif self.get_state() == SCJDStateMachine.State.IDMB_FIRST_IDM:
						self.sta = SCJDStateMachine.State.IDMB_NOT_FIRST_IDM

				elif 'exg' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.EXG_SPAN)

					if self.get_state() in SCJDStateMachine.State.get_gramb_cluster():
						self.sta = SCJDStateMachine.State.GRAMB_EXG

					elif self.get_state() in SCJDStateMachine.State.get_idmb_cluster():
						self.sta = SCJDStateMachine.State.IDMB_EXG

					elif self.get_state() in SCJDStateMachine.State.get_cb_cluster():
						self.sta = SCJDStateMachine.State.CB_EXG

					else:
						raise RuntimeError(f'{self.get_state()}')

				elif 'ex' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.EX_SPAN)

				elif 'ind' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.IND_SPAN)

				elif 'fld' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.IND_SPAN)

				elif 'cb' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.CB_SPAN)
					self.sta = SCJDStateMachine.State.CB

				elif 'csec' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.CSEC_SPAN)

				elif 'cw' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.CW_SPAN)

				elif 'cv' in attrs_class_values:
					self.push_node(SCJDStateMachine.Node.CV_SPAN)

				else:
					raise RuntimeError(f"SPAN with {attrs_class_values} in class key is not defined")

			else:
				raise RuntimeError(f"SPAN with {attrs_keys} key is not defined")

		elif tag == 'd:prn':
			self.push_node(SCJDStateMachine.Node.D_PRN)

		elif tag == 'd:def':
			self.push_node(SCJDStateMachine.Node.D_DEF)

		elif tag == 'a':
			self.push_node(SCJDStateMachine.Node.A)

		else:
			raise RuntimeError(f"TAG {tag} is not defined")

	def endelement_move(self, tag):
		self.pop_node()

	def startdocument_move(self):
		self.reinit()

	def enddocument_move(self):
		pass

def log(func):
	@functools.wraps(func)
	def wrapper(*args, **kwargs):
		if args[0].debug:
			if func.__name__ == 'startElement':
				print(f'SM: {args[0].sm.stk}, State: {args[0].sm.sta}')
				print(f'Tag: {args[1]}, class: {args[2].getValueByQName("class") if "class" in args[2].getQNames() else None}')
				print(f'{args[0].entry.get_data()}')
				print(f'==========')
			elif func.__name__ == 'endElement':
				print(f'Tag: {args[1]}')
				print(f'==========')
			elif func.__name__ == 'characters':
				print(f'Chars: {args[1]}')
				print(f'==========')
		return func(*args, **kwargs)
	return wrapper

class SCJDController:
	def __init__(self, debug=False):
		self.reinit()
		self.debug = debug

	def reinit(self):
		self.sm = SCJDStateMachine()
		self.entry = SCJDEntry()

	def get_entry(self):
		return self.entry.data

	@log
	def startElement(self, tag, attrs):
		if self.sm.is_numb():
			return

		self.sm.startelement_move(tag, attrs)

		if self.sm.get_node() == SCJDStateMachine.Node.D_ENTRY:
			if re.search("^j_CRJC.*", attrs['id']) is not None:
				self.entry.set_title(attrs['d:title'])
				self.entry.set_id(attrs['id'])
			else:
				self.sm.numb()

		elif self.sm.get_node() == SCJDStateMachine.Node.SEMB_SPAN:
			if self.sm.get_state() in SCJDStateMachine.State.get_gramb_cluster():
				self.entry.push_void_definition()
				if 'ord' in attrs.getQNames():
					self.entry.set_definition_order(attrs['ord'])
				else:
					self.entry.set_definition_order('1')

			elif self.sm.get_state() in SCJDStateMachine.State.get_idmb_cluster():
				self.entry.push_void_phrase_section()

			elif self.sm.get_state() in SCJDStateMachine.State.get_cb_cluster():
				self.entry.push_void_phrase_section()

			else:
				raise RuntimeError(f"Node {self.sm.get_node()} with State {self.sm.get_state()} startElement function is not defined")

		elif self.sm.get_node() == SCJDStateMachine.Node.IDMSEC_SPAN:
			self.entry.push_void_phrase()
			self.entry.set_phrase_idx(attrs['id'])

		elif self.sm.get_node() == SCJDStateMachine.Node.EXG_SPAN:
			if self.sm.get_state() in SCJDStateMachine.State.get_idmb_cluster():
				self.entry.push_void_phrase_section_sentence_example()

			elif self.sm.get_state() in SCJDStateMachine.State.get_gramb_cluster():
				self.entry.push_void_definition_sentence_example()

			elif self.sm.get_state() in SCJDStateMachine.State.get_cb_cluster():
				self.entry.push_void_phrase_section_sentence_example()

			else:
				raise RuntimeError(f"Node {self.sm.get_node()} with State {self.sm.get_state()} startElement function is not defined")

		elif self.sm.get_node() == SCJDStateMachine.Node.CSEC_SPAN:
			self.entry.push_void_phrase()
			self.entry.set_phrase_idx(attrs['id'])

	@log
	def endElement(self, tag):
		if self.sm.is_numb():
			return
		self.sm.endelement_move(tag)

	@log
	def characters(self, chars):
		if self.sm.is_numb():
			return
		def process_chars(chars):
			return chars.strip()

		if self.sm.get_node() == SCJDStateMachine.Node.HW_SPAN:
			self.entry.glue_pronounce(process_chars(chars))

		elif self.sm.get_node() == SCJDStateMachine.Node.HV_SPAN:
			self.entry.push_kanji(process_chars(chars))

		elif self.sm.get_node() == SCJDStateMachine.Node.TRANS_SPAN:
			if self.sm.get_state() == SCJDStateMachine.State.GRAMB_SEMB:
				self.entry.push_definition_chinese_translation(process_chars(chars))

			elif self.sm.get_state() == SCJDStateMachine.State.GRAMB_TRG_AFTER_OUP_LABEL:
				self.entry.push_definition_chinese_translation(process_chars(chars))

			elif self.sm.get_state() == SCJDStateMachine.State.GRAMB_OUP_LABEL:
				self.entry.push_definition_english_translation(process_chars(chars))

			elif self.sm.get_state() == SCJDStateMachine.State.GRAMB_EXG:
				self.entry.glue_definition_sentence_example(None, process_chars(chars))

			elif self.sm.get_state() == SCJDStateMachine.State.IDMB_SEMB:
				self.entry.push_phrase_section_translation(chars.strip())

			elif self.sm.get_state() == SCJDStateMachine.State.IDMB_EXG:
				self.entry.glue_phrase_section_sentence_example(None, chars.strip())

			elif self.sm.get_state() == SCJDStateMachine.State.CB_SEMB:
				self.entry.push_phrase_section_translation(chars.strip())

			elif self.sm.get_state() == SCJDStateMachine.State.CB_EXG:
				self.entry.glue_phrase_section_sentence_example(None, chars.strip())

			else:
				raise RuntimeError(f"Node {self.sm.get_node()} with State {self.sm.get_state()} characters function is not defined")

		elif self.sm.get_node() == SCJDStateMachine.Node.IDM_SPAN:
			if self.sm.get_state() == SCJDStateMachine.State.IDMB_FIRST_IDM:
				self.entry.glue_phrase_first_title(process_chars(chars))

			elif self.sm.get_state() == SCJDStateMachine.State.IDMB_NOT_FIRST_IDM:
				self.entry.push_phrase_title(process_chars(chars))

			else:
				raise RuntimeError(f"Node {self.sm.get_node()} with State {self.sm.get_state()} characters function is not defined")

		elif self.sm.get_node() == SCJDStateMachine.Node.EX_SPAN:
			if self.sm.get_state() == SCJDStateMachine.State.IDMB_EXG:
				self.entry.glue_phrase_section_sentence_example(process_chars(chars), None)

			elif self.sm.get_state() == SCJDStateMachine.State.CB_EXG:
				self.entry.glue_phrase_section_sentence_example(process_chars(chars), None)

			elif self.sm.get_state() == SCJDStateMachine.State.GRAMB_EXG:
				self.entry.glue_definition_sentence_example(process_chars(chars), None)

			else:
				raise RuntimeError(f"Node {self.sm.get_node()} with State {self.sm.get_state()} characters function is not defined")

		elif self.sm.get_node() == SCJDStateMachine.Node.IND_SPAN:
			self.entry.glue_definition_indicator(process_chars(chars))

		elif self.sm.get_node() == SCJDStateMachine.Node.CW_SPAN:
			self.entry.glue_phrase_first_title(process_chars(chars))

		elif self.sm.get_node() == SCJDStateMachine.Node.CV_SPAN:
			self.entry.push_phrase_title(process_chars(chars))

	def startDocument(self):
		self.sm.startdocument_move()
		self.reinit()

	def endDocument(self):
		if self.sm.is_numb():
			return
		self.sm.enddocument_move()
		# print(self.entry.data['title'])


from gevent import monkey
monkey.patch_all()
import gevent
import socket
import re

import dns
import log

LOG = log.get_logger('dns-proxy')

class DNSServer(object):
    def __init__(self, host='0.0.0.0', port=53, nameserver='114.114.114.114'):
        self.sock = None
        self.host = host
        self.port = port
        self.nameserver = nameserver
        self.engine = MatchEngine('./resolv.txt', const={'current': '192.168.199.180'})

    def on_query(self, sip, sport, req):
        def lookup_remote_nameserver(que):
            s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
            if s.sendto(dns.pack(que), (self.nameserver, 53)) == 0:
                LOG.error('failed to query')
                raise Exception('query failed')

            _resp = s.recv(2048)
            LOG.debug("raw response: {}".format(repr(_resp)))
            resp = dns.unpack(_resp)
            return resp
        # end lookup_remote_nameserver
        LOG.debug("raw query: {}".format(repr(req)))
        que = dns.unpack(req)
        LOG.debug("query: {}".format(que))
        host = self.engine.lookup(que.questions[0].qname)

        if not host:
            # reslov from remote nameserver.
            resp = lookup_remote_nameserver(que)
        else:
            qh = que.header
            qq = que.questions[0]
            resp = dns.DNSResponse(
                header=dns.DNSHeader(
                    id=qh.id, qr=1, opcode=qh.opcode,
                    aa=qh.aa, tc=qh.tc, rd=qh.rd, ra=qh.ra,
                    rcode=qh.rcode, qdcount=1, ancount=1, nscount=0, arcount=0),
                questions=que.questions,
                answers=[dns.DNSAnswer(
                    name=qq.qname, type=1, class_=1, ttl=255,
                    rdlength=4, rdata=host)])

        _resp = dns.pack(resp)
        LOG.debug("raw response: {}".format(repr(_resp)))
        LOG.debug("response: {}".format(resp))
        self.sock.sendto(_resp, (sip, sport))

    def serve_forever(self):
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        self.sock.bind((self.host, self.port))
        try:
            while True:
                msg, (ip, port) = self.sock.recvfrom(2048)
                gevent.spawn(self.on_query, ip, port, msg)
        except KeyboardInterrupt:
            LOG.info("exit.")
        finally:
            self.sock.close()

class MatchEngine(object):
    def _read_rules_from_file(self, f):
        _rules = {}
        with open(f) as fr:
            rules = fr.read().split('\n')[:-1]
        for rule in rules:
            domain, host = rule.split()
            if host[0] == '<' and host[-1] == '>':
                host = self._const[host[1:-1]]
            _rules[re.compile(domain)] = host
        return _rules

    def __init__(self, resolv_file, const=None):
        self.resolv_file = resolv_file
        self._const = const if isinstance(const, dict) else {}
        self._rules = self._read_rules_from_file(self.resolv_file)

    def lookup(self, domain):
        for domain_rule, host in self._rules.items():
            if domain_rule.match(domain):
                return host
        return None

    def reload(self):
        self._rules = self._read_rules_from_file(self.resolv_file)

if __name__ == '__main__':
    ds = DNSServer()
    ds.serve_forever()


import os
import numpy as np
import pandas as pd
from PIL import Image
from collections import Counter

import torch
from torch.utils.data import Dataset
from torch.nn.utils.rnn import pad_sequence
import torchvision.transforms as T

import spacy

spacy_eng = spacy.load("en_core_web_sm")

# defining the transform to be applied
transforms = T.Compose([
    T.Resize(226),
    T.RandomCrop(224),
    T.ToTensor(),
    T.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
])

transforms_advanced = T.Compose([
    T.Resize(226),
    T.RandomCrop(224),
    T.ToTensor(),
    T.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
    T.RandomAffine(10),
    T.RandomGrayscale(0.05),
    T.RandomHorizontalFlip(0.05),
    T.RandomVerticalFlip(0.05),
    T.GaussianBlur(5),
    T.RandomErasing(0.05)
])

class Vocabulary:
    def __init__(self, freq_threshold):
        # setting the pre-reserved tokens int to string tokens
        self.pad_idx = 0
        self.sos_idx = 1
        self.eos_idx = 2
        self.unk_idx = 3
        self.pad = "<PAD>"
        self.sos = "<SOS>"
        self.eos = "<EOS>"
        self.unk = "<UNK>"
        self.itos = {self.pad_idx: self.pad,
                     self.sos_idx: self.sos,
                     self.eos_idx: self.eos,
                     self.unk_idx: self.unk}

        # string to int tokens
        # its reverse dict self.itos
        self.stoi = {v: k for k, v in self.itos.items()}

        self.freq_threshold = freq_threshold

    def __len__(self):
        return len(self.itos)

    def size(self):
        return len(self.itos)

    @staticmethod
    def tokenize(text):
        return [token.text.lower() for token in spacy_eng.tokenizer(text)]

    def build_vocab(self, sentence_list):
        frequencies = Counter()
        idx = 4

        for sentence in sentence_list:
            for word in self.tokenize(sentence):
                frequencies[word] += 1

                # add the word to the vocab if it reaches minum frequecy threshold
                if frequencies[word] == self.freq_threshold:
                    self.stoi[word] = idx
                    self.itos[idx] = word
                    idx += 1

    def numericalize(self, text):
        """ For each word in the text corresponding index token for that word form the vocab built as list """
        tokenized_text = self.tokenize(text)
        return [self.stoi[token] if token in self.stoi else self.stoi["<UNK>"] for token in tokenized_text]

class FlickrDataset(Dataset):
    """
    FlickrDataset
    """

    def __init__(self,
                 root_dir,
                 captions_file,
                 transform=None,
                 freq_threshold=5,
                 vocab=None,
                 data_limit=None,
                 do_augmentation=False,
                 augmentation_probability=0.2):
        self.root_dir = root_dir
        self.df = pd.read_csv(captions_file)
        self.transform = transform
        self.random = np.random.RandomState()
        self.do_augmentation = do_augmentation
        self.augmentation_probability = augmentation_probability

        # Get image and caption colum from the dataframe
        self.imgs = self.df["image"]
        self.captions = self.df["caption"]
        # If needed truncating the data for faster running
        if data_limit is not None:
            self.imgs = self.imgs[:data_limit]
            self.captions = self.captions[:data_limit]

        # Initialize vocabulary and build vocab
        if vocab is None:
            self.vocab = Vocabulary(freq_threshold)
            self.vocab.build_vocab(self.captions.tolist())
        else:
            self.vocab = vocab

    def __len__(self):
        # return len(self.df)
        return self.imgs.shape[0]

    def __getitem__(self, idx):
        caption = self.captions[idx]
        img_name = self.imgs[idx]
        img_location = os.path.join(self.root_dir, img_name)
        img_pil = Image.open(img_location).convert("RGB")

        # do some random augmentations
        if not self.do_augmentation:
            img = transforms(img_pil)
        else:
            img = transforms_advanced(img_pil)

        # numericalize the caption text
        caption_vec = []
        caption_vec += [self.vocab.stoi["<SOS>"]]
        caption_vec += self.vocab.numericalize(caption)
        caption_vec += [self.vocab.stoi["<EOS>"]]

        return img, torch.tensor(caption_vec)

class CapsCollate:
    """
    Collate to apply the padding to the captions with dataloader
    """

    def __init__(self, vocab, batch_first=False, max_len=0):
        self.pad_idx = vocab.pad_idx
        self.eos_idx = vocab.eos_idx
        self.batch_first = batch_first
        self._max_len = max_len

    def __call__(self, batch):
        imgs = [item[0].unsqueeze(0) for item in batch]
        imgs = torch.cat(imgs, dim=0)

        targets = [item[1] for item in batch]
        targets = pad_sequence(targets, batch_first=self.batch_first, padding_value=self.pad_idx)
        if self._max_len > 0:
            if targets.shape[1] >= self._max_len:
                targets = targets[:, :self._max_len]
            else:
                pad_tensor = torch.ones(size=(targets.shape[0], self._max_len - targets.shape[1]),
                                        dtype=torch.long) * self.pad_idx
                targets = torch.cat([targets, pad_tensor], dim=1)

        targets[:, -1] = targets[:, -1].where(targets[:, -1] == self.pad_idx, torch.tensor(self.eos_idx))
        return imgs, targets


import numpy as np
import cv2
import copy
from time import sleep
import datetime
# from progress.bar import Bar

def Rodar(cam):
    capture = cv2.VideoCapture(cam)
    background_subtractor = cv2.bgsegm.createBackgroundSubtractorMOG()
    #length = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))

    # bar = Bar('Processing Frames', max=length)

    first_iteration_indicator = 1
    while True:
        sleep(1/60)
        ret, frame = capture.read()

        # If first frame
        if first_iteration_indicator == 1:

            first_frame = copy.deepcopy(frame)
            height, width = frame.shape[:2]
            accum_image = np.zeros((height, width), np.uint8)
            first_iteration_indicator = 0
        else:

            filter = background_subtractor.apply(frame)  # remove the background

            threshold = 2
            maxValue = 2
            ret, th1 = cv2.threshold(filter, threshold, maxValue, cv2.THRESH_BINARY)

            # add to the accumulated image
            accum_image = cv2.add(accum_image, th1)

            color_image_video = cv2.applyColorMap(accum_image, cv2.COLORMAP_SUMMER)

            video_frame = cv2.addWeighted(frame, 0.7, color_image_video, 0.7, 0)
            color_image = cv2.applyColorMap(accum_image, cv2.COLORMAP_HOT)
            result_overlay = cv2.addWeighted(frame, 0.7, color_image, 0.7, 0)

            relatorio = datetime.datetime.now();
            data = "{:02d}-{:02d}-{:02d}".format(relatorio.day, relatorio.month, relatorio.replace(year=20).year)
            if relatorio.hour == 22 and relatorio.minute == 29 and relatorio.second == 1:
                cv2.imwrite("static/reports/report_" + data + ".jpg", result_overlay)

            #cv2.imshow("Video Original" , result_overlay)
            ret, jpeg = cv2.imencode('.jpg', result_overlay)
            send_frame = jpeg.tobytes()
            yield (b'--frame\r\n'
                   b'Content-Type: image/jpeg\r\n\r\n' + send_frame + b'\r\n\r\n')
        # bar.next()

    # bar.finish()

    # save the final heatmap
    # cv2.imwrite('diff-overlay.jpg', result_overlay)

    # cleanup
    #capture.release()
    #cv2.destroyAllWindows()


import cv2
import numpy as np

img = cv2.imread('images/saitama.jpg')
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)  # convert it to hsv

width = img.shape[1]
height = img.shape[0]
channel = img.shape[2]

increase_v = 40
decrease_s = 10
step = 2

# bien doi hinh anh
print("chon huong di cua animation: ")
print("1.Left -> Right")
print("2.Right -> Left")
print("3.Down")
print("4.Up")
flag = input()

if flag == '3':
    # huong di xuong
    for y in range(1, height, step):
        h, s, v = cv2.split(hsv[0: y, :])

        v = np.where(v <= 255 - increase_v, v + increase_v, 255)
        s = np.where(s >= 0 + decrease_s, s - decrease_s, 0)

        hsv[0: y, :] = cv2.merge((h, s, v))

        img = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
        cv2.imshow('animation', img)
        cv2.waitKey(1)
elif flag == '1':
    # huong sang phai
    for x in range(1, width, step):
        h, s, v = cv2.split(hsv[:, 0: x])

        v = np.where(v <= 255 - increase_v, v + increase_v, 255)
        s = np.where(s >= 0 + decrease_s, s - decrease_s, 0)

        hsv[:, 0:x] = cv2.merge((h, s, v))

        img = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
        cv2.imshow('animation', img)
        cv2.waitKey(1)
elif flag == '2':
    # huong sang trai
    for x in range(width - 2, 0, -step):
        h, s, v = cv2.split(hsv[:, x: -1])

        v = np.where(v <= 255 - increase_v, v + increase_v, 255)
        s = np.where(s >= 0 + decrease_s, s - decrease_s, 0)

        hsv[:, x:-1] = cv2.merge((h, s, v))

        img = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
        cv2.imshow('animation', img)
        cv2.waitKey(1)
elif flag == '4':
    # huong len tren
    for y in range(height - 2, 0, -step):
        h, s, v = cv2.split(hsv[y: -1, :])

        v = np.where(v <= 255 - increase_v, v + increase_v, 255)
        s = np.where(s >= 0 + decrease_s, s - decrease_s, 0)

        hsv[y: -1, :] = cv2.merge((h, s, v))

        img = cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR)
        cv2.imshow('animation', img)
        cv2.waitKey(1)
else:
    print('nhap cac gia tri tu 1 den 4')
cv2.destroyAllWindows()


import pandas as pd
import numpy as np
import json
import os
from pydub import AudioSegment
from scipy.io.wavfile import read
import stft
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import normalize
from sklearn.decomposition import PCA
import random
import signal
import cPickle as pickle
from functools import partial
import multiprocessing as mp
import time
from multiprocessing.pool import Pool

audio_path = '/media/jon/external_data/audio/'
# audio_path = "data/target_species/"
species_list = ["Poecile_atricapillus",
                "Poecile_rufescens",
                "Regulus_calendula",
                "Regulus_satrapa"]

def make_dirs(audio_path):
    '''
    Read species names from json and create wav file directory for each species
    '''
    f = []
    for (_, _, filename) in os.walk(audio_path + "json_files"):
        f.extend(filename)
        for name in f:
            os.makedirs(audio_path + name + '_wav')

def dir_list(audio_path):
    '''
    INPUT:
        path to data file directories
    OUTPUT:
        For each directory containing mp3 files, generate a new directory
        to recieve wav files.  Return a list of tuples containing mp3 and wav
        directory paths.
    '''
    directory_list = []
    for (_, _, filename) in os.walk(audio_path + "json_files"):
        for name in filename:
            input_directory = audio_path + name
            output_directory = input_directory + "_wav"
            directory_list.append((input_directory, output_directory))
    return directory_list

def make_file_list(directory_list):
    '''
    INPUT:
        list of tuples containing input, output directories
    OUTPUT:
        list of tuples containing input, output file names
    '''
    file_list = []
    for directory in directory_list:
        for (_, _, filenames) in os.walk(directory[0]):
            for file_id in filenames:
                mp3_file = (directory[0] + "/" + file_id)
                wav_file = (directory[1] + "/" + file_id[:-3] + "wav")
                file_list.append((mp3_file, wav_file))
    return file_list

def make_wav_list(directory_list):
    '''
    INPUT:
        list of tuples containing wav file directories
    OUTPUT:
        list of tuples containing path and file_id for all wav files
    '''
    wav_list = []
    for directory in directory_list:
        for (_, _, filenames) in os.walk(directory[1]):
            for file_id in filenames:
                wav_file = (directory[1] + "/" + file_id)
                wav_list.append((wav_file, file_id))
    return wav_list

def file_list(path, species):
    '''
    Create a list of files for further processing
    '''
    file_list = []
    for sp in species:
        for (_, _, filenames) in os.walk(path + sp + "_wav/"):
            for f in filenames:
                file_list.append(path + sp + "_wav/" + f)
    return file_list

def make_mono(file_list):
    '''
    overwrite wav files as mono - other functions will have errors with stereo files
    '''
    for f in file_list:
        sound = AudioSegment.from_wav(f)
        sound = sound.set_channels(1)
        sound.export(f, format="wav")

def make_spec(file_list):
    '''
    INPUT:
        list of wav file - files will be converted to mono in function
    OUTPUT:
        dictionary with filename as key, spectrogram as value
    '''
    spectrograms = {}
    for f in file_list:
        sound = AudioSegment.from_wav(f)
        sound = sound.set_channels(1)
        sound.export("temp", format="wav")
        a = read("temp")
        # arr = np.array(a[1], dtype=float)  already np array - don't need to convert
        spec = stft.spectrogram(a[1])
        spectrograms[f] = spec
    return spectrograms

def norm_spec(spectrograms):
    '''
    INPUT:
        dict of file name: spectrogram
    OUTPUT:
        dict of file name: l2 normalized spectrogram
    '''
    norm = {}
    for k in spectrograms.keys():
        norm[k] = normalize(spectrograms[k], norm="l2")
    return norm

def whiten(normalized):
    '''
    INPUT:
        dict of file name: spectrogram
    OUTPUT:
        dict of file name: pca whitened spectrogram
    '''
    whitened = {}
    pca = PCA(n_components=40, copy=False, whiten=True)
    for k in normalized.keys():
        whitened[k] = pca.fit_transform(normalized[k])

def random_sample(species_files, n=10):
    '''
    INPUT:
        a dict of species, file list pairs
    OUTPUT:
        a randomly selected list of n files from each species
    '''
    subset = []
    for k, v in species_files:
        subset.extend([v[i] for i in random.sample(xrange(len(v)))])
    return subset


import numpy as np
from scipy import ndimage

import matplotlib.pyplot as plt
from matplotlib.colors import LogNorm as LogNorm

def compare_fields(delta_F, delta_DM, R_sm, pc_meta):
    # from Metin 2019
    '''
    A function for comparing the fields of delta_F and delta_DM with hist2d.
    '''
    fig = plt.figure(figsize = (10, 7))

    bins_f = np.arange(-0.6,0.5, 0.01)
    bins_dm = np.arange(0., 8., 0.04)

    hist2d, edges_dm, edges_f = np.histogram2d(np.ndarray.flatten(delta_DM.field_data), np.ndarray.flatten(delta_F.field_data),
                                bins=[bins_dm, bins_f],density=True)

    X, Y = np.meshgrid(edges_dm,  edges_f, indexing='ij')
    plt.pcolormesh(X,Y, hist2d, cmap='Greys',
                  norm=LogNorm(vmin=2e-3, vmax=100.))

    cbar = plt.colorbar()
    cbar.set_label('normalized density')

    XCon, YCon = np.meshgrid(edges_dm[0:-1]+(edges_dm[1]-edges_dm[0])/2 ,
                             edges_f[0:-1]+(edges_f[1]-edges_f[1])/2 ,
                             indexing='ij')
    # plt.contour(XCon,YCon, hist2d, levels = 3)

    plt.xlabel('$\\delta_{DM}$')
    plt.ylabel('$\\delta_{F}$')

    plt.title('$\\delta_{DM} - \\delta_{F}$ of '
              + '{}, \nRA: {}, DE: {} '.format(pc_meta['Name'], pc_meta['RA'], pc_meta['DE'])
              + '$R_{sm}$ = ' + str(R_sm))
    return fig

def compare_fields_general(field_1, field_2, extent, ncell_1, ncell_2, vmin = 2e-3, vmax = 100, countour = True):
    # from Metin 2019
    '''
    extent = [x_1, y_1, x_2, y_2]
    '''
    fig = plt.figure(figsize = (10, 10))

    x_1, y_1, x_2, y_2 = extent

    bins_1 = np.linspace(x_1, x_2, ncell_1)
    bins_2 = np.linspace(y_1, y_2, ncell_2)

    hist2d, edges_1, edges_2 = np.histogram2d(np.ndarray.flatten(field_1.field_data), np.ndarray.flatten(field_2.field_data),
                                bins=[bins_1, bins_2],density=True)

    X, Y = np.meshgrid(edges_1,  edges_2, indexing='ij')
    plt.pcolormesh(X,Y, hist2d, cmap='Greys',
                  norm=LogNorm(vmin=vmin, vmax=vmax))

    cbar = plt.colorbar()
    cbar.set_label('normalized density')

    XCon, YCon = np.meshgrid(edges_1[0:-1]+(edges_1[1]-edges_1[0])/2 ,
                             edges_2[0:-1]+(edges_2[1]-edges_2[1])/2 ,
                             indexing='ij')

    if countour:
        plt.contour(XCon,YCon, hist2d, levels = 5)
    return fig

def sphere_kernel(radius, normalize = True):
    size = int(radius)*2+1
    grid = np.array(np.meshgrid(np.arange(size), np.arange(size), np.arange(size)))
    kernel = ((grid - int(radius))**2).sum(axis=0) < int(radius)**2
    if normalize:
        kernel = kernel / kernel.sum()
    return kernel


import sys
import time
from datetime import datetime
from textwrap import dedent

import requests
import telegram
from environs import Env
from loguru import logger
from telegram import ParseMode

def send_telegram_message(chat_id: int, bot: telegram.Bot, telegram_message: str) -> None:
    bot.send_message(chat_id=chat_id, text=telegram_message, parse_mode=ParseMode.HTML)
    logger.debug(f'Сообщение {telegram_message} отправлено в чат Телеграмма')

def generate_telegram_message(response_json: dict) -> str:
    emojy = {
        'ufo': '&#128126',
        'true': '&#9989',
        'false': '&#10060',
    }
    new_attempts = response_json['new_attempts'][0]
    is_negative = f'{emojy["false"]}Работа не выполнена{emojy["false"]}' \
        if new_attempts['is_negative'] is True \
        else f'{emojy["true"]}Работа сдана{emojy["true"]}'

    telegram_message = dedent(f"""
        {emojy["ufo"]}<b>{new_attempts['lesson_title']}</b>{emojy["ufo"]}

        {is_negative}
        {new_attempts['lesson_url']}
        """)

    return telegram_message

def get_new_checks(devman_api_token: str, bot: telegram.Bot, chat_id: int, timeout: int = 300) -> None:
    timestamp = datetime.now().timestamp()

    headers = {'Authorization': f'Token {devman_api_token}'}
    params = {'timestamp': timestamp}

    reconnect_time = 0.1

    while True:
        url = f'https://dvmn.org/api/long_polling/'
        try:
            response = requests.get(url, headers=headers, params=params, timeout=timeout)
            response.raise_for_status()

            checked_tasks = response.json()
            logger.debug(checked_tasks)

            if checked_tasks.get('status') == 'found':
                telegram_message = generate_telegram_message(checked_tasks)
                send_telegram_message(chat_id, bot, telegram_message)

            timestamp = checked_tasks.get('timestamp_to_request') or checked_tasks.get('last_attempt_timestamp')
            params = {'timestamp': timestamp}
            reconnect_time = 0.1

        except requests.exceptions.ReadTimeout as error:
            logger.warning(f'Таймаут запроса отработал раньше чем сервер ответил: {error}. Делаем повторный запрос.')
            params = {'timestamp': timestamp}
            continue

        except requests.exceptions.ConnectionError:
            time.sleep(reconnect_time)
            reconnect_time *= 2
            logger.warning(f'Потеря соединения. Повторный запрос через {reconnect_time} секунд')
            continue

        except requests.exceptions.HTTPError as http_error:
            time.sleep(reconnect_time)
            reconnect_time *= 2
            logger.warning(f'Запрос вернул ответ {http_error.response}. Повторное подключение через {reconnect_time}')
            continue

def main():
    env = Env()
    env.read_env()

    devman_api_token = env.str('DEVMAN_TOKEN_API')
    telegram_api_key = env.str('TELEGRAM_API_KEY')
    telegram_chat_id = env.int('TELEGRAM_CHAT_ID')

    bot = telegram.Bot(token=telegram_api_key)

    logger_level = 'DEBUG' if env.bool('DEBUG_MODE', False) else 'INFO'
    logger.level(logger_level)
    logger.add(sys.stdout, format='{time} {level} {message}')

    send_telegram_message(telegram_chat_id, bot, telegram_message='Бот запущен')
    while True:
        try:
            get_new_checks(devman_api_token, bot, telegram_chat_id)
        except Exception as exception:
            telegram_message = dedent(
                f"""
                Бот упал с ошибкой:
                {exception}""",
            )
            send_telegram_message(telegram_chat_id, bot, telegram_message)

if __name__ == '__main__':
    main()


import argparse
import sys
import logging
from mutagene.profiles.profile import calc_profile

logger = logging.getLogger(__name__)
genome_error_message = """requires genome name argument -g hg19, hg38, mm10, see http://hgdownload.cse.ucsc.edu/downloads.html for more
                          Use mutagene fetch to download genome assemblies"""

class ProfileMenu(object):
    def __init__(self, parser):
        parser.add_argument("--infile", "-i", nargs='*', help="Input file format", type=argparse.FileType('r'))
        parser.add_argument('--outfile', "-o", nargs='?', type=argparse.FileType('w'), default=sys.stdout,
                            help="Name of output file, will be generated in TSV format")
        parser.add_argument('--genome', "-g", help="Location of genome assembly file", type=str)
        parser.add_argument('--input-format', "-f", help="Input format: auto, MAF, VCF", type=str, default='auto')

        # for backwards compatibility with 0.8.X add a hidden action that would just take anything as a valid input
        parser.add_argument('action', nargs="?", metavar="")

    def callback(self, args):
        # print('ProfileMenu', args.action)
        self.calculate(args)

    def calculate(self, args):
        # print("Calculating...")
        if not args.infile:
            logger.warning("Provide input file in VCF or MAF format (-i) and a corresponding genome assembly (-g)")
            return
        if not args.genome:
            logger.warning(genome_error_message)
            return
        calc_profile(args.infile, args.outfile, args.genome, args.input_format)


from sqlalchemy import Column
from sqlalchemy import Integer, String
from sqlalchemy.orm import relationship

from app.models.base import Base

class User(Base):
    __tablename__ = 'users'

    id = Column(Integer, primary_key=True)
    username = Column(String, nullable=False)
    email = Column(String, nullable=False, unique=True, index=True)

    lists = relationship('List', back_populates='owner', cascade='all, delete-orphan')

    def get_api_repr(self, include_email=False):
        api_repr = {
            "id": self.id,
            "username": self.username,
        }
        if include_email:
            api_repr['email'] = self.email
        return api_repr


import os
import tempfile
from shutil import rmtree

import pytest
import responses
from faker import Faker
from flask import Response, Flask
from flask.testing import FlaskClient

from statuspage2slack.statuspage_constants import ComponentStatus, \
    IncidentStatus, IncidentImpact

fake = Faker()

STATUSPAGE_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ'

test_file_path = os.path.realpath(__file__)
test_file_folder = os.path.dirname(test_file_path)

@pytest.fixture
def component_update_request(old_component_status, new_component_status):
    creation_datetime = fake.past_datetime()
    update_datetime = fake.past_datetime(start_date=creation_datetime)
    component_id = fake.bothify(text='????????????')
    update_id = fake.bothify(text='????????????')
    return {
        "component_update": {
            "created_at": update_datetime.strftime(STATUSPAGE_DATETIME_FORMAT),
            "new_status": new_component_status.value,
            "old_status": old_component_status.value,
            "id": update_id,
            "component_id": component_id
        },
        "component": {
            "created_at": creation_datetime.strftime(
                STATUSPAGE_DATETIME_FORMAT),
            "id": component_id,
            "name": "Some Component",
            "status": new_component_status.value
        }
    }

@pytest.fixture
def incident_update_request(incident_update, incident_impact, incident_status):
    creation_datetime = fake.past_datetime()
    monitoring_datetime = fake.past_datetime(start_date=creation_datetime)
    resolved_datetime = fake.past_datetime(start_date=creation_datetime)
    update_datetime = fake.past_datetime(start_date=creation_datetime)

    name = fake.sentence(nb_words=6, variable_nb_words=True,
                         ext_word_list=None)

    return {
        "incident": {
            "backfilled": False,
            "created_at":
                creation_datetime.strftime(STATUSPAGE_DATETIME_FORMAT),
            "impact": incident_impact.value,
            "impact_override": None,
            "monitoring_at":
                monitoring_datetime.strftime(STATUSPAGE_DATETIME_FORMAT),
            "resolved_at":
                resolved_datetime.strftime(STATUSPAGE_DATETIME_FORMAT),
            "shortlink": fake.url(),
            "status": incident_status.value,
            "updated_at": update_datetime.strftime(STATUSPAGE_DATETIME_FORMAT),
            "name": name,
            "incident_updates": [incident_update]
        }
    }

@pytest.fixture()
def incident_update(incident_status):
    body = fake.paragraph()
    creation_datetime = fake.past_datetime()
    display_datetime = fake.past_datetime(start_date=creation_datetime)
    update_datetime = fake.past_datetime(start_date=creation_datetime)

    return {
        "body": body,
        "created_at": creation_datetime.strftime(STATUSPAGE_DATETIME_FORMAT),
        "display_at": display_datetime.strftime(STATUSPAGE_DATETIME_FORMAT),
        "status": incident_status.value,
        "updated_at": update_datetime.strftime(STATUSPAGE_DATETIME_FORMAT),
    }

@pytest.mark.parametrize("old_component_status", ComponentStatus)
@pytest.mark.parametrize("new_component_status", ComponentStatus)
def test_component_update(flask_client: FlaskClient,
                          component_update_request, used_templates,
                          request_mocker: responses.RequestsMock):
    response: Response = flask_client.post('/', json=component_update_request)

    assert 200 <= response.status_code < 300

    assert len(used_templates) == 1
    (template, context) = used_templates.pop()
    assert template.name == 'component_update.json'
    component_update = component_update_request['component_update']
    component = component_update_request['component']
    assert context['component_update'] == component_update
    assert context['component'] == component

@pytest.mark.parametrize("incident_status", IncidentStatus)
@pytest.mark.parametrize("incident_impact", IncidentImpact)
def test_incident_update(flask_client: FlaskClient, incident_update_request,
                         used_templates):
                         #request_mocker: responses.RequestsMock):
    response: Response = flask_client.post('/', json=incident_update_request)

    assert 200 <= response.status_code < 300

    assert len(used_templates) == 1
    (template, context) = used_templates.pop()
    assert template.name == 'incident_update.json'
    assert context['incident'] == incident_update_request['incident']

def test_invalid_request(flask_client: FlaskClient):
    response: Response = flask_client.post('/', data='dummy')
    assert 400 <= response.status_code < 500

@pytest.mark.parametrize("old_component_status",
                         [ComponentStatus.DEGRADED_PERFORMANCE])
@pytest.mark.parametrize("new_component_status", [ComponentStatus.OPERATIONAL])
@pytest.mark.parametrize("incident_status", [IncidentStatus.MONITORING])
@pytest.mark.parametrize("incident_impact", [IncidentImpact.CRITICAL])
@pytest.mark.parametrize("flag", ['COMPONENT_MESSAGES_ENABLED',
                                  'INCIDENT_MESSAGES_ENABLED'])
def test_false_enabled_flags(flask_app: Flask, flask_client: FlaskClient,
                             component_update_request, incident_update_request,
                             used_templates, flag):
    flask_app.config.update({
        flag: False
    })
    if flag == 'INCIDENT_MESSAGES_ENABLED':
        response: Response = flask_client.post('/',
                                               json=incident_update_request)
    elif flag == 'COMPONENT_MESSAGES_ENABLED':
        response: Response = flask_client.post('/',
                                               json=component_update_request)
    else:
        assert False, "Unexpected flag value"

    assert 200 <= response.status_code < 300
    assert len(used_templates) == 0

@pytest.mark.parametrize("incident_status", [IncidentStatus.MONITORING])
@pytest.mark.parametrize("incident_impact", [IncidentImpact.CRITICAL])
@pytest.mark.parametrize("env_dict", [
    {'TEMPLATE_FOLDER': test_file_folder + '/templates'}
])
def test_change_template_folder(change_env, flask_client: FlaskClient,
                                incident_update_request, used_templates,
                                request_mocker: responses.RequestsMock,
                                env_dict):
    template_name = 'incident_update.json'
    response: Response = flask_client.post('/', json=incident_update_request)
    assert 200 <= response.status_code < 300

    assert len(used_templates) == 1
    (template, context) = used_templates.pop()
    assert template.name == template_name
    assert os.path.realpath(template.filename) == os.path.realpath(
        env_dict['TEMPLATE_FOLDER'] + '/' + template_name)

def test_copy_templates(flask_app: Flask):
    runner = flask_app.test_cli_runner()
    folder = tempfile.gettempdir() + '/templates/'
    rmtree(folder, ignore_errors=True)
    result = runner.invoke(args=['webhook', 'copy-templates', folder])


import pickle
from flask import Flask, request, jsonify
import numpy as np

# Load model and dv
with open("dv.pkl", "rb") as f_in:
    dv = pickle.load(f_in)

with open("rf_model.pkl", "rb") as f_in:
    model = pickle.load(f_in)

# instantiate
app = Flask('atx_housing_prediction')

# set path: /predict
@app.route('/predict', methods=['POST']) # HTTP Request: Post
def predict():
    # Get data
    data = request.get_json()

    # Extract features
    X = dv.transform([data])

    # Make prediction
    y_pred = np.expm1(model.predict(X)[0]) # predict and convert back from log1p

    return jsonify({'price': float(y_pred)}) # cast

if __name__ == '__main__':
    app.run(debug=True, host='0.0.0.0', port=9696)


from typing import List, Optional

import filters as f

from iota import Address
from iota.commands import FilterCommand, RequestFilter
from iota.commands.core.find_transactions import FindTransactionsCommand
from iota.commands.core.were_addresses_spent_from import \
    WereAddressesSpentFromCommand
from iota.crypto.addresses import AddressGenerator
from iota.crypto.types import Seed
from iota.filters import SecurityLevel, Trytes
import asyncio

__all__ = [
    'GetNewAddressesCommand',
]

class GetNewAddressesCommand(FilterCommand):
    """
    Executes ``getNewAddresses`` extended API command.

    See :py:meth:`iota.api.Iota.get_new_addresses` for more info.
    """
    command = 'getNewAddresses'

    def get_request_filter(self):
        return GetNewAddressesRequestFilter()

    def get_response_filter(self):
        pass

    async def _execute(self, request: dict) -> dict:
        checksum: bool = request['checksum']
        count: Optional[int] = request['count']
        index: int = request['index']
        security_level: int = request['securityLevel']
        seed: Seed = request['seed']

        return {
            'addresses':
                await self._find_addresses(
                    seed,
                    index,
                    count,
                    security_level,
                    checksum,
                ),
        }

    async def _find_addresses(
            self,
            seed: Seed,
            index: int,
            count: Optional[int],
            security_level: int,
            checksum: bool
    ) -> List[Address]:
        """
        Find addresses matching the command parameters.
        """
        generator = AddressGenerator(seed, security_level, checksum)

        if count is None:
            # Connect to Tangle and find the first unused address.
            for addy in generator.create_iterator(start=index):
                # We use addy.address here because the commands do
                # not work on an address with a checksum
                # Execute two checks concurrently
                responses = await asyncio.gather(
                    WereAddressesSpentFromCommand(self.adapter)(
                        addresses=[addy.address],
                    ),
                    FindTransactionsCommand(self.adapter)(
                        addresses=[addy.address],
                    ),
                )
                # responses[0] -> was it spent from?
                # responses[1] -> any transaction found?
                if responses[0]['states'][0] or responses[1].get('hashes'):
                    continue

                return [addy]

        return generator.get_addresses(start=index, count=count)

class GetNewAddressesRequestFilter(RequestFilter):
    def __init__(self) -> None:
        super(GetNewAddressesRequestFilter, self).__init__(
            {
                # Everything except ``seed`` is optional.
                'checksum': f.Type(bool) | f.Optional(default=False),
                'count': f.Type(int) | f.Min(1),
                'index': f.Type(int) | f.Min(0) | f.Optional(default=0),
                'securityLevel': SecurityLevel,

                'seed': f.Required | Trytes(Seed),
            },

            allow_missing_keys={
                'checksum',
                'count',
                'index',
                'securityLevel',
            },
        )