'''
@author:  Michael Trigoboff
@contact: mtrigoboff@comcast.net
@contact: http://spot.pcc.edu/~mtrigobo
'''

import configparser
import os.path
import sys
from tkinter import BooleanVar, StringVar, ttk
import tkinter
from tkinter.filedialog import askopenfilename

class CheckBox:
	def __init__(self, label, underlineIndex, frame, state):
		self.label =		label
		self.variable =		BooleanVar()
		self.checkBtn = 	ttk.Checkbutton(frame, text = label, variable = self.variable,
											command = setCreateBtnState, underline = underlineIndex)
		checkBoxShortcuts[label[underlineIndex].lower()] = self
		self.variable.set(state)

class BlockSpec:
	def __init__(self, name, underline):
		self.name =				name
		self.underline =		underline		# which checkbox char to underline in GUI

blockSpecs = (
	(BlockSpec('Songs',				0)),		\
	(BlockSpec('Patterns',			0)),		\
	(BlockSpec('Masters',			0)),		\
	(BlockSpec('Performances',		4)),		\
	(BlockSpec('Voices',			0)),		\
	(BlockSpec('Pattern Chains',	8)),		\
	(BlockSpec('User Arpeggios',	6)),		\
	(BlockSpec('Mixing Voices',		2)),		\
	(BlockSpec('Song Mixings',		3)),		\
	(BlockSpec('Pattern Mixings',	4)),		\
	(BlockSpec('Waveforms',			0)),		\
	(BlockSpec('Sample Voices',		4)),		\
)

# global variables
VERSION =				'1.2'
checkBoxes =			[]
checkBoxShortcuts = 	{}

# global constants for app state .ini file
STATE_FILE_NAME =		'motif2text.ini'
STATE_SECTION_NAME =	'APP_STATE'
WN_POSN_KEY = 			'WindowPosition'
MOTIF_FILE_DIR_KEY =	'MotifFileDir'
MOTIF_FILE_NAME_KEY =	'MotifFileName'
CHECK_BOX_STATES = 		'CheckBoxStates'

def launchFile(filePath):			# launch file with default app for that file type
	# need to enclose filePath in "..." so that space chars don't break path
	filePath = '\"' + filePath + '\"'
	# open file with default app
	if sys.platform == 'win32' or sys.platform == 'win64':		# windows
		os.startfile(filePath)
	else:														# mac, linux
		os.system("open " + filePath)
	#os.system("notepad.exe \"" + textFilePath + "\"")			#open .txt file with notepad

def setCreateBtnState():
	if len(motifFileDir) == 0:
		createTextBtn['state'] = 'disabled'
	else:
		atLeastOneChecked = False
		for checkBox in checkBoxes:
			if checkBox.variable.get():
				atLeastOneChecked = True
		if atLeastOneChecked:
			createTextBtn['state'] = 'enabled'
		else:
			createTextBtn['state'] = 'disabled'

def allFn():
	for checkBox in checkBoxes:
		checkBox.variable.set(True)
	setCreateBtnState()

def noneFn():
	for checkBox in checkBoxes:
		checkBox.variable.set(False)
	setCreateBtnState()

def selectFileFn():
	global motifFileDir, motifFileName

	motifFilePath = askopenfilename(initialdir = motifFileDir)
	if motifFilePath == '':								# user hit Cancel
		return
	# can't use realpath until here because it turns '' into something else, disabling Cancel
	motifFilePath = os.path.realpath(motifFilePath)
		# use realpath so that path separators are platform-specific, e.g. Windows '\'
	motifFileDir = os.path.dirname(motifFilePath)
	motifFileName = os.path.basename(motifFilePath)
	fileNameEntryVar.set(motifFileName)
	setCreateBtnState()

def createTextFn():
	global motifFileDir, motifFileName
	selectedItems = []
	for checkBox in checkBoxes:
		if checkBox.variable.get():
			selectedItems.append(checkBox.label)
	if len(selectedItems) == 0:
		return
	realStdOut = sys.stdout
	motifFilePath = os.path.join(motifFileDir, motifFileName)
	textFilePath = motifFilePath + '.txt'
	try:
		textFile = open(textFilePath, 'w')
		sys.stdout = textFile
		# GUI only version: don't analyze actual file
		# printMotifFile(motifFilePath, selectedItems)
		print(selectedItems)
		textFile.close()
		launchFile(textFilePath)
	except Exception as _:
		fileNameEntryVar.set('problem reading \'%s\'' % motifFileName)
		motifFileDir = ''
		motifFileName = ''
		setCreateBtnState()
		if not textFile.closed:
			textFile.close()
		os.remove(textFilePath)
	sys.stdout = realStdOut

def helpFn():
	helpFileName = 'motif2textHelp.pdf'
	if os.name == 'mac':
		# this section adapted from code written by Vikas Sharma
		if getattr(sys, 'frozen', False):			# application is frozen
			datadir = os.path.join(os.path.dirname(sys.executable), '..', 'Resources')
		else:										# application is not frozen
			datadir = os.path.join(os.path.dirname(__file__), '..', 'Resources')
		helpFilePath = os.path.join(datadir, helpFileName)
	else:										# Windows, Linux, etc
		helpFilePath = helpFileName
	launchFile(helpFilePath)

def checkBoxKeyboardShortcutFn(ch):
	try:
		checkBox = checkBoxShortcuts[ch]
		checkBox.variable.set(not checkBox.variable.get())
		setCreateBtnState()
	except KeyError:
		pass

def keyPressFn(kpEvent):
	try:
		fn = {'a' 		:	'allFn()',
			  'c'		:	'checkBoxKeyboardShortcutFn(kpEvent.keysym)',
			  'e'		:	'checkBoxKeyboardShortcutFn(kpEvent.keysym)',
			  'f'		:	'selectFileFn()',
			  'g'		:	'checkBoxKeyboardShortcutFn(kpEvent.keysym)',
			  'l'		:	'checkBoxKeyboardShortcutFn(kpEvent.keysym)',
			  'm'		:	'checkBoxKeyboardShortcutFn(kpEvent.keysym)',
			  'h'		:	'helpFn()',
			  'n' 		:	'noneFn()',
			  'o'		:	'checkBoxKeyboardShortcutFn(kpEvent.keysym)',
			  'p'		:	'checkBoxKeyboardShortcutFn(kpEvent.keysym)',
			  'q' 		:	'windowCloseRequested()',
			  'r'		:	'checkBoxKeyboardShortcutFn(kpEvent.keysym)',
			  's'		:	'checkBoxKeyboardShortcutFn(kpEvent.keysym)',
			  't' 		:	'createTextFn()',
			  'v'		:	'checkBoxKeyboardShortcutFn(kpEvent.keysym)',
			  'w'		:	'checkBoxKeyboardShortcutFn(kpEvent.keysym)',
			  'x'		:	'checkBoxKeyboardShortcutFn(kpEvent.keysym)',
			  'F1'		:	'helpFn()',
			  'Escape'	:	'windowCloseRequested()' } \
			 [kpEvent.keysym]
		eval(fn)
	except KeyError:				# the 'default' case
		pass

def setupGUI(checkBoxStates):
	global createTextBtn, fileNameEntryVar

	root.bind_all('<KeyPress>', keyPressFn)
	rootFrame = ttk.Frame(root, padding = '12 12 12 12')
	rootFrame.pack()

	selectItemsFrame = ttk.LabelFrame(rootFrame, text = 'Select Items', padding = '6 0 6 6')
	checkBoxFrame = ttk.Frame(selectItemsFrame, padding = '6 6 6 6')
	i = 0
	for bsItem, cbState in zip(blockSpecs, checkBoxStates):
		checkBox = CheckBox(bsItem.name, bsItem.underline, checkBoxFrame, cbState)
		checkBoxes.append(checkBox)
		checkBox.checkBtn.grid(row = int(i % 6), column = int(i / 6), sticky = "w", padx = 6)
		i += 1
	checkBoxFrame.pack(side = 'left')
	checkBoxBtnsFrame = ttk.Frame(selectItemsFrame, padding = '6 6 6 6')
	allBtn = ttk.Button(checkBoxBtnsFrame, text = 'All', command = allFn, underline = 0)
	allBtn.grid(row = 0, column = 2, padx = 6, pady = 12)
	noneBtn = ttk.Button(checkBoxBtnsFrame, text = 'None', command = noneFn, underline = 0)
	noneBtn.grid(row = 1, column = 2, padx = 6, pady = 12)
	checkBoxBtnsFrame.pack(side = 'left')
	selectItemsFrame.grid(row = 0, column = 0, columnspan = 2)

	fileFrame = ttk.Frame(rootFrame, padding = '16 20 12 12')
	fileLabel = ttk.Label(fileFrame, text = 'File:  ')
	fileLabel.grid(row = 0, column = 0, sticky = 'w')
	fileNameEntryVar = StringVar()
	fileNameEntry = ttk.Entry(fileFrame, textvariable = fileNameEntryVar, width = 48, state='disabled')
	fileNameEntryVar.set(motifFileName)
	fileNameEntry.grid(row = 0, column = 1, sticky = 'w')
	fileFrame.grid(row = 1, column = 0, sticky = 'w', columnspan = 2)

	btnsFrame = ttk.Frame(rootFrame, padding = '8 12 12 12')
	selectFileBtn = ttk.Button(btnsFrame, text = 'Select File', command = selectFileFn, underline = 7)
	selectFileBtn.grid(row = 0, column = 0, sticky = 'w', padx = 6)
	createTextBtn = \
		ttk.Button(btnsFrame, text = 'Create Text', command = createTextFn, state = 'disabled', underline = 7)
	createTextBtn.grid(row = 0, column = 1, sticky = 'w', padx = 12)
	btnsFrame.grid(row = 2, column = 0, padx = 12, sticky = 'ew')

	helpBtnFrame = ttk.Frame(rootFrame, padding = '12 12 12 12')
	helpBtn = ttk.Button(helpBtnFrame, text = 'Help', command = helpFn, underline = 0)
	helpBtn.grid(row = 0, column = 0, sticky = 'e')
	helpBtnFrame.grid(row = 2, column = 1, padx = 12, sticky = 'ew')

	setCreateBtnState()

def getAppDirectory():									# returns directory app is in
	# see http://cx-freeze.readthedocs.org/en/latest/faq.html
	if getattr(sys, 'frozen', False):					# running in version created by cx_Freeze
		appDir = os.path.dirname(sys.executable)
	else:												# running in normal Python mode
		appDir = os.path.dirname(__file__)
	return os.path.realpath(appDir)
		# realpath to get os-specific path separators, e.g. '\' for Windows

def windowCloseRequested():
	config[STATE_SECTION_NAME] = { \
		WN_POSN_KEY			:	'%d, %d' % (root.winfo_x(), root.winfo_y()),
		MOTIF_FILE_DIR_KEY	:	motifFileDir,
		MOTIF_FILE_NAME_KEY	:	motifFileName,
		CHECK_BOX_STATES	:	str([bool(cb.variable.get()) for cb in checkBoxes]).strip('[]')}
	with open(stateFilePath, 'w') as configFile:
		config.write(configFile)
	root.quit()

def run():
	global root					# required by 'Escape' and 'q' keyboard shortcuts
	global config, stateFilePath
	global motifFileDir, motifFileName

	config = configparser.ConfigParser()
	config.optionxform = str		# preserve case in key names

	# set up app state
	stateFilePath = os.path.join(getAppDirectory(), STATE_FILE_NAME)
	if os.path.isfile(stateFilePath):
		config.read(stateFilePath)
		stateSection = config[STATE_SECTION_NAME]
		windowPosn = \
			[int(coord) for coord in stateSection[WN_POSN_KEY].replace(' ', '').split(',')]
		motifFileDir = stateSection[MOTIF_FILE_DIR_KEY]
		motifFileName = stateSection[MOTIF_FILE_NAME_KEY]
		if not os.path.isdir(motifFileDir):
			motifFileDir = ''
			motifFileName = ''
		elif not os.path.isfile(os.path.join(motifFileDir, motifFileName)):
			motifFileName = ''
		strToBool = lambda strg : strg == 'True'
		checkBoxStates = \
			[strToBool(val) for val in stateSection[CHECK_BOX_STATES].replace(' ', '').split(',')]
	else:
		windowPosn = (40, 40)
		motifFileDir = ''
		motifFileName = ''
		checkBoxStates = [False] * len(blockSpecs)

	root = tkinter.Tk()											# open window
	root.geometry('+%d+%d' % (windowPosn[0], windowPosn[1]))	# position window (x, y)
	root.protocol('WM_DELETE_WINDOW', windowCloseRequested)

	setupGUI(checkBoxStates)

	root.title('motif2text    v%s' % (VERSION))
	root.resizable(False, False)
	root.mainloop()

if __name__ == '__main__':
	run()


# ========================================================================
# Copyright 2017 Emory University
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ========================================================================

from mxnet import nd
from mxnet.gluon.loss import _apply_weighting, Loss

__author__ = 'Jinho D. Choi'

class BILOU:
    B = 'B'     # beginning
    I = 'I'     # inside
    L = 'L'     # last
    O = 'O'     # outside
    U = 'U'     # unit

    @classmethod
    def collect(cls, tags):
        """
        :param tags: a list of tags encoded by the BILOU format.
        :type tags: list of str
        :return: a dictionary where the keys represent the chunk spans and the values represent the tags
        """
        entities = {}
        begin = -1

        for i, tag in enumerate(tags):
            t = tag[0]

            if t == cls.B:
                begin = i
            elif t == cls.I:
                pass
            elif t == cls.L:
                if begin >= 0: entities[(begin, i+1)] = tag[2:]
                begin = -1
            elif t == cls.O:
                begin = -1
            elif t == cls.U:
                entities[(i, i+1)] = tag[2:]
                begin = -1

        return entities

    def quick_fix(self, tags):
        def fix(i, pt, ct, t1, t2):
            if pt == ct: tags[i][0] = t1
            else: tags[i-1][0] = t2

        def aux(i):
            p = tags[i-1][0]
            c = tags[i][0]
            pt = tags[i-1][1:]
            ct = tags[i][1:]

            if p == self.B:
                if   c == self.B: fix(i, pt, ct, self.I, self.U)  # BB -> BI or UB
                elif c == self.U: fix(i, pt, ct, self.L, self.U)  # BU -> BL or UU
                elif c == self.O: tags[i-1][0] = self.U           # BO -> UO
            elif p == self.I:
                if   c == self.B: fix(i, pt, ct, self.I, self.L)  # IB -> II or LB
                elif c == self.U: fix(i, pt, ct, self.I, self.L)  # IU -> II or LU
                elif c == self.O: tags[i-1][0] = self.L           # IO -> LO
            elif p == self.L:
                if   c == self.I: fix(i, pt, ct, self.I, self.B)  # LI -> II or LB
                elif c == self.L: fix(i, pt, ct, self.I, self.B)  # LL -> IL or LB
            elif p == self.O:
                if   c == self.I: tags[i][0] = self.B             # OI -> OB
                elif c == self.L: tags[i][0] = self.B             # OL -> OB
            elif p == self.U:
                if   c == self.I: fix(i, pt, ct, self.B, self.B)  # UI -> BI or UB
                elif c == self.L: fix(i, pt, ct, self.B, self.B)  # UL -> BL or UB

        for i in range(1, len(tags)): aux(i)
        p = tags[-1][0]
        if   p == self.B: tags[-1][0] = self.U
        elif p == self.I: tags[-1][0] = self.L

class MultiLabelSoftmaxCrossEntropyLoss(Loss):
    def __init__(self, axis=-1, sparse_label=True, from_logits=False, weight=None,
                 batch_axis=0, **kwargs):
        super(MultiLabelSoftmaxCrossEntropyLoss, self).__init__(weight, batch_axis, **kwargs)
        self._axis = axis
        self._sparse_label = sparse_label
        self._from_logits = from_logits

    def hybrid_forward(self, F, output, label, sample_weight=None):
        if not self._from_logits:
            output = F.log_softmax(output)
        if self._sparse_label:
            # loss = -F.pick(output, label, axis=self._axis, keepdims=True)
            l = -F.pick(output, label, axis=self._axis, keepdims=True)
            d = nd.array([0 if i.asscalar() < 0 else 1 for i in label]).reshape((-1, 1))
            loss = l * d
        else:
            loss = -F.sum(output*label, axis=self._axis, keepdims=True)
        loss = _apply_weighting(F, loss, self._weight, sample_weight)
        return F.mean(loss, axis=self._batch_axis, exclude=True)

# class ConvBlock(gluon.Block):
#     def __init__(self):
#         super(ConvBlock, self).__init__()
#         with self.name_scope():
#             self.fc0 = gluon.nn.Dense(5)
#             self.fc1 = gluon.nn.Dense(5)
#             self.fc2 = gluon.nn.Dense(5)
#             self.fcn = [self.fc0, self.fc1, self.fc2]
#
#     def forward(self, x):
#         t = [fc(x) for fc in self.fcn]
#         x = nd.concat(*t, dim=0)
#         return x
#
# X = nd.array([
#     [[1,0,0,0,0], [2,0,0,0,0], [3,0,0,0,0], [4,0,0,0,0]],
#     [[0,1,0,0,0], [0,2,0,0,0], [0,3,0,0,0], [0,4,0,0,0]],
#     [[0,0,1,0,0], [0,0,2,0,0], [0,0,3,0,0], [0,0,4,0,0]],
#     [[0,0,0,1,0], [0,0,0,2,0], [0,0,0,3,0], [0,0,0,4,0]],
#     [[0,0,0,0,1], [0,0,0,0,2], [0,0,0,0,3], [0,0,0,0,4]]
# ])
#
# Y = nd.array([[0,1,2],[0,1,2],[0,1,2],[0,-1,-1],[0,1,-1]])
#
# ctx = mx.cpu()
# net = ConvBlock()
# net.collect_params().initialize(mx.init.Xavier(magnitude=2.24), ctx=ctx)
# trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.1})
#
# batch_size = 2
# L = MultiLabelSoftmaxCrossEntropyLoss()
# data = gluon.data.DataLoader(gluon.data.ArrayDataset(X, Y), batch_size=batch_size)
#
# for x, y in data:
#     x = x.as_in_context(ctx)
#     y = y.as_in_context(ctx)
#     y = nd.transpose(y).reshape((-1,1))
#     with autograd.record():
#         output = net(x)
#         print(output)
#         print(y)
#         loss = L(output, y)
#         loss.backward()
#
#
#     trainer.step(x.shape[0])
#
#
# @property
#     def x(self):
#         """
#         If the total number of words in the document is greater than max_len, the rest gets discarded.
#         :param document: a sentence or a list of sentences, where each sentence is represented by a dictionary.
#         :type document: Union[list of dict, dict]
#         :return: a list of feature vectors for the corresponding words across sentences.
#         :rtype: list of numpy.array -> max_len * (word_vsm.dim + ambi_vsm.dim + num_class)
#         """
#         def position(i, size):
#             return self.fst_word if i == 0 else self.lst_word if i+1 == size else self.mid_word
#
#         def aux(sentence):
#             word_emb = sentence.setdefault(WORD_EMB, self.word_vsm.get_list(sentence[TOKEN]))
#             ambi_emb = sentence.setdefault(AMBI_EMB, self.ambi_vsm.get_list(sentence[TOKEN]))
#             pos_scores = sentence.get(POS_OUT, None)
#
#             return [np.concatenate((
#                 position(i, len(sentence)),
#                 word_emb[i],
#                 ambi_emb[i],
#                 self.zero_scores if pos_scores is None else pos_scores[i]))
#                 for i in range(len(sentence))]
#
#         matrix = [v for s in document for v in aux(s)]
#
#         # zero padding
#         if len(matrix) < self.max_len:
#             matrix.extend([np.zeros(len(matrix[0]))] * (self.max_len - len(matrix)))
#         elif len(matrix) > self.max_len:
#             matrix = matrix[:self.max_len]
#
#         return np.array(matrix)
#
#     def gold_labels(self, document):
#         labels = np.concatenate([d[POS_GOLD] for d in document])
#
#         # zero padding
#         if len(labels) < self.max_len:
#             return np.append(labels, np.full(self.max_len - len(labels), -1))
#         elif len(labels) > self.max_len:
#             return labels[:self.max_len]
#         else:
#             return labels
#
#     def set_scores(self, document, output):
#         """
#         :param document: a sentence or a list of sentences, where each sentence is represented by a dictionary.
#         :type document: Union[list of dict, dict]
#         :param output: the output of the part-of-speech tag predictions.
#         :param output: numpy.array -> max_len * num_class
#         """
#         def index(i):
#             return (begin + i) * self.num_class
#
#         def get(sentence):
#             return [output[index(i):index(i+1)] for i in range(0, len(sentence))]
#
#         begin = 0
#
#         if isinstance(document, dict):
#             document[POS_OUT] = get(document)
#         else:
#             for d in document:
#                 sc = get(d)
#                 d[POS_OUT] = sc
#                 begin += sc
#
# def trunc_pads(labels, output):
#     idx = next((i for i, label in enumerate(labels) if label.asscalar() == -1), None)
#     return (labels[:idx], output[:idx]) if idx else (labels, output)

# def evaluate(dat_path, word_vsm, ambi_vsm, model_path):
#     word_vsm = FastText(word_vsm)
#     ambi_vsm = Word2Vec(ambi_vsm) if ambi_vsm else None
#     comp = POSTagger(mx.gpu(0), word_vsm, ambi_vsm, model_path=model_path)
#
#     cols = {TOKEN: 0, POS: 1}
#     dev_states = read_tsv(dat_path, cols, comp.create_state)
#     dev_eval = comp.evaluate(dev_states, 512, Accuracy())
#     print(dev_eval)


from toee import *
from combat_standard_routines import *

def san_dialog( attachee, triggerer ):
	if (game.global_flags[154] == 0 and game.global_flags[155] == 0):
		triggerer.begin_dialog( attachee, 1 )
	return SKIP_DEFAULT

def san_heartbeat( attachee, triggerer ):
	if (not game.combat_is_active()):
		if (game.global_flags[154] == 0 and game.global_flags[155] == 0):
			for obj in game.obj_list_vicinity(attachee.location,OLC_PC):
				if (is_safe_to_talk(attachee,obj)):
					if (attachee.distance_to(obj) <= 10):
						obj.begin_dialog( attachee, 1 )
	return RUN_DEFAULT


# Runtime: 20 ms, faster than 82.28% of Python online submissions for Vertical Order Traversal of a Binary Tree.
# Memory Usage: 13.1 MB, less than 25.00% of Python online submissions for Vertical Order Traversal of a Binary Tree.

# Definition for a binary tree node.
# class TreeNode(object):
#     def __init__(self, x):
#         self.val = x
#         self.left = None
#         self.right = None

class Solution(object):
    def verticalTraversal(self, root):
        """
        :type root: TreeNode
        :rtype: List[List[int]]
        """

        mp = {}

        def f(root, x, y):

            if root == None:
                return

            if mp.get(x) == None:
                mp[x] = {}

            if mp[x].get(y) == None:
                mp[x][y] = []

            mp[x][y].append(root.val)

            f(root.left, x-1, y+1)
            f(root.right, x+1, y+1)

        ans = []
        f(root, 0, 0)

        for xs in sorted(mp.keys()):
            tans = []
            for ys in sorted(mp[xs].keys()):
                tans = tans + sorted(mp[xs][ys])

            ans.append(tans)
        return ans


#!/usr/bin/env python3
# -*- coding:utf-8 -*-

'''
把一个阿拉伯数字因式分解后重组，使所有位不超过十，输出最小的数值。
例：
20 = 2 * 2 * 5，可看作'225'。但是 20 = 4 * 5，45 < 225，所以输出45。
'''

__author__ = 'ManiaJack'

from functools import reduce

# 将数字分解为数组并返回。若有大于十的质数，则在数组为后一位。
def factory(num, fac_list):
    for i in range(9, 1, -1):
        if num % i == 0:
            fac_list.append(i)
            num /= i
            if num == 1:
                return sorted(fac_list)
            return factory(num, fac_list)
    fac_list.append(int(num))
    return fac_list

def num_factory(num):
    fac_list = factory(num, [])
    # 若最后一位为大于十的质数，直接返回0
    if fac_list[-1] >= 10:
        return 0
    return reduce(lambda x, y: 10 * x + y, fac_list)


from flask import Flask
from route import api

application = Flask(__name__)
application.debug = True

application.register_blueprint(api)
if __name__ == '__main__':
    try:
        application.run(host='0.0.0.0')
    except IndexError:
        print('Wrong run command.\n')
        print('python app.py <API_HOST> <APP_TOKEN> <BOT_ID>')


# -*- coding: utf-8 -*-

import requests
import os
import unicodedata
import validators
import threading
import json
#from tkinter import *
from tkinter import messagebox
import tkinter as tk

from bs4 import BeautifulSoup
from setuptools.package_index import HREF
from tinydb import TinyDB,Query,where
#from PIL import ImageTk, Image

class Web_scraping:

    '''
    Created on 18.04.2018 - 20.05.2018

    @author: anjawolf

    Data will be scraped from the Tripadvisors Website and stored in variables.
    '''
    def get_single_data(self,url):
        source_code = requests.get(url)
        source_code.decoding = ('utf-16BE')
        plain_text = source_code.text
        soup = BeautifulSoup(plain_text, "html.parser")

        self.name = soup.find('h1', {'class': 'heading_title'})
        print("Restaurantname: " + self.name.string)

        self.overallPoints = soup.find('div', {'class':'rs rating'})
        print ("Punkteskala: " + self.overallPoints.div.span['content'])

        for self.rating_amount in soup.find('span', {'property': 'count'}):
            print("Anzahl_Bewertungen: " + self.rating_amount.string)

        for self.popularity in soup.find_all('span', {'class': 'header_popularity popIndexValidation'}):
            print("Popularitaet: " + self.popularity.text)

        for self.price_level1 in soup.find('span', {'class': 'header_tags rating_and_popularity'}):
            self.price_level = unicodedata.normalize('NFD', self.price_level1.string)
            print("Preis_Level: " + self.price_level)

        for self.cuisine in soup.find_all('span', {'class': 'header_links rating_and_popularity'}):
            print('Kueche: ' + self.cuisine.text)

        for self.address in soup.find('span', {'class': 'street-address'}):
            print("Strasse: " + self.address.string)

        for self.locality in soup.find('span', {'class': 'locality'}):
            print("PLZ_Ort: " + self.locality.string)

        for self.phonenumber in soup.find_all('div', {'class': 'blEntry phone'}):
            print("Telefonnummer: " + self.phonenumber.text)

        #define auto increment ID for each Restaurant, Review and User
        self.idRestaurant = 0
        self.idReview = 0
        self.idUser = 0

        #load all relevant restaurant data into database (table: RESTAURANTS)
        self.parse_to_tinydb_rest()

        #get all the review containers from current page
        review_containers = soup.find_all('div', class_= 'review-container')
        reviewsPerPage = len(review_containers)

        #get the pageNumbers for all reviews per restaurant
        pageNumbers = soup.find_all('a', class_= 'pageNum')
        pages = len(pageNumbers)
        counter = 1

        #Extract data from single containers
        for container in review_containers:

            for link in container.find_all('a', href=True):
                href = "https://www.tripadvisor.de" + str(link.get('href'))
                #pass it to the function where single data is scraped
                self.get_single_review_data(href)
                counter = counter + 1

        #create a new_url for each page of the reviews
        for index in range (10,pages*10, 10):

            data = url.split("Reviews-")
            new_url = data[0]+"Reviews-or"+str(index)+'-'+data[1]
            #pass the new_url to the loop_trough_review_pages function
            self.loop_through_review_pages(new_url)

        print ("Es wurden " + str(counter) + " textuelle von " + self.rating_amount.string + " Bewertungen abgegeben!")

    '''
    @author: JohannesKnippel

    loop through the restaurant review pages and store the new link in variable
    then pass it to the function where single data is scraped
    '''
    def loop_through_review_pages(self, loop_url):
        source_code = requests.get(loop_url)
        plain_text = source_code.text
        soup = BeautifulSoup(plain_text, "html.parser")

        #get all review containers but on the nuw_url
        review_containers = soup.find_all('div', class_= 'review-container')
        reviewsPerPage = len(review_containers)

        #search trough all reviews on the page
        for container in review_containers:

            for link in container.find_all('a', href=True):
                href = "https://www.tripadvisor.de" + str(link.get('href'))
                #pass it to the function where single data is scraped
                self.get_single_review_data(href)

    '''
    @author: JohannesKnippel, anjawolf, JohannaSickendiek

    loop through the reviews and scrape the data. use link of loop_through_review_pages()
    '''
    def get_single_review_data(self,review_url):
        source_code2 = requests.get(review_url)
        plain_text2 = source_code2.text
        soup = BeautifulSoup(plain_text2, "html.parser")

        #Lists to store scraped data in
        self.username = soup.find('span', {'class': 'expand_inline scrname'}).text
        print('Benutzername: ' + self.username)

        for self.numberOfReviews in soup.findAll('span', {'class': 'badgetext'})[0]:
            print("Anzahl Reviews: " + self.numberOfReviews.string)

        for self.numberOfLikes in soup.findAll('span', {'class': 'badgetext'})[1]:
            print("Anzahl Likes: " + self.numberOfLikes.string)

        #intizialise and load the json from TA to search for relevant data
        self.item_name = soup.find('script', {'type': 'application/ld+json'})
        json_string = str(self.item_name.string)
        obj = json.loads(json_string)

        #get title of the review
        self.title = obj["name"]
        print ("Titel: " + self.title)

        #get content of the review
        self.review = obj["reviewBody"]
        print ("Bewertung: " + self.review)

        #get rating of the review
        self.points = soup.find('div', {'class':'rating'})
        print ("Rating: "+ self.points.span.span['alt'].split()[0])

        #Get URL from review pictures
        for link in soup.findAll('div', {'id': 'taplc_location_reviews_list_sur_callout_0'}):
            if link.find('div', {'class': 'inlinePhotosWrapper'}):
                for pic in link.findAll('img', {'class': 'centeredImg'}):
                    self.src = pic.get('src')
                    print("Quelle Bild " + self.src)
            else:
                print("No pictures have been uploaded for this review!")
        #load all relevant review, User, Picture data into database (table: REVIEWS, USERS, PICTURES)
        self.parse_to_tinydb_us_rev_pic()

    '''
    @author: JohannesKnippel

    TinyDB - Database. Function to handle exit of programm when pressing button on popup.Window
    '''
    def endProgram(self):
        exit()

    '''
    @author: JohannesKnippel

    TinyDB - Database. Scraped Data from get_single_data() function will be parsed to a Database saved in .json-Format. These are all the Restaurant related data
    '''
    def parse_to_tinydb_rest(self):
        cwd = os.getcwd()
        try:
            #create the database or use the existing one
            db = TinyDB('db.json', sort_keys=True, indent=4, separators=(',', ': '))
            #create the tables inside the database
            tableReviews = db.table('REVIEWS')
            tableUsers = db.table('USERS')
            tablePictures = db.table('PICTURES')
            tableRestaurants = db.table('RESTAURANTS')

        except:
            print("Error opening file")

        #check if there are already Restaurants in the table, if so, count the Restaurants and set R_ID. check also if the newly set R_ID already exists, if so, count and check again
        if tableRestaurants.contains((where('R_ID') > 0)):
            for ids3 in tableRestaurants:
                self.idRestaurant = self.idRestaurant + 1
                if tableRestaurants.contains((where('R_ID') == self.idRestaurant)):
                    # print ("next")
                    x = 1
                else:
                    print ("jetzt wird ein neues Restaurant mit RestaurantID : " + str(self.idRestaurant) + " eingefügt")
                    break
        else:
            self.idRestaurant = 1

        #HANDLE RESTAURANTS
        #check if there already exists an entry in the Restaurans-table with the same name and the same address (these two attributes don't change/are not variable so there is a more constant way to check for duplicates)
        if tableRestaurants.contains((where('RESTAURANTNAME') == self.name.string) & (where('PLZ_ORT') == self.locality.string)):
            #pop up a message box
            msg = "Dieses Restaurant ist bereits in der Datenbank hinterlegt!"
            popup = tk.Tk()
            popup.wm_title("!")
            label = tk.Label(popup, text=msg)
            label.pack(side="top", fill="x", pady=10)
            B1 = tk.Button(popup, text="Okay und Beenden", command = self.endProgram)
            B1.pack()
            popup.mainloop()
        # if there is no such entry, parse the data into the corresponding table of the database and define the data to insert into database including an auto increment ID for each Table
        else:
            #self.popularity2 = unicodedata.normalize('NFD', self.pupularity.text)
            print("Preis_Level: " + self.price_level)
            dataRestaurants = {'R_ID':self.idRestaurant, 'RESTAURANTNAME':self.name.string, 'PUNKTESKALA':self.overallPoints.div.span['content'], 'ANZAHL_BEWERTUNGEN':self.rating_amount.string, 'POPULARITAET':self.popularity.text, 'PREIS_LEVEL':self.price_level, 'KUECHE':self.cuisine.text, 'STRASSE':self.address.string, 'PLZ_ORT':self.locality.string, 'TELEFONNUMMER':self.phonenumber.text}
            tableRestaurants.insert_multiple([dataRestaurants])

    '''
    @author: JohannesKnippel

    TinyDB - Database. Scraped Data from get_single_review_data() function will be parsed to a Database saved in .json-Format. These are all the Review, Users and Pictures related data
    '''
    def parse_to_tinydb_us_rev_pic(self):
        cwd = os.getcwd()
        try:
            #create the database or use the existing one
            db = TinyDB('db.json', sort_keys=True, indent=4, separators=(',', ': '))
            #create the tables inside the database
            tableReviews = db.table('REVIEWS')
            tableUsers = db.table('USERS')
            tablePictures = db.table('PICTURES')
            tableRestaurants = db.table('RESTAURANTS')

        except:
            print("Error opening file")

        #AUTOINCREMENT U_ID: check if there are already Users in the table, if so, count the Users and set U_ID. check also if the newly set U_ID already exists, if so, count and check again
        if tableUsers.contains((where('U_ID') > 0)):
            for ids in tableUsers:
                self.idUser = self.idUser + 1
                if tableUsers.contains((where('U_ID') == self.idUser)):
                    #print ("next")
                    x = 1
                else:
                    print ("jetzt wird neuer User mit UserID : " + str(self.idUser) + " eingefügt")
                    break
        else:
            self.idUser = 1

        #AUTINCREMENT REV_ID: check if there are already Reviwes in the table, if so, count the Reviews and set REV_ID. check also if the newly set REV_ID already exists, if so, count and check again
        if tableReviews.contains((where('REV_ID') > 0)):
            for ids2 in tableReviews:
                self.idReview = self.idReview + 1
                if tableReviews.contains((where('REV_ID') == self.idReview)):
                    #print ("next")
                    x = 1
                else:
                    print ("jetzt wird neuer Review mit ReviewID : " + str(self.idReview) + " eingefügt")
                    break
        else:
            self.idReview = 1

        #HANDLE USERS
        #check if there already exists an entry in the USERS-table with the same name (this attribute doesn't change/is not variable so there is a more constant way to check for duplicates)
        if tableUsers.contains((where('BENUTZERNAME') == self.username) & (where('REV_ID') == self.idReview)):
            #print a message
            msg = ("Der User" + self.username + " ist bereits in der Datenbank hinterlegt! Es wird weitergesucht...")
            print(msg)
        #if there is no such entry, parse the data into the corresponding table of the database and define the data to insert into database including an auto increment ID for each Table
        else:
            dataUsers = {'U_ID':self.idUser, 'REV_ID':self.idReview, 'BENUTZERNAME':self.username, 'ANZAHL_REVIEWS':self.numberOfReviews, 'ANZAHL_LIKES':self.numberOfLikes}
            tableUsers.insert(dataUsers)

        #HANDLE REVIEWS
        #check if there already exists an entry in the REVIEWSs-table with the same name and the same address (these two attributes don't change/are not variable so there is a more constant way to check for duplicates)
        if tableReviews.contains((where('TITEL') == self.title) & (where('BEWERTUNG') == self.review)):
            #print a message
            msg = ("Dieses Review mit dem Titel: " + self.title + " wurde bereits in der Datenbank hinterlegt! Es wird weitergesucht...")
            print(msg)
        #if there is no such entry, parse the data into the corresponding table of the database and define the data to insert into database including an auto increment ID for each Table
        else:
            dataReviews = {'R_ID':self.idRestaurant, 'U_ID':self.idUser, 'REV_ID':self.idReview, 'TITEL':self.title, 'BEWERTUNG':self.review, 'RATING':self.points.span.span['alt'].split()[0]}
            tableReviews.insert(dataReviews)

        #HANDLE PICTURES
        #check if there already exists an entry in the PICTURES-table with the same name and the same address (these two attributes don't change/are not variable so there is a more constant way to check for duplicates)
        if tablePictures.contains((where('QUELLE') == self.src)):
            #pop up a message box
            msg = ("Dieses Bild mit dem Link: " + self.src + " wurde bereits in der Datenbank hinterlegt! Es wird weitergesucht...")
            print(msg)
        # if there is no such entry, parse the data into the corresponding table of the database and define the data to insert into database including an auto increment ID for each Table
        else:
            dataPictures = {'REV_ID':self.idReview, 'QUELLE':self.src}
            tablePictures.insert(dataPictures)

    '''
    @author: Skanny Morandi

    opens a GUI that validates a given url-string and starts the scraping on button click
    '''
    def start_GUI(self):

        roots = tk.Tk()
        roots.title('Tripadvisor Scraper')
        instruction = tk.Label(roots, text='Please provide Restaurant-Url\n')
        instruction.grid(row=0, column=0, sticky=tk.E)

        restaurant_label = tk.Label(roots, text='Restaurant-URL ')
        restaurant_label.grid(row=1, column=0, sticky=tk.W)
        restaurant_entry = tk.Entry(roots, width=100)
        restaurant_entry.grid(row=1, column=1, sticky=tk.W)
        restaurant_name_label = tk.Label(roots, text='Restaurant Name')
        restaurant_name_label.grid(row=2, column=0, sticky=tk.W)
        restaurant_city_label = tk.Label(roots, text='City')
        restaurant_city_label.grid(row=3, column=0, sticky=tk.W)
        restaurant_rating_label = tk.Label(roots, text='Average Rating')
        restaurant_rating_label.grid(row=4, column=0, sticky=tk.W)

        def preview_restaurant():
            base_restaurant_url_= "https://www.tripadvisor.de/Restaurant_Review"
            url_to_check = restaurant_entry.get()
            if  (not validators.url(url_to_check)) or \
                (base_restaurant_url_ not in url_to_check):

                messagebox.showwarning("Warning", "This seems not to be valid Tripadvisor restaurant URL")
            else:
                source_code = requests.get(url_to_check)
                source_code.decoding = ('utf-16BE')
                plain_text = source_code.text
                soup = BeautifulSoup(plain_text, "html.parser")

                prev_restuarant_name = soup.find('h1', {'class': 'heading_title'}).string

                prev_restaurant_rating = soup.find('div', {'class': 'rs rating'}).div.span['content']

                prev_restaurant_city = soup.find('span', {'class': 'locality'}).string[:-2]

                prev_restaurant_name_label = tk.Label(roots, text=prev_restuarant_name)
                prev_restaurant_name_label.grid(row=2, column=1, sticky=tk.W)
                prev_restaurant_city_label = tk.Label(roots, text=prev_restaurant_city)
                prev_restaurant_city_label.grid(row=3, column=1, sticky=tk.W)
                prev_restaurant_rating_label = tk.Label(roots, text=prev_restaurant_rating)
                prev_restaurant_rating_label.grid(row=4, column=1, sticky=tk.W)

                roots.pack_slaves()

        preview_button = tk.Button(roots, text='Preview', command=preview_restaurant)
        preview_button.grid(columnspan=3, sticky=tk.W)

        def go_scrape():
            self.get_single_data(restaurant_entry.get())

        scrape_button = tk.Button(roots, text='Start Scraping', command=go_scrape)

        scrape_button.grid(columnspan=5, sticky=tk.W)
        roots.mainloop()

'''
@author: Skanny Morandi, JohannesKnippel

Maisn function, will be executed when starting the python-script
'''
if __name__ == "__main__":
    ws = Web_scraping()
    ws.start_GUI()
    #example URL:
    # https://www.tripadvisor.de/Restaurant_Review-g946452-d8757235-Reviews-The_Forge_Tea_Room-Hutton_le_Hole_North_York_Moors_National_Park_North_Yorkshire_.html
    # https://www.tripadvisor.de/Restaurant_Review-g187309-d2656918-Reviews-Savanna-Munich_Upper_Bavaria_Bavaria.html


# !/usr/bin/env python
# -*- coding: utf-8 -*-
"""
__title__ = 'main'
__author__ = 'JieYuan'
__mtime__ = '19-1-8'
"""
# !/usr/bin/env python
# -*- coding: utf-8 -*-

import socket

socket.SO_REUSEPORT = 15

import re
from datetime import datetime
from vibora import Vibora, Response, Request
from vibora.responses import JsonResponse
from urllib import parse
from poetry.poetry_gen import PoetryGen
import numpy as np
import random
import requests
from bs4 import BeautifulSoup
from collections import OrderedDict

chinese = re.compile('[^\u4e00-\u9fa5]+')

p = PoetryGen(False)

get_unique = lambda iterable: list(OrderedDict.fromkeys(list(iterable)))

def get_poetry(corpus, zishu=7):
    url = "http://www.shicimingju.com/cangtoushi/index.html"
    text = requests.get(url, {'kw': corpus, 'zishu': zishu}, timeout=1).text
    soup = BeautifulSoup(text, 'lxml')
    l = list(map(lambda x: x.text.split(), soup.find_all('div', class_='cangtoushi-item')))
    if l:
        np.random.shuffle(l)
        return l[0]

app = Vibora()

@app.route('/poetry/getpoem/<corpus>', methods=['GET'])
async def page(corpus: str):
    rst = {}
    rst['time'] = str(datetime.now())
    corpus = parse.unquote(corpus)
    corpus = chinese.sub('', corpus)
    corpus_unique = get_unique(corpus)
    try:
        if 2 < len(corpus) < 13 and len(corpus_unique) > 2:
            # 判断重复字
            if len(set(corpus)) < len(corpus):
                idxs = set(range(len(corpus))) - set(np.unique(list(corpus), True)[1])  # 重复字索引
                poem = get_poetry(''.join(corpus_unique))

                if poem:
                    for idx in idxs:
                        poem.insert(idx, p.gen(start_words=corpus[idx]).split())
                else:
                    poem = p.gen(start_words=corpus).split()

                rst['poem'] = poem
                rst['warning'] = '含重复字: 模型作诗'
            else:
                poem = get_poetry(corpus)
                rst['poem'] = poem if poem else p.gen(start_words=corpus).split()
                rst['warning'] = '无重复字: 原诗优先'
        else:
            rst['poem'] = p.gen(start_words=corpus).split()
            rst['warning'] = '字数不足: 模型作诗'

    except Exception as e:
        rst['poem'] = p.gen(start_words=corpus).split()
        rst['warning'] = str(e)
    finally:
        return JsonResponse([rst])

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=5000, workers=4, verbose=False)


# -*- coding: utf-8 -*-
"""
"""
try:
    from .common import get_raw_csv_filename, get_labelled_csv_filename
    from .common import yield_csv_rows, dump_iter_to_csv
    from .spec import load_spec
except:
    from common import get_raw_csv_filename, get_labelled_csv_filename
    from common import yield_csv_rows, dump_iter_to_csv
    from spec import load_spec

#______________________________________________________________________________
#
#  Make CSV with labelled rows
#______________________________________________________________________________

def yield_labelled_rows(p):
    f = get_raw_csv_filename(p)
    # open csv
    gen_in = yield_csv_rows(f)
    # produce new rows
    headline_dict, support_dict = load_spec(p)
    return yield_row_with_labels(gen_in, headline_dict, support_dict)

def dump_labelled_rows_to_csv(p):
    gen_out = yield_labelled_rows(p)
    # save to file
    f = get_labelled_csv_filename(p)
    r = dump_iter_to_csv(gen_out, f)
    return r

#______________________________________________________________________________
#
#  Inspection functions
#______________________________________________________________________________

def list_as_string(l):
    return  " ".join(sorted(l))

def check_vars_not_in_labelled_csv(p):
    """Returns varnames not written to labelled csv file. Prints explaination."""
    infile = get_raw_csv_filename(p)
    headline_dict, support_dict = load_spec(p)
    gen_in = yield_csv_rows(infile)
    gen_out = yield_row_with_labels(gen_in, headline_dict, support_dict)

    z2 = list(v[0] for k,v in headline_dict.items())
    print ("\nVars in spec:")
    print(list_as_string(z2))

    z1 = list(set(row[0] for row in gen_out))
    print ("Vars in labelled csv:")
    print(list_as_string(z1))

    not_in_file = [x for x in z2 if x not in z1]

    if not_in_file:
        print ("Not loaded to labelled csv:")
        print (list_as_string(not_in_file))
    else:
        print ("Variables in spec and in labelled csv file match.\n")

    return not_in_file

def inspect_labelled_output(p):
    f = get_raw_csv_filename(p)
    # open csv
    gen_in = yield_csv_rows(f)
    # produce new rows
    headline_dict, support_dict = load_spec(p)
    print_rows_with_labels(gen_in, headline_dict, support_dict)

#______________________________________________________________________________
#
#  Rows with assigned labels
#______________________________________________________________________________

def yield_row_with_labels(incoming_rows, dict_headline, dict_support):
    """ Return data rows with assigned labels."""
    for incoming_row, labels, data_row in yield_row_with_labels_core(incoming_rows,
                                                                     dict_headline, dict_support):
        if data_row is not None:
            yield labels + data_row

def print_rows_with_labels(incoming_rows, dict_headline, dict_support):
    for row, labels, data_row in yield_row_with_labels_core(incoming_rows,
                                                                     dict_headline, dict_support):

        def _console_filter(s):
            s = s.replace("\u201c", '"')
            s = s.replace("\u201d", '"')
            s = s.replace("\u2026", "***")
            return s

        print("\nIncoming row:", [_console_filter(x) for x in row])
        print("Elements in row:", len(row))
        if data_row is None:
            if labels:
                print("Labels:", labels)
            else:
                print("Length of first element in row is 0.")
        else:
            print("Emitted data row:", labels + data_row)

UNKNOWN_LABELS = ["unknown_var", "unknown_unit"]

def yield_row_with_labels_core(incoming_rows, dict_headline, dict_support):
    """ Returns (incoming_row, labels, data_row) tuple.
    Useful data is when *data_row* is not None.
    Rest of slack is for verbose printing in yield_row_with_labels_with_print().
    """
    labels = [x for x in UNKNOWN_LABELS]
    # unpack incoming iterator
    for row in incoming_rows:
        if len(row[0]) > 0:
            if not is_year(row[0]):
                # not a data row, change label
                labels = adjust_labels(row[0], labels, dict_headline, dict_support)
                yield row, labels, None
            else:
                # data row, assign label and yield
                yield row, labels, row
        else:
            yield row, None, None

def is_year(s):
    # "20141)"
    s = s.replace(")", "")
    try:
        int(s)
        return True
    except:
        return False

def adjust_labels(line, cur_labels, dict_headline, dict_support):
    """Set new primary and secondary label based on *line* contents.
    *line* is first element of csv row.

    line = 'Производство транспортных средств и оборудования  / Manufacture of  transport equipment
    causes change in primary label

    line = 'отчетный месяц в % к предыдущему месяцу  / reporting month as percent of previous mon
    causes change in secondary label

    ASSUMPTIONS:
      - primary label appears only once in csv file
      - pri label followed by sec label
    """

    labels = cur_labels
    # Does anything from 'dict_headline' appear in 'line'?
    z = get_label_in_text(line, dict_headline)
    # Does anything from 'dict_support' appear at the start of 'line'?
    w = get_label_on_start(line, dict_support)

    if z:
       # reset to new var - change both pri and sec label
       labels[0], labels[1] = z
    elif w:
       # change sec label
       labels[1] = w
    else:
       # unknown var, reset labels
       labels = [x for x in UNKNOWN_LABELS]

    return labels

#______________________________________________________________________________
#
#  Extract labels from text based on dictionaries
#______________________________________________________________________________

# End-use wrappers
def get_label_on_start(text, lab_dict):
     return get_label(text, lab_dict, sf_start)

def get_label_in_text(text, lab_dict):
     return get_label(text, lab_dict, sf_anywhere)

# *is_label_found_func* search functions
def sf_start(text, pat):
   return text.strip().startswith(pat)

def sf_anywhere(text, pat):
   return pat in text

# search function for labels
def get_label(text, label_dict, is_label_found_func):
    """Return new label for *text* based on *lab_dict* and *is_label_found_func*
    """
    for pat in label_dict.keys():
        if is_label_found_func(text, pat):
            return label_dict[pat]
    # False will not cause change in labels
    return False

#______________________________________________________________________________

doc = """1.10. Внешнеторговый оборот – всего1),  млрд.долларов США / Foreign trade turnover – total1),  bln US dollars
1999	115,1	24,4	27,2	28,4	35,1	7,2	7,9	9,3	9,8	8,0	9,3	9,5	9,3	9,6	10,4	11,1	13,7"""

def test_iter():
    for row in [x.split("\t") for x in doc.split("\n")]:
        yield row

if __name__ == "__main__":
#    import os
#    p = os.path.abspath("../data/1-07/1-07.txt")
#    gen = yield_labelled_rows(p)
#    for x in gen:
#        print(x)
#
#    inspect_labelled_output(p)
    import os
    p = os.path.abspath("../data/minitab/minitab.csv")
    f = get_raw_csv_filename(p)
    # open csv
    gen_in = test_iter()
    # produce new rows
    headline_dict, support_dict = load_spec(p)
    print_rows_with_labels(gen_in, headline_dict, support_dict)


from flask import Blueprint, jsonify, request

from app.education.service import get_teen_pregnancy_by_education

education_controller = Blueprint('education', __name__)

@education_controller.route('/')
def birth_rate_usa_per_year_and_education():
    year = request.args.get('year', 1995, int)
    data = get_teen_pregnancy_by_education(year)
    return jsonify(data)


from datetime import datetime
import json
import sys

from utils.args_parser import create_parser
from utils.calc_moving_average import MovingAverage
from utils.process_input import check_file_path, get_timestamp_from_string, get_duration_from_string
from utils.read_input_obj import InputSourceFile
from utils.sys_logger import init_sys_logger

logger = init_sys_logger('main_calc_av')

def main():
    """
    Main pipeline for processing data
    Returns: Non

    """
    parser = create_parser()
    args = parser.parse_args()

    input_file = args.input_file
    window = args.window_size
    logger.info('arguments parsed')
    input_file_is_valid = check_file_path(input_file)
    if not input_file_is_valid:
        sys.exit('Input file is not valid. Please, check')
    logger.info('arguments are valid')
    input_obj = InputSourceFile(input_file)
    first_line_as_dict = json.loads(input_obj.first_line())
    first_date_str = first_line_as_dict.get('timestamp', None)
    if not first_date_str:
        sys.exit('No timestamp in first line. Please, check')
    moving_av = MovingAverage(datetime.strptime(first_date_str, '%Y-%m-%d %H:%M:%S.%f').replace(second=0, microsecond=0)
                              , window)

    with input_obj:
        for line in input_obj:
            line_as_dict = json.loads(line)
            timestamp_str = line_as_dict.get('timestamp', None)
            duration_str = line_as_dict.get('duration', None)
            if not all([timestamp_str, duration_str]):
                logger.info(f'line "{line}" missing some data, ignoring it')
                continue
            timestamp_str_is_valid, timestamp = get_timestamp_from_string(timestamp_str)
            duration_str_is_valid, duration = get_duration_from_string(duration_str)
            if not all([timestamp_str_is_valid, duration_str_is_valid]):
                logger.info(f'line "{line}" has malformed data, ignoring it')
                continue

            moving_av.set_records_count()
            while timestamp != moving_av.last_output_date:
                with open('output.json', 'a') as f:
                    output_str = json.dumps({"date": str(moving_av.last_output_date),
                                             "average_delivery_time": moving_av.get_current_average()})
                    f.write(output_str + '\n')
                moving_av.increase_last_output_date()
                moving_av.check_dates_against_window()

            moving_av.add_duration_value(timestamp, int(duration_str))
            with open('output.json', 'a') as f:
                output_str = json.dumps({"date": str(moving_av.last_output_date),
                                         "average_delivery_time": moving_av.get_current_average()})
                f.write(output_str + '\n')
            moving_av.increase_last_output_date()

if __name__ == '__main__':
    main()


import discord
from discord.ext import commands

# TODO

class CustomChannels():
    def __init__(self, bot, utils):
        self.bot = bot
        self.utils = utils

        if 'custom_channels' not in self.utils.settings:
            self.utils.settings['custom_channels'] = {}
        if 'category' not in self.utils.settings['custom_channels']:
            self.utils.settings['custom_channels']['category'] = None
        if 'public_channels' not in self.utils.settings['custom_channels']:
            self.utils.settings['custom_channels']['public_channels'] = []
        if 'private_channels' not in self.utils.settings['custom_channels']:
            self.utils.settings['custom_channels']['private_channels'] = []
        if 'invite_message' not in self.utils.settings['custom_channels']:
            self.utils.settings['custom_channels']['invite_message'] = "{member.mention}, is toegevoegd aan dit kanaal!"

        self.utils.save_settings()

        self.offtopic_category = self.utils.settings['custom_channels']['category']
        self.public_channels = self.utils.settings['custom_channels']['public_channels']
        self.private_channels = self.utils.settings['custom_channels']['private_channels']
        self.invite_message = self.utils.settings['custom_channels']['invite_message']
        self.description_message = "De beschrijving van dit kanaal is aangepast."

    async def on_guild_channel_delete(self, channel):
        if channel.id in self.public_channels:
            public_channels = self.utils.settings['custom_channels']['public_channels']
            public_channels.remove(channel.id)

            self.utils.settings['custom_channels']['public_channels'] = public_channels
            self.public_channels = self.utils.settings['custom_channels']['public_channels']
            self.utils.save_settings()
        elif channel.id in self.private_channels:
            private_channels = self.utils.settings['custom_channels']['private_channels']
            private_channels.remove(channel.id)

            self.utils.settings['custom_channels']['private_channels'] = private_channels
            self.private_channels = self.utils.settings['custom_channels']['private_channels']
            self.utils.save_settings()

    @commands.group(name='customchannel', aliases=['cc'])
    async def customchannel(self, context):
        '''Laat gebruikers hun eigen kanalen aanmaken'''
        if not context.invoked_subcommand:
            await self.utils.send_cmd_help(context)

    @customchannel.command(name='list', aliases=['lijst'])
    async def customchannel_list_directory(self, context):
        '''Laat alle publieke kanalen zien'''

        channels = '\n\n'
        for channel in self.public_channels:
            ch = self.bot.get_channel(channel)
            channels += '{} ({})\n'.format(ch.mention, ch.id)
        embed = discord.Embed(color=discord.Color.red(), description=channels, title='Alle publieke kanalen')
        await context.send(embed=embed)
        # await context.message.add_reaction('\U0001F44D')

    @customchannel.command(name='join', aliases=['betreed'])
    async def customchannel_join_channel(self, context, channel: discord.TextChannel):
        '''Betreed een publiek kanaal'''
        if channel.id in self.public_channels:
            channel = self.bot.get_channel(channel.id)
            member = context.author
            await channel.set_permissions(member,
                                          read_messages=True)
            await context.message.delete()

    @customchannel.command(name='description', aliases=['beschrijving'])
    async def customchannel_change_description_channel(self, context, *, description: str):
        '''Verander beschrijving van een kanaal'''
        channel = context.channel
        description = ''.join(description)
        if channel.id in self.public_channels or channel.id in self.private_channels:
            channel = self.bot.get_channel(channel.id)
            await channel.edit(topic=description)
            await context.channel.send(self.description_message)
            await context.message.delete()

    @customchannel.command(name='category')
    @commands.is_owner()
    async def customchannel_set_category(self, context, categorie: discord.CategoryChannel):
        '''Kies welke categorie je wilt gebruiken om alle kanalen in kwijt te kunnen'''
        self.utils.settings['custom_channels']['category'] = categorie.id
        self.offtopic_category = self.utils.settings['custom_channels']['category']
        self.utils.save_settings()
        await context.message.add_reaction('\U0001F44D')

    @customchannel.command(name='private', aliases=['prive'])
    async def customchannel_new_channel(self, context, *, channel_name: str):
        '''Open een nieuw private kanaal'''
        if self.offtopic_category:
            guild = context.guild
            channel_name = channel_name.replace(' ', '-')

            overwrites = {
                guild.default_role: discord.PermissionOverwrite(read_messages=False),
                guild.me: discord.PermissionOverwrite(read_messages=True),
                context.author: discord.PermissionOverwrite(read_messages=True)
                }

            category = discord.utils.get(guild.categories,
                                         id=self.offtopic_category)

            channel = await guild.create_text_channel(channel_name,
                                                      category=category,
                                                      overwrites=overwrites)

            private_channels = self.utils.settings['custom_channels']['private_channels']
            private_channels.append(channel.id)

            self.utils.settings['custom_channels']['private_channels'] = private_channels
            self.private_channels = self.utils.settings['custom_channels']['private_channels']
            self.utils.save_settings()

            await context.message.add_reaction('\U0001F44D')

    @customchannel.command(name='public', aliases=['publiek'])
    async def customchannel_new_public_channel(self, context, *, channel_name: str):
        '''Open een nieuw publiek kanaal'''
        if self.offtopic_category:
            guild = context.guild
            channel_name = channel_name.replace(' ', '-')

            overwrites = {
                guild.default_role: discord.PermissionOverwrite(read_messages=False),
                guild.me: discord.PermissionOverwrite(read_messages=True),
                context.author: discord.PermissionOverwrite(read_messages=True)
                }

            category = discord.utils.get(guild.categories,
                                         id=self.offtopic_category)

            channel = await guild.create_text_channel(channel_name,
                                                      category=category,
                                                      overwrites=overwrites)

            public_channels = self.utils.settings['custom_channels']['public_channels']
            public_channels.append(channel.id)

            self.utils.settings['custom_channels']['public_channels'] = public_channels
            self.public_channels = self.utils.settings['custom_channels']['public_channels']
            self.utils.save_settings()

            await context.message.add_reaction('\U0001F44D')

    @customchannel.command(name='invite', aliases=['verzoek'])
    async def customchannel_invite_to_channel(self, context, member: discord.Member):
        '''Nodig iemand uit om een private kanaal te betreden'''
        await context.channel.set_permissions(member,
                                              read_messages=True)
        await context.channel.send(self.invite_message.format(member=member))
        await context.message.delete()

    @customchannel.command(name='set_invite')
    @commands.is_owner()
    async def customchannel_set_invite_message(self, context, *, message: str):
        '''Stel een bericht in waarmee de uitgenodigde mensen begroet moeten worden.'''
        self.utils.settings['custom_channels']['invite_message'] = ''.join(message)
        self.invite_message = self.utils.settings['custom_channels']['invite_message']
        self.utils.save_settings()
        await context.message.add_reaction('\U0001F44D')

    # Kleuters...
    # @privatechannel.command(name='schop', aliases=['kick'])
    # async def privatechannel_kick_from_channel(self, context, member: discord.Member):
    #    '''Schop iemand uit het private kanaal'''
    #    await context.channel.set_permissions(member,
    #                                          read_messages=False)
    #    await context.message.add_reaction('\U0001F44D')

    @customchannel.command(name='leave', aliases=['verlaat'])
    async def customchannel_leave_from_channel(self, context):
        '''Verlaat een private kanaal'''
        member = context.author
        await context.channel.set_permissions(member,
                                              read_messages=False)
        await context.message.delete()


# -*- coding: utf-8 -*-
import scrapy
import json
from locations.items import GeojsonPointItem

STATES = ['AL', 'AK', 'AZ', 'AR', 'CA', 'CO', 'CT', 'DC', 'DE', 'FL', 'GA',
          'HI', 'ID', 'IL', 'IN', 'IA', 'KS', 'KY', 'LA', 'ME', 'MD',
          'MA', 'MI', 'MN', 'MS', 'MO', 'MT', 'NE', 'NV', 'NH', 'NJ',
          'NM', 'NY', 'NC', 'ND', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC',
          'SD', 'TN', 'TX', 'UT', 'VT', 'VA', 'WA', 'WV', 'WI', 'WY']

WEEKDAYS = ['Mo', 'Tu', 'We', 'Th', 'Fr', 'Sa', 'Su']

class GiantFoodSpider(scrapy.Spider):
    name = "giantfood"
    item_attributes = { 'brand': "Giant Food" }
    allowed_domains = ["giantfood.com"]

    def start_requests(self):
        yield scrapy.http.Request(
            'https://giantfood.com/auth/oauth/token',
            callback=self.handle_oauth_token,
            body='grant_type=client_credentials&scope=profile',
            method='POST',
            headers={
                'content-type': 'application/x-www-form-urlencoded',
                'Authorization': 'Basic NzJkNTBhZDctNjk4MC00OTQxLWFiNGQtNThkYzM0NjVmMDY5OjczMGUyNzgwMDMxNTkwNWMwYThiYzE0ODRmYTUzM2I2NWM0YWI5Mjc4NzdjZTdiZDYyMzUxODcwMWQ0MDY1ODA=',
            }
        )

    def handle_oauth_token(self, response):
        result = json.loads(response.body_as_unicode())
        access_token = result.get('access_token')

        url = 'https://giantfood.com/auth/api/public/synergy/locator/GNTL/locate/grocery/15/details/'
        headers = {
            'authorization': 'Bearer ' + access_token
        }
        for state in STATES:
            yield scrapy.http.Request(
                '{}?q={}'.format(url, state),
                self.parse,
                method='GET',
                headers=headers
            )

    def parse(self, response):
        result = json.loads(response.body_as_unicode())
        for store in result['stores']:
            ref             = store['storeId']
            name            = 'Giant'
            website         = 'https://giantfood.com/location-details/#/store/detail/' + store['storeNo']
            street          = '{} {}'.format(store['address1'], store['address2'])
            city            = store['city']
            state           = store['state']
            postcode        = store['zip']
            lat             = store['latitude']
            lon             = store['longitude']
            phone           = store['details']['departments'][0]['phone']
            opening_hours   = self.hours(store['details']['departments'][0]['storeDeptHours'])

            properties = {
                'ref'           : ref,
                'name'          : name,
                'website'       : website,
                'lat'           : lat,
                'lon'           : lon,
                'website'       : website,
                'phone'         : phone,
                'opening_hours' : opening_hours,
                'street'        : street,
                'city'          : city,
                'state'         : state,
                'postcode'      : postcode
            }

            yield GeojsonPointItem(**properties)

    def hours(self, store_hours):
        hours = ''
        for data in store_hours:
            if data['24hr']:
                open_time = '00:00'
                close_time = '24:00'
            if 'openTime' in data.keys():
                open_time = data['openTime']
            elif 'pickupStartTime' in data.keys():
                open_time = data['pickupStartTime']
            else:
                open_time = '00:00'

            if 'closeTime' in data.keys():
                close_time = data['closeTime']
            elif 'pickupEndTime' in data.keys():
                close_time = data['pickupEndTime']
            else:
                close_time = '24:00'

            hours = hours + '{} {}-{}; '.format(
                WEEKDAYS[data['day'] - 1],
                open_time,
                close_time
            )

        return hours


from PyPDF2 import PdfFileWriter, PdfFileReader

output = PdfFileWriter()
input1 = PdfFileReader(open("document1.pdf", "rb"))
string = '1-3,5-10,15'

for x in string.split(','):
    #import ipdb; ipdb.set_trace()
    if '-' in x:
        output1 = PdfFileWriter()
        gap = x.split('-')
        y = int(gap[0])
        z = int(gap[1])
        for w in range(z-y+1):
            output1.addPage(input1.getPage(y+w-1))
        outputStream = file("pagesfrom%d.pdf" % w, "wb")
        output1.write(outputStream)
        outputStream.close()
    else:
        output2 = PdfFileWriter()
        output2.addPage(input1.getPage(int(x)-1))
        outputStream = file("uniquepage.pdf", "wb")
        output2.write(outputStream)
        outputStream.close()


import os
import json
import time
import logging
from glob import glob
from threading import Thread
from threading import RLock
from queue import Queue
from queue import Empty
from datetime import datetime
from datetime import timedelta
from enum import Enum
from sbws.globals import RESULT_VERSION, fail_hard
from sbws.util.filelock import DirectoryLock
from sbws.lib.relaylist import Relay
from .. import settings

log = logging.getLogger(__name__)

def merge_result_dicts(d1, d2):
    '''
    Given two dictionaries that contain Result data, merge them.  Result
    dictionaries have keys of relay fingerprints and values of lists of results
    for those relays.
    '''
    for key in d2:
        if key not in d1:
            d1[key] = []
        d1[key].extend(d2[key])
    return d1

def load_result_file(fname, success_only=False):
    ''' Reads in all lines from the given file, and parses them into Result
    structures (or subclasses of Result). Optionally only keeps ResultSuccess.
    Returns all kept Results as a result dictionary. This function does not
    care about the age of the results '''
    assert os.path.isfile(fname)
    d = {}
    num_total = 0
    num_ignored = 0
    with DirectoryLock(os.path.dirname(fname)):
        with open(fname, 'rt') as fd:
            for line in fd:
                num_total += 1
                try:
                    r = Result.from_dict(json.loads(line.strip()))
                except json.decoder.JSONDecodeError:
                    log.warning('Could not decode result %s', line.strip())
                    r = None
                if r is None:
                    num_ignored += 1
                    continue
                if success_only and isinstance(r, ResultError):
                    continue
                fp = r.fingerprint
                if fp not in d:
                    d[fp] = []
                d[fp].append(r)
    num_kept = sum([len(d[fp]) for fp in d])
    log.debug('Keeping %d/%d read lines from %s', num_kept, num_total, fname)
    if num_ignored > 0:
        log.warning('Had to ignore %d results due to not knowing how to '
                    'parse them.', num_ignored)
    return d

def trim_results(fresh_days, result_dict):
    ''' Given a result dictionary, remove all Results that are no longer valid
    and return the new dictionary '''
    assert isinstance(fresh_days, int)
    assert isinstance(result_dict, dict)
    data_period = fresh_days * 24*60*60
    oldest_allowed = time.time() - data_period
    out_results = {}
    for fp in result_dict:
        for result in result_dict[fp]:
            if result.time >= oldest_allowed:
                if fp not in out_results:
                    out_results[fp] = []
                out_results[fp].append(result)
    num_in = sum([len(result_dict[fp]) for fp in result_dict])
    num_out = sum([len(out_results[fp]) for fp in out_results])
    log.debug('Keeping %d/%d results after removing old ones', num_out, num_in)
    return out_results

def trim_results_ip_changed(result_dict, on_changed_ipv4=False,
                            on_changed_ipv6=False):
    """When there are results for the same relay with different IPs,
    create a new results' dictionary without that relay's results using an
    older IP.

    :param dict result_dict: a dictionary of results
    :param bool on_changed_ipv4: whether to trim the results when a relay's
        IPv4 changes
    :param bool on_changed_ipv6: whether to trim the results when a relay's
        IPv6 changes
    :returns: a new results dictionary
    """
    assert isinstance(result_dict, dict)
    new_results_dict = {}
    if on_changed_ipv4 is True:
        for fp in result_dict.keys():
            results = result_dict[fp]
            # find if the results for a relay have more than one ipv4
            # address
            ipv4s = set([result.address for result in results])
            if len(ipv4s) > 1:
                # keep only the results for the last ip used
                # probably we should not just discard all the results for
                # a relay that change address
                ordered_results = sorted(results, key=lambda r: r.time)
                latest_address = ordered_results[-1].address
                last_ip_results = [result for result in results
                                   if result.address == latest_address]
                new_results_dict[fp] = last_ip_results
            else:
                new_results_dict[fp] = results
        return new_results_dict
    if on_changed_ipv6 is True:
        log.warning("Reseting bandwidth results when IPv6 changes,"
                    " is not yet implemented.")
    return result_dict

def load_recent_results_in_datadir(fresh_days, datadir, success_only=False,
                                   on_changed_ipv4=False,
                                   on_changed_ipv6=False):
    ''' Given a data directory, read all results files in it that could have
    results in them that are still valid. Trim them, and return the valid
    Results as a list '''
    assert isinstance(fresh_days, int)
    assert os.path.isdir(datadir)
    # Inform the results are being loaded, since it takes some seconds.
    log.info("Reading and processing previous measurements.")
    results = {}
    today = datetime.utcfromtimestamp(time.time())
    data_period = fresh_days + 2
    oldest_day = today - timedelta(days=data_period)
    working_day = oldest_day
    while working_day <= today:
        # Cannot use ** and recursive=True in glob() because we support 3.4
        # So instead settle on finding files in the datadir and one
        # subdirectory below the datadir that fit the form of YYYY-MM-DD*.txt
        d = working_day.date()
        patterns = [os.path.join(datadir, '{}*.txt'.format(d)),
                    os.path.join(datadir, '*', '{}*.txt'.format(d))]
        for pattern in patterns:
            for fname in glob(pattern):
                new_results = load_result_file(
                    fname, success_only=success_only)
                results = merge_result_dicts(results, new_results)
        working_day += timedelta(days=1)
    results = trim_results(fresh_days, results)
    # in time fresh days is possible that a relay changed ip,
    # if that's the case, keep only the results for the last ip
    results = trim_results_ip_changed(results, on_changed_ipv4,
                                      on_changed_ipv6)
    num_res = sum([len(results[fp]) for fp in results])
    if num_res == 0:
        log.warning('Results files that are valid not found. '
                    'Probably sbws scanner was not run first or '
                    'it ran more than %d days ago or '
                    'it was using a different datadir than %s.', data_period,
                    datadir)
    return results

def write_result_to_datadir(result, datadir):
    ''' Can be called from any thread '''
    assert isinstance(result, Result)
    assert os.path.isdir(datadir)
    dt = datetime.utcfromtimestamp(result.time)
    ext = '.txt'
    result_fname = os.path.join(
        datadir, '{}{}'.format(dt.date(), ext))
    with DirectoryLock(datadir):
        log.debug('Writing a result to %s', result_fname)
        with open(result_fname, 'at') as fd:
            fd.write('{}\n'.format(str(result)))

class _StrEnum(str, Enum):
    pass

class _ResultType(_StrEnum):
    Success = 'success'
    Error = 'error-misc'
    ErrorCircuit = 'error-circ'
    ErrorStream = 'error-stream'
    ErrorAuth = 'error-auth'
    # When it can not be found a second relay suitable to measure a relay.
    # It is used in ``ResultErrorSecondRelay``.
    ErrorSecondRelay = 'error-second-relay'
    # When there is not a working destination Web Server.
    # It is used in ``ResultErrorDestionation``.
    ErrorDestination = 'error-destination'

class Result:
    """A bandwidth measurement for a relay.

    It re-implements :class:`~sbws.lib.relaylist.Relay` as a inner class.
    """

    class Relay:
        """A Tor relay.

        It re-implements :class:`~sbws.lib.relaylist.Relay`
        with the attributes needed.

        .. note:: in a future refactor it would be simpler if a ``Relay`` has
           measurements and a measurement has a relay,
           instead of every measurement re-implementing ``Relay``.
        """
        def __init__(self, fingerprint, nickname, address, master_key_ed25519,
                     average_bandwidth=None, burst_bandwidth=None,
                     observed_bandwidth=None, consensus_bandwidth=None,
                     consensus_bandwidth_is_unmeasured=None,
                     # Counters to be stored by relay and not per measurement,
                     # since the measurements might fail.
                     relay_in_recent_consensus_count=None,
                     relay_recent_measurement_attempt_count=None,
                     relay_recent_priority_list_count=None):
            """
            Initializes a ``Result.Relay``.

            .. note:: in a future refactor the attributes should be dinamic
               to easy adding/removing them.
               They are shared by  :class:`~sbws.lib.relaylist.Relay` and
               :class:`~sbws.lib.v3bwfile.V3BWLine` and there should not be
               repeated in every class.
            """
            self.fingerprint = fingerprint
            self.nickname = nickname
            self.address = address
            self.master_key_ed25519 = master_key_ed25519
            self.average_bandwidth = average_bandwidth
            self.burst_bandwidth = burst_bandwidth
            self.observed_bandwidth = observed_bandwidth
            self.consensus_bandwidth = consensus_bandwidth
            self.consensus_bandwidth_is_unmeasured = \
                consensus_bandwidth_is_unmeasured
            self.relay_in_recent_consensus_count = \
                relay_in_recent_consensus_count
            self.relay_recent_measurement_attempt_count = \
                relay_recent_measurement_attempt_count
            self.relay_recent_priority_list_count = \
                relay_recent_priority_list_count

    def __init__(self, relay, circ, dest_url, scanner_nick, t=None,
                 relay_in_recent_consensus_count=None):
        """
        Initilizes the measurement and the relay with all the relay attributes.
        """
        self._relay = Result.Relay(
            relay.fingerprint, relay.nickname,
            relay.address, relay.master_key_ed25519,
            relay.average_bandwidth,
            relay.burst_bandwidth,
            relay.observed_bandwidth,
            relay.consensus_bandwidth,
            relay.consensus_bandwidth_is_unmeasured,
            relay.relay_in_recent_consensus_count,
            relay.relay_recent_measurement_attempt_count,
            relay.relay_recent_priority_list_count
            )
        self._circ = circ
        self._dest_url = dest_url
        self._scanner = scanner_nick
        self._time = time.time() if t is None else t

    @property
    def type(self):
        raise NotImplementedError()

    @property
    def relay_average_bandwidth(self):
        return self._relay.average_bandwidth

    @property
    def relay_burst_bandwidth(self):
        return self._relay.burst_bandwidth

    @property
    def relay_observed_bandwidth(self):
        return self._relay.observed_bandwidth

    @property
    def consensus_bandwidth(self):
        return self._relay.consensus_bandwidth

    @property
    def consensus_bandwidth_is_unmeasured(self):
        return self._relay.consensus_bandwidth_is_unmeasured

    @property
    def fingerprint(self):
        return self._relay.fingerprint

    @property
    def nickname(self):
        return self._relay.nickname

    @property
    def address(self):
        return self._relay.address

    @property
    def master_key_ed25519(self):
        return self._relay.master_key_ed25519

    @property
    def relay_in_recent_consensus_count(self):
        """Number of times the relay was in a consensus."""
        return self._relay.relay_in_recent_consensus_count

    @property
    def relay_recent_measurement_attempt_count(self):
        """Returns the relay recent measurements attemps.

        It is initialized in :class:`~sbws.lib.relaylist.Relay` and
        incremented in :func:`~sbws.core.scanner.main_loop`.
        """
        return self._relay.relay_recent_measurement_attempt_count

    @property
    def relay_recent_priority_list_count(self):
        """Returns the relay recent "prioritization"s to be measured.

        It is initialized in :class:`~sbws.lib.relaylist.Relay` and
        incremented in :func:`~sbws.core.scanner.main_loop`.
        """
        return self._relay.relay_recent_priority_list_count

    @property
    def circ(self):
        return self._circ

    @property
    def dest_url(self):
        return self._dest_url

    @property
    def scanner(self):
        return self._scanner

    @property
    def time(self):
        return self._time

    @property
    def version(self):
        return RESULT_VERSION

    def to_dict(self):
        return {
            'fingerprint': self.fingerprint,
            'nickname': self.nickname,
            'address': self.address,
            'master_key_ed25519': self.master_key_ed25519,
            'circ': self.circ,
            'dest_url': self.dest_url,
            'time': self.time,
            'type': self.type,
            'scanner': self.scanner,
            'version': self.version,
            'relay_in_recent_consensus_count':
                self.relay_in_recent_consensus_count,
            'relay_recent_measurement_attempt_count':
                self.relay_recent_measurement_attempt_count,
            'relay_recent_priority_list_count':
                self.relay_recent_priority_list_count,
        }

    @staticmethod
    def from_dict(d):
        """
        Returns a :class:`~sbws.lib.resultdump.Result` subclass from a
        dictionary.

        Returns None if the ``version`` attribute is not
        :const:`~sbws.globals.RESULT_VERSION`

        It raises ``NotImplementedError`` when the dictionary ``type`` can not
        be parsed.

        .. note:: in a future refactor, the conversions to/from
           object-dictionary will be simpler using ``setattr`` and ``__dict__``

           ``version`` is not being used and should be removed.
        """
        assert 'version' in d
        if d['version'] != RESULT_VERSION:
            return None
        assert 'type' in d
        if d['type'] == _ResultType.Success.value:
            return ResultSuccess.from_dict(d)
        elif d['type'] == _ResultType.Error.value:
            return ResultError.from_dict(d)
        elif d['type'] == _ResultType.ErrorCircuit.value:
            return ResultErrorCircuit.from_dict(d)
        elif d['type'] == _ResultType.ErrorStream.value:
            return ResultErrorStream.from_dict(d)
        elif d['type'] == _ResultType.ErrorAuth.value:
            return ResultErrorAuth.from_dict(d)
        elif d['type'] == _ResultType.ErrorSecondRelay.value:
            return ResultErrorSecondRelay.from_dict(d)
        elif d['type'] == _ResultType.ErrorDestination.value:
            return ResultErrorDestination.from_dict(d)
        else:
            raise NotImplementedError(
                'Unknown result type {}'.format(d['type']))

    def __str__(self):
        return json.dumps(self.to_dict())

class ResultError(Result):
    def __init__(self, *a, msg=None, **kw):
        super().__init__(*a, **kw)
        self._msg = msg

    @property
    def type(self):
        return _ResultType.Error

    @property
    def freshness_reduction_factor(self):
        '''
        When the RelayPrioritizer encounters this Result, how much should it
        adjust its freshness? (See RelayPrioritizer.best_priority() for more
        information about "freshness")

        A higher factor makes the freshness lower (making the Result seem
        older). A lower freshness leads to the relay having better priority,
        and better priority means it will be measured again sooner.

        The value 0.5 was chosen somewhat arbitrarily, but a few weeks of live
        network testing verifies that sbws is still able to perform useful
        measurements in a reasonable amount of time.
        '''
        return 0.5

    @property
    def msg(self):
        return self._msg

    @staticmethod
    def from_dict(d):
        assert isinstance(d, dict)
        return ResultError(
            Result.Relay(
                d['fingerprint'], d['nickname'], d['address'],
                d['master_key_ed25519'],
                relay_in_recent_consensus_count=  # noqa
                    d.get('relay_in_recent_consensus_count', None),  # noqa
                relay_recent_measurement_attempt_count=  # noqa
                    d.get('relay_recent_measurement_attempt_count', None),  # noqa
                relay_recent_priority_list_count=  # noqa
                    d.get('relay_recent_priority_list_count', None),  # noqa
                ),
            d['circ'], d['dest_url'], d['scanner'],
            msg=d['msg'], t=d['time'])

    def to_dict(self):
        d = super().to_dict()
        d.update({
            'msg': self.msg,
        })
        return d

class ResultErrorCircuit(ResultError):
    def __init__(self, *a, **kw):
        super().__init__(*a, **kw)

    @property
    def type(self):
        return _ResultType.ErrorCircuit

    @property
    def freshness_reduction_factor(self):
        '''
        There are a few instances when it isn't the relay's fault that the
        circuit failed to get built. Maybe someday we'll try detecting whose
        fault it most likely was and subclassing ResultErrorCircuit. But for
        now we don't. So reduce the freshness slightly more than ResultError
        does by default so priority isn't hurt quite as much.

        A (hopefully very very rare) example of when a circuit would fail to
        get built is when the sbws client machine suddenly loses Internet
        access.
        '''
        return 0.6

    @staticmethod
    def from_dict(d):
        assert isinstance(d, dict)
        return ResultErrorCircuit(
            Result.Relay(
                d['fingerprint'], d['nickname'], d['address'],
                d['master_key_ed25519'],
                relay_in_recent_consensus_count=  # noqa
                    d.get('relay_in_recent_consensus_count', None),  # noqa
                relay_recent_measurement_attempt_count=  # noqa
                    d.get('relay_recent_measurement_attempt_count', None),  # noqa
                relay_recent_priority_list_count=  # noqa
                    d.get('relay_recent_priority_list_count', None),  # noqa
                ),
            d['circ'], d['dest_url'], d['scanner'],
            msg=d['msg'], t=d['time'])

    def to_dict(self):
        d = super().to_dict()
        return d

class ResultErrorStream(ResultError):
    def __init__(self, *a, **kw):
        super().__init__(*a, **kw)

    @property
    def type(self):
        return _ResultType.ErrorStream

    @staticmethod
    def from_dict(d):
        assert isinstance(d, dict)
        return ResultErrorStream(
            Result.Relay(
                d['fingerprint'], d['nickname'], d['address'],
                d['master_key_ed25519'],
                relay_in_recent_consensus_count=  # noqa
                    d.get('relay_in_recent_consensus_count', None),  # noqa
                relay_recent_measurement_attempt_count=  # noqa
                    d.get('relay_recent_measurement_attempt_count', None),  # noqa
                relay_recent_priority_list_count=  # noqa
                    d.get('relay_recent_priority_list_count', None),  # noqa
                ),
            d['circ'], d['dest_url'], d['scanner'],
            msg=d['msg'], t=d['time'])

    def to_dict(self):
        d = super().to_dict()
        return d

class ResultErrorSecondRelay(ResultError):
    """
    Error when it could not be found a second relay suitable to measure
    a relay.

    A second suitable relay is a relay that:
    - Has at least equal bandwidth as the relay to measure.
    - If the relay to measure is not an exit,
      the second relay is an exit without `bad` flag and can exit to port 443.
    - If the relay to measure is an exit, the second relay is not an exit.

    It is instanciated in :func:`~sbws.core.scanner.measure_relay`.

    .. note:: this duplicates code and add more tech-debt,
       since it's the same as the other
       :class:`~sbws.lib.resultdump.ResultError` classes except for the
       ``type``.
       In a future refactor, there should be only one ``ResultError`` class
       and assign the type in the ``scanner`` module.
    """
    def __init__(self, *a, **kw):
        super().__init__(*a, **kw)

    @property
    def type(self):
        return _ResultType.ErrorSecondRelay

    @staticmethod
    def from_dict(d):
        assert isinstance(d, dict)
        return ResultErrorSecondRelay(
            Result.Relay(
                d['fingerprint'], d['nickname'], d['address'],
                d['master_key_ed25519']),
            d['circ'], d['dest_url'], d['scanner'],
            msg=d['msg'], t=d['time'])

    def to_dict(self):
        d = super().to_dict()
        return d

class ResultErrorDestination(ResultError):
    """
    Error when there is not a working destination Web Server.

    It is instanciated in :func:`~sbws.core.scanner.measure_relay`.

    .. note:: this duplicates code and add more tech-debt,
       since it's the same as the other
       :class:`~sbws.lib.resultdump.ResultError` classes except for the
       ``type``.
       In a future refactor, there should be only one ``ResultError`` class
       and assign the type in the ``scanner`` module.
    """
    def __init__(self, *a, **kw):
        super().__init__(*a, **kw)

    @property
    def type(self):
        return _ResultType.ErrorSecondRelay

    @staticmethod
    def from_dict(d):
        assert isinstance(d, dict)
        return ResultErrorSecondRelay(
            Result.Relay(
                d['fingerprint'], d['nickname'], d['address'],
                d['master_key_ed25519']),
            d['circ'], d['dest_url'], d['scanner'],
            msg=d['msg'], t=d['time'])

    def to_dict(self):
        d = super().to_dict()
        return d

class ResultErrorAuth(ResultError):
    def __init__(self, *a, **kw):
        super().__init__(*a, **kw)

    @property
    def type(self):
        return _ResultType.ErrorAuth

    @property
    def freshness_reduction_factor(self):
        '''
        Override the default ResultError.freshness_reduction_factor because a
        ResultErrorAuth is most likely not the measured relay's fault, so we
        shouldn't hurt its priority as much. A higher reduction factor means a
        Result's effective freshness is reduced more, which makes the relay's
        priority better.

        The value 0.9 was chosen somewhat arbitrarily.
        '''
        return 0.9

    @staticmethod
    def from_dict(d):
        assert isinstance(d, dict)
        return ResultErrorAuth(
            Result.Relay(
                d['fingerprint'], d['nickname'], d['address'],
                d['master_key_ed25519'],
                relay_in_recent_consensus_count=  # noqa
                    d.get('relay_in_recent_consensus_count', None),  # noqa
                relay_recent_measurement_attempt_count=  # noqa
                    d.get('relay_recent_measurement_attempt_count', None),  # noqa
                relay_recent_priority_list_count=  # noqa
                    d.get('relay_recent_priority_list_count', None),  # noqa
                ),
            d['circ'], d['dest_url'], d['scanner'],
            msg=d['msg'], t=d['time'])

    def to_dict(self):
        d = super().to_dict()
        return d

class ResultSuccess(Result):
    def __init__(self, rtts, downloads, *a, **kw):
        super().__init__(*a, **kw)
        self._rtts = rtts
        self._downloads = downloads

    @property
    def type(self):
        return _ResultType.Success

    @property
    def rtts(self):
        return self._rtts

    @property
    def downloads(self):
        return self._downloads

    @staticmethod
    def from_dict(d):
        assert isinstance(d, dict)
        return ResultSuccess(
            d['rtts'] or [], d['downloads'],
            Result.Relay(
                d['fingerprint'], d['nickname'], d['address'],
                d['master_key_ed25519'], d['relay_average_bandwidth'],
                d.get('relay_burst_bandwidth'), d['relay_observed_bandwidth'],
                d.get('consensus_bandwidth'),
                d.get('consensus_bandwidth_is_unmeasured'),
                relay_in_recent_consensus_count=  # noqa
                    d.get('relay_in_recent_consensus_count', None),  # noqa
                relay_recent_measurement_attempt_count=  # noqa
                    d.get('relay_recent_measurement_attempt_count', None),  # noqa
                relay_recent_priority_list_count=  # noqa
                    d.get('relay_recent_priority_list_count', None),  # noqa
                ),
            d['circ'], d['dest_url'], d['scanner'],
            t=d['time'])

    def to_dict(self):
        d = super().to_dict()
        d.update({
            'rtts': self.rtts,
            'downloads': self.downloads,
            'relay_average_bandwidth': self.relay_average_bandwidth,
            'relay_burst_bandwidth': self.relay_burst_bandwidth,
            'relay_observed_bandwidth': self.relay_observed_bandwidth,
            'consensus_bandwidth': self.consensus_bandwidth,
            'consensus_bandwidth_is_unmeasured':
                self.consensus_bandwidth_is_unmeasured,
        })
        return d

class ResultDump:
    ''' Runs the enter() method in a new thread and collects new Results on its
    queue. Writes them to daily result files in the data directory '''
    def __init__(self, args, conf):
        assert os.path.isdir(conf.getpath('paths', 'datadir'))
        self.conf = conf
        self.fresh_days = conf.getint('general', 'data_period')
        self.datadir = conf.getpath('paths', 'datadir')
        self.data = {}
        self.data_lock = RLock()
        self.thread = Thread(target=self.enter)
        self.queue = Queue()
        try:
            self.thread.start()
        except RuntimeError as e:
            fail_hard(e)

    def store_result(self, result):
        ''' Call from ResultDump thread '''
        assert isinstance(result, Result)
        with self.data_lock:
            fp = result.fingerprint
            if fp not in self.data:
                self.data[fp] = []
            self.data[fp].append(result)
            self.data = trim_results(self.fresh_days, self.data)
            # Not calling trim_results_ip_changed here to do not remove
            # the results for a relay that has changed address.
            # It will be called when loading the results to generate a v3bw
            # file.

    def handle_result(self, result):
        ''' Call from ResultDump thread. If we are shutting down, ignores
        ResultError* types '''
        assert isinstance(result, Result)
        fp = result.fingerprint
        nick = result.nickname
        if isinstance(result, ResultError) and settings.end_event.is_set():
            log.debug('Ignoring %s for %s %s because we are shutting down',
                      type(result).__name__, nick, fp)
            return
        self.store_result(result)
        write_result_to_datadir(result, self.datadir)
        if result.type == "success":
            msg = "Success measuring {} ({}) via circuit {} and " \
                  "destination {}".format(
                    result.fingerprint, result.nickname, result.circ,
                    result.dest_url)
        else:
            msg = "Error measuring {} ({}) via circuit {} and " \
                  "destination {}: {}".format(
                    result.fingerprint, result.nickname, result.circ,
                    result.dest_url, result.msg)
        # When the error is that there are not more functional destinations.
        if result.type == "error-destination":
            log.info("Shutting down because there are not functional "
                     "destinations.")
            # NOTE: Because this is executed in a thread, stop_threads can not
            # be call from here, it has to be call from the main thread.
            # Instead set the singleton end event, that will call stop_threads
            # from the main process.
            settings.end_event.set()
        log.info(msg)

    def enter(self):
        """Main loop for the ResultDump thread.

        When there are results in the queue, queue.get will get them until
        there are not anymore or timeout happen.

        For every result it gets, it process it and store in the filesystem,
        which takes ~1 millisecond and will not trigger the timeout.
        It can then store in the filesystem ~1000 results per second.

        I does not accept any other data type than Results or list of Results,
        therefore is not possible to put big data types in the queue.

        If there are not any results in the queue, it waits 1 second and checks
        again.

        """
        with self.data_lock:
            self.data = load_recent_results_in_datadir(
                self.fresh_days, self.datadir)
        while not (settings.end_event.is_set() and self.queue.empty()):
            try:
                event = self.queue.get(timeout=1)
            except Empty:
                continue
            data = event
            if data is None:
                log.debug('Got None in ResultDump')
                continue
            elif isinstance(data, list):
                for r in data:
                    assert isinstance(r, Result)
                    self.handle_result(r)
            elif isinstance(data, Result):
                self.handle_result(data)
            else:
                log.warning('The only thing we should ever receive in the '
                            'result thread is a Result or list of Results. '
                            'Ignoring %s', type(data))

    def results_for_relay(self, relay):
        assert isinstance(relay, Relay)
        fp = relay.fingerprint
        with self.data_lock:
            if fp not in self.data:
                return []
            return self.data[fp]