import argparse
import os
import sys
import time
from watchdog.observers import Observer
from watchdog.events import LoggingEventHandler
from watchdog.events import FileSystemEventHandler
#import multiprocessing as mp
from collections import OrderedDict
import re
import copy
import json
import subprocess
#import sched
import webbrowser
from shutil import copyfile, which
import dateutil
from datetime import datetime
from operator import itemgetter
from .version import __version__
from .statsparser import get_argument_parser as sp_get_argument_parser
from .statsparser import parse_args as sp_parse_args
from .helper import initLogger, resources_dir, get_script_dir, hostname, ArgHelpFormatter, r_file, r_dir, rw_dir, defaults, jinja_env
import threading
import logging
import queue
from pathlib import Path
from jinja2 import Environment, PackageLoader, select_autoescape

ALL_RUNS = {}
ALL_RUNS_LOCK = threading.RLock()
SP_DIRS = {}
SP_DIRS_LOCK = threading.RLock()
MUX_RESULTS = {}
MUX_RESULTS_LOCK = threading.RLock()
UPDATE_OVERVIEW = False
UPDATE_OVERVIEW_LOCK = threading.RLock()
logger = None

class parse_statsparser_args(argparse.Action):
	def __call__(self, parser, namespace, values, option_string=None):
		to_test = values.split(' ')
		argument_parser = sp_get_argument_parser()
		args = sp_parse_args(argument_parser, to_test)
		setattr(namespace,self.dest,to_test)

def parse_args():
	parser = argparse.ArgumentParser(description='''A tool for monitoring and protocoling sequencing runs
												 performed on the Oxford Nanopore Technologies GridION
												 sequencer and for automated post processing and transmission
												 of generated data. It collects information on QC and
												 sequencing experiments and displays summaries of mounted
												 flow cells as well as comprehensive reports about currently
												 running and previously performed experiments.''',
									 formatter_class=ArgHelpFormatter,
									 add_help=False)

	general_group = parser.add_argument_group('General arguments',
											  "arguments for advanced control of the program's behavior")

	general_group.add_argument('-n', '--no_transfer',
							   action='store_true',
							   help='''no data transfer to the remote host''')
	general_group.add_argument('-a', '--all_fast5',
							   action='store_true',
							   help='''also put fast5 files of reads removed by length and quality
							           filtering into barcode bins''')
	general_group.add_argument('-p', '--pass_only',
							   action='store_true',
							   help='''use data from fastq_pass only''')
	general_group.add_argument('-l', '--min_length',
							   type=int,
							   default=1000,
							   help='''minimal length to pass filter''')
	general_group.add_argument('-r', '--min_length_rna',
							   type=int,
							   default=50,
							   help='''minimal length to pass filter for rna libraries''')
	general_group.add_argument('-q', '--min_quality',
							   type=int,
							   default=5,
							   help='''minimal quality to pass filter''')
	general_group.add_argument('-d', '--rsync_dest',
							   default="{}@{}:{}".format(defaults()["user"], defaults()["host"], defaults()["dest"]),
							   help='''destination for data transfer with rsync, format USER@HOST[:DEST].
							           Key authentication for the specified destination must be set up (see option -i),
							           otherwise data transfer will fail. Default value is parsed from setting
							           file {}'''.format(os.path.join(resources_dir, "defaults.ini")))
	general_group.add_argument('-i', '--identity_file',
							   default="{}".format(defaults()["identity"]),
							   help='''file from which the identity (private key) for public key authentication is read.
							           Default value is parsed from setting file {}'''.format(os.path.join(resources_dir, "defaults.ini")))
	general_group.add_argument('--bc_kws',
							   nargs='*',
							   default=['RBK', 'NBD', 'RAB', 'LWB', 'PBK', 'RPB', 'arcod'],
							   help='''if at least one of these key words is a substring of the run name,
									   porechop is used to demultiplex the fastq data''')
	general_group.add_argument('-u', '--update_interval',
							   type=int,
							   default=300,
							   help='minimum time interval in seconds for updating the content of a report page')
	general_group.add_argument('-m', '--ignore_file_modifications',
							   action='store_true',
							   help='''Ignore file modifications and only consider file creations regarding
							           determination of the latest log files''')

	io_group = parser.add_argument_group('I/O arguments',
										 'Further input/output arguments. Only for special use cases')
	io_group.add_argument('-o', '--output_dir',
						  action=rw_dir,
						  default="/data/dominION/",
						  help='Path to the base directory where experiment reports shall be saved')
	arg_data_basedir = \
	io_group.add_argument('--data_basedir',
						  action=rw_dir,
						  default='/data',
						  help='Path to the directory where basecalled data is saved')
	io_group.add_argument('--minknow_log_basedir',
						  action=r_dir,
						  default='/var/log/MinKNOW',
						  help='''Path to the base directory of GridIONs log files''')

	io_group.add_argument('--logfile',
						  help='''File in which logs will be safed
						  (default: OUTPUTDIR/logs/YYYY-MM-DD_hh:mm_HOSTNAME_LOGLVL.log''')

	sp_arguments = parser.add_argument_group('Statsparser arguments',
										   'Arguments passed to statsparser for formatting html reports')
	sp_arguments.add_argument('--statsparser_args',
							action=parse_statsparser_args,
							default=[],
							help='''Arguments that are passed to the statsparser script.
								   See a full list of available arguments with --statsparser_args " -h" ''')

	help_group = parser.add_argument_group('Help')
	help_group.add_argument('-h', '--help',
							action='help',
							default=argparse.SUPPRESS,
							help='Show this help message and exit')
	help_group.add_argument('--version',
							action='version',
							version=__version__,
							help="Show program's version string and exit")
	help_group.add_argument('-v', '--verbose',
							action='store_true',
							help='Additional debug messages are printed to stdout')
	help_group.add_argument('--quiet',
							action='store_true',
							help='Only errors and warnings are printed to stdout')

	args = parser.parse_args()

	ns = argparse.Namespace()
	arg_data_basedir(parser, ns, args.data_basedir, '')

	if not os.path.exists(args.identity_file):
		print("Identity file {} does not exists. Please check key authentication settings or specify a different key with option -i.".format(args.identity_file))
		exit()

	args.watchnchop_args = []
	if args.no_transfer:
		args.watchnchop_args.append('-n')
	if args.all_fast5:
		args.watchnchop_args.append('-a')
	if args.pass_only:
		args.watchnchop_args.append('-p')
	#args.watchnchop_args.extend(['-l', str(args.min_length)])
	#args.watchnchop_args.extend(['-r', str(args.min_length_rna)])
	args.watchnchop_args.extend(['-q', str(args.min_quality)])
	args.watchnchop_args.extend(['-d', args.rsync_dest])
	args.watchnchop_args.extend(['-i', args.identity_file])

	return args

def main(args):
	global ALL_RUNS
	global ALL_RUNS_LOCK
	global UPDATE_OVERVIEW
	global logger

	for p in [args.output_dir,
			  os.path.join(args.output_dir, 'runs'),
			  os.path.join(args.output_dir, 'qc'),
			  os.path.join(args.output_dir, 'logs')]:
		if not os.path.exists(p):
			os.makedirs(p)

	if args.verbose:
		loglvl = logging.DEBUG
	elif args.quiet:
		loglvl = logging.WARNING
	else:
		loglvl = logging.INFO
	if not args.logfile:
		logs_filename = "{}_{}_{}.log".format(datetime.now().strftime("%Y-%m-%d_%H:%M"), hostname, loglvl)
		args.logfile = os.path.join(args.output_dir, 'logs', logs_filename)
	initLogger(logfile=args.logfile, level=loglvl)

	logger = logging.getLogger(name='gw')
	logger.info("##### starting dominION {} #####\n".format(__version__))

	logger.info("setting up dominION status page environment")
	if not os.path.exists(os.path.join(args.output_dir, 'res')):
		os.makedirs(os.path.join(args.output_dir, 'res'))
	for res_file in ['style.css', 'flowcell.png', 'no_flowcell.png']:
		copyfile(os.path.join(resources_dir, res_file),
				 os.path.join(args.output_dir, 'res', res_file))

	import_qcs(os.path.join(args.output_dir, "qc"))
	import_runs(os.path.join(args.output_dir, "runs"))

	logger.info("starting to observe runs directory for changes to directory names")
	observed_dir = os.path.join(args.output_dir, 'runs')
	event_handler = RunsDirsEventHandler(observed_dir)
	observer = Observer()
	observer.schedule(event_handler,
					  observed_dir,
					  recursive=True)
	observer.start()

	logger.info("starting channel watchers:")
	watchers = []
	for channel in range(5):
		watchers.append(Watcher(args.minknow_log_basedir,
								channel,
								args.ignore_file_modifications,
								args.output_dir,
								args.data_basedir,
								args.statsparser_args,
								args.update_interval,
								args.watchnchop_args,
								args.min_length,
								args.min_length_rna,
								args.bc_kws))

	logger.info("initiating dominION overview page")
	update_overview(watchers, args.output_dir)
	webbrowser.open('file://' + os.path.realpath(os.path.join(args.output_dir, "{}_overview.html".format(hostname))))
	logger.info("entering main loop")
	try:
		n = 0
		while True:
			for watcher in watchers:
				watcher.check_q()
			if UPDATE_OVERVIEW:
				update_overview(watchers, args.output_dir)
				UPDATE_OVERVIEW = False
			time.sleep(0.2)
			n += 1
			if n == 100:
				n = 0
				set_update_overview()
	except KeyboardInterrupt:
		for watcher in watchers:
			watcher.observer.stop()
			if watcher.spScheduler.is_alive() if watcher.spScheduler else None:
				watcher.stop_statsparser(0.05)
			for wcScheduler in watcher.wcScheduler:
				if wcScheduler.is_alive() if wcScheduler else None:
					wcScheduler.join(timeout=0.05)
	for watcher in watchers:
		logger.info("joining GA{}0000's observer".format(watcher.channel))
		watcher.observer.join()
		for wcScheduler in watcher.wcScheduler:
			if wcScheduler.is_alive() if wcScheduler else None:
				logger.info("joining GA{}0000's watchnchop scheduler".format(watcher.channel))
				wcScheduler.join()
	for watcher in watchers:
		if watcher.spScheduler.is_alive() if watcher.spScheduler else None:
			logger.info("joining GA{}0000's statsparser scheduler".format(watcher.channel))
			watcher.stop_statsparser()

def set_update_overview():
	global UPDATE_OVERVIEW
	UPDATE_OVERVIEW_LOCK.acquire()
	UPDATE_OVERVIEW = True
	UPDATE_OVERVIEW_LOCK.release()

def add_database_entry(flowcell, run_data, mux_scans):
	ALL_RUNS_LOCK.acquire()
	#TODO: check for all mandatory entries
	asic_id_eeprom = flowcell['asic_id_eeprom']
	run_id = run_data['run_id']
	if asic_id_eeprom in ALL_RUNS:
		if run_id in ALL_RUNS[asic_id_eeprom]:
			logger.warning("{} exists multiple times in database!".format(run_id))
			logger.warning("conflicting runs: {}, {}".format(ALL_RUNS[asic_id_eeprom][run_id]['run_data']['relative_path'],
															 run_data['relative_path']))
			ALL_RUNS_LOCK.release()
			return False
	else:
		ALL_RUNS[asic_id_eeprom] = {}

	ALL_RUNS[asic_id_eeprom][run_id] = {'flowcell'	: flowcell,
										'run_data'	: run_data,
										'mux_scans'	: mux_scans}
	logger.debug('{} - added experiment of type "{}" performed on flowcell "{}" on "{}"'.format(asic_id_eeprom,
																								run_data['experiment_type'],
																								flowcell['flowcell_id'],
																								run_data['protocol_start']))
	ALL_RUNS_LOCK.release()
	return True

def add_mux_scan_results(flowcell_data, mux_scans):
	MUX_RESULTS_LOCK.acquire()
	asic_id_eeprom = flowcell_data['asic_id_eeprom']
	flowcell_id = flowcell_data['flowcell_id']
	if asic_id_eeprom not in MUX_RESULTS:
		MUX_RESULTS[asic_id_eeprom] = []
	for mux_scan in mux_scans:
		mux_scan_copy = copy.deepcopy(mux_scan)
		if not 'total' in mux_scan:
			if 'group * total' in mux_scan:
				mux_scan_copy['total'] = mux_scan['group * total']
				del mux_scan_copy['group * total']
			else:
				continue
		mux_scan_copy['flowcell_id'] = flowcell_id
		mux_scan_copy['timestamp'] = dateutil.parser.parse(mux_scan['timestamp'])
		for i in range(len(MUX_RESULTS[asic_id_eeprom])):
			if mux_scan_copy['timestamp'] < MUX_RESULTS[asic_id_eeprom][i]['timestamp']:
				MUX_RESULTS[asic_id_eeprom].insert(i, mux_scan_copy)
				break
		else:
			MUX_RESULTS[asic_id_eeprom].append(mux_scan_copy)
	MUX_RESULTS_LOCK.release()

def import_qcs(qc_dir):
	logger.info("importing platform qc entries from files in directory {}".format(qc_dir))
	for fp in [os.path.join(qc_dir, fn) for fn in os.listdir(qc_dir) if fn.endswith('.json')]:
		if os.path.isfile(fp):
			with open(fp, "r") as f:
				try:
					flowcell, run_data, mux_scans = json.loads(f.read(), object_pairs_hook=OrderedDict)
				except:
					logger.warning("failed to parse {}, json format or data structure corrupt".format(fn))
					continue
			asic_id_eeprom = flowcell['asic_id_eeprom']
			add_mux_scan_results(flowcell, mux_scans)

def import_runs(base_dir, refactor=False):
	logger.info("importing sequencing run entries from files in directory {}".format(base_dir))
	for experiment in [d for d in os.listdir(base_dir) if os.path.isdir(os.path.join(base_dir, d))]:
		experiment_dir = os.path.join(base_dir, experiment)
		for sample in [d for d in os.listdir(experiment_dir) if os.path.isdir(os.path.join(experiment_dir, d))]:
			sample_dir = os.path.join(experiment_dir, sample)
			for fp in [os.path.join(sample_dir, fn) for fn in os.listdir(sample_dir) if fn.endswith('.json')]:
				if os.path.isfile(fp):
					with open(fp, "r") as f:
						try:
							flowcell, run_data, mux_scans = json.loads(f.read(), object_pairs_hook=OrderedDict)
						except:
							logger.warning("failed to parse {}, json format or data structure corrupt".format(fn))
							continue
					# temporarily change attributes experiment and sample according to directory names
					prev = (run_data['experiment'] if 'experiment' in run_data else None,
							run_data['sample'] if 'sample' in run_data else None)
					changed = prev == (experiment, sample)
					run_data['experiment'] = experiment
					run_data['sample'] = sample
					if refactor and changed:
						# make changes permanent
						logging.info("writing changes to attributes 'experiment' and 'sample' to file")
						data = (flowcell, run_data, mux_scans)
						with open( fp, 'w') as f:
							print(json.dumps(data, indent=4), file=f)

					if not add_database_entry(flowcell, run_data, mux_scans):
						logger.error("failed to add content from {} to the database".format(fp))
						continue
					# add mux scans
					add_mux_scan_results(flowcell, mux_scans)

def get_runs_by_flowcell(asic_id_eeprom):
	ALL_RUNS_LOCK.acquire()
	runs = {}
	if asic_id_eeprom:
		if asic_id_eeprom in ALL_RUNS:
			for run_id in ALL_RUNS[asic_id_eeprom]:
				if 'qc' not in ALL_RUNS[asic_id_eeprom][run_id]['run_data']['experiment_type'].lower():
					runs[run_id] = ALL_RUNS[asic_id_eeprom][run_id]
	ALL_RUNS_LOCK.release()
	return runs

def get_latest_mux_scan_result(asic_id_eeprom):
	latest_result = None
	MUX_RESULTS_LOCK.acquire()
	if asic_id_eeprom in MUX_RESULTS:
		latest_result =  MUX_RESULTS[asic_id_eeprom][0]
	MUX_RESULTS_LOCK.release()
	return latest_result

def get_latest(runs):
	latest_qc = None
	for run_id in runs:
		if latest_qc:
			_protocol_start = dateutil.parser.parse(runs[latest_qc]['run_data']['protocol_start'])
			if protocol_start > _protocol_start:
				latest_qc = run_id
		else:
			latest_qc = run_id
			protocol_start = dateutil.parser.parse(runs[run_id]['run_data']['protocol_start'])
	return latest_qc

def update_overview(watchers, output_dir):
	channel_to_css = {0:"one", 1:"two", 2:"three", 3:"four", 4:"five"}
	render_dict = {"version"		:	__version__,
				   "dateTimeNow"	:	datetime.now().strftime("%Y-%m-%d_%H:%M"),
				   "channels"		: 	[],
				   "all_exp" 		:	[]
				   }
	for watcher in watchers:
		channel = watcher.channel
		render_dict["channels"].append({})
		asic_id_eeprom = None
		try:
			asic_id_eeprom = watcher.channel_status.flowcell['asic_id_eeprom']
		except:
			pass

		runs = get_runs_by_flowcell(asic_id_eeprom)
		#qcs  = get_qcs_by_flowcell(asic_id_eeprom)

		render_dict["channels"][channel]['latest_qc'] = {}
		latest_qc = get_latest_mux_scan_result(asic_id_eeprom)
		if latest_qc:
			render_dict["channels"][channel]['latest_qc']['timestamp'] 	= latest_qc['timestamp'].date()
			render_dict["channels"][channel]['latest_qc']['total'] 		= latest_qc['total']
			if 'in_use' in latest_qc and watcher.channel_status.sequencing:
				render_dict["channels"][channel]['latest_qc']['in_use'] = latest_qc['in_use']
			else:
				render_dict["channels"][channel]['latest_qc']['in_use'] = 0

		render_dict["channels"][channel]['runs'] = []
		for run_id in runs:
			experiment = runs[run_id]['run_data']['experiment']
			if not experiment:
				if 'user_filename_input' in runs[run_id]['run_data']:
					experiment = runs[run_id]['run_data']['user_filename_input']
				if not experiment:
					logger.WARNING('not adding run with id {} to overview because no experiment name is set'.format(run_id))
			sample = runs[run_id]['run_data']['sample']
			if not sample:
				sample = experiment
			link = os.path.abspath(os.path.join(output_dir,'runs',experiment,sample,'report.html'))
			render_dict["channels"][channel]['runs'].append({'experiment':experiment,
															 'link':link})

		render_dict["channels"][channel]['channel'] = channel_to_css[watcher.channel]
		render_dict["channels"][channel]['asic_id_eeprom'] = asic_id_eeprom
		if asic_id_eeprom:
			if not latest_qc:
				render_dict["channels"][channel]['flowcell_id'] = "NO RECORDS"
			else:
				render_dict["channels"][channel]['flowcell_id'] = latest_qc['flowcell_id']
		else:
			render_dict["channels"][channel]['flowcell_id'] = '-'

	ALL_RUNS_LOCK.acquire()
	all_runs_info = []
	for asic_id_eeprom in ALL_RUNS:
		for run_id in ALL_RUNS[asic_id_eeprom]:
			experiment_type = ALL_RUNS[asic_id_eeprom][run_id]['run_data']['experiment_type']
			if not 'qc' in experiment_type.lower():
				protocol_start = dateutil.parser.parse(ALL_RUNS[asic_id_eeprom][run_id]['run_data']['protocol_start'])
				duration = "N/A"
				if 'protocol_end' in ALL_RUNS[asic_id_eeprom][run_id]['run_data']:
					if ALL_RUNS[asic_id_eeprom][run_id]['run_data']['protocol_end']:
						protocol_end = dateutil.parser.parse(ALL_RUNS[asic_id_eeprom][run_id]['run_data']['protocol_end'])
						duration = "{}".format(protocol_end - protocol_start).split('.')[0]
				sequencing_kit = ALL_RUNS[asic_id_eeprom][run_id]['run_data']['sequencing_kit']
				experiment = ALL_RUNS[asic_id_eeprom][run_id]['run_data']['experiment']
				sample = ALL_RUNS[asic_id_eeprom][run_id]['run_data']['sample']
				if not sample:
					sample = experiment
				link = os.path.abspath(os.path.join(output_dir,'runs',experiment,sample,'report.html'))
				all_runs_info.append({'link':link,
									  'experiment':experiment,
									  'sample': sample,
									  'sequencing_kit': sequencing_kit,
									  'protocol_start': protocol_start,
									  'duration': duration})
	ALL_RUNS_LOCK.release()

	if all_runs_info:
		all_runs_info = sorted(all_runs_info, key=lambda k: k['protocol_start'], reverse=True)

		run = 0
		sample = 0
		grouped = [[[all_runs_info[0]]]] if all_runs_info else [[[]]]
		for run_info in all_runs_info[1:]:
			if grouped[run][sample][0]['experiment'] == run_info['experiment']:
				if grouped[run][sample][0]['sample'] == run_info['sample']:
					grouped[run][sample].append(run_info)
				else:
					grouped[run].append( [run_info] )
					sample += 1
			else:
				grouped.append( [[run_info]] )
				run += 1
				sample = 0

		for exp in grouped:
			render_dict['all_exp'].append(
				{'num_samples':str(sum([len(sample) for sample in exp])),
				 'experiment':exp[0][0]['experiment'],
				 'samples':[]})
			for sample in exp:
				render_dict['all_exp'][-1]['samples'].append(
					{'num_runs':str(len(sample)),
					 'link':sample[0]['link'],
					 'sample':sample[0]['sample'],
					 'runs':[]})
				for run in sample:
					render_dict['all_exp'][-1]['samples'][-1]['runs'].append(run)

	template = jinja_env.get_template('overview.template')
	with open(os.path.join(output_dir, "{}_overview.html".format(hostname)), 'w') as f:
		print(template.render(render_dict), file=f)

class ChannelStatus():
	empty_run_data = OrderedDict([
		('run_id', None),
		('minion_id', None),
		('sequencing_kit', None),
		('protocol_start', None),
		('protocol_end', None),
		('relative_path', None),
		('sample', None),
		('experiment', None)
		])

	empty_flowcell = OrderedDict([
		('flowcell_id', None),
		('asic_id', None),
		('asic_id_eeprom', None),
		('flowcell', None)
		])

	empty_mux = OrderedDict()

	def __init__(self, minion_id, channel):
		self.minion_id = minion_id
		self.flowcell = copy.deepcopy(self.empty_flowcell)
		self.run_data = copy.deepcopy(self.empty_run_data)
		self.mux_scans = []
		self.run_data['minion_id'] = minion_id
		self.logger = logging.getLogger(name='gw.w{}.cs'.format(channel+1))
		self.sequencing = False

	def update(self, content, overwrite=False):
		for key in content:
			if key in self.flowcell:
				if self.flowcell[key]:
					if overwrite:
						self.logger.info("changing the current value of {} ({}) to {}".format(key, self.flowcell[key], content[key]))
						self.flowcell[key] = content[key]
					else:
						self.logger.debug("not changing the current value of {} ({}) to {}".format(key, self.flowcell[key], content[key]))
					continue
				else:
					self.flowcell[key] = content[key]
					self.logger.info("new flowcell value for {} : {}".format(key, content[key]))
					continue
			elif key in self.run_data:
				if self.run_data[key]:
					if overwrite:
						self.logger.info("changing the current value of {} ({}) to {}".format(key, self.run_data[key], content[key]))
						self.run_data[key] = content[key]
					else:
						self.logger.debug("not changing the current value of {} ({}) to {}".format(key, self.run_data[key], content[key]))
					continue
			self.run_data[key] = content[key]
			self.logger.info("new run value for {} : {}".format(key, content[key]))

	def add_mux_scan(self, timestamp, active_pores, in_use=None):
		self.mux_scans.append(copy.deepcopy(self.empty_mux))
		self.mux_scans[-1]['timestamp'] = timestamp
		self.mux_scans[-1]['total'] = active_pores
		if in_use:
			self.mux_scans[-1]['in_use'] = in_use
		add_mux_scan_results(self.flowcell, [self.mux_scans[-1]])
		self.logger.debug("added new mux scan result")

	def flowcell_disconnected(self):
		self.logger.info("resetting flowcell and run data")
		self.flowcell = copy.deepcopy(self.empty_flowcell)
		self.run_data = copy.deepcopy(self.empty_run_data)
		self.run_data['minion_id'] = self.minion_id
		self.mux_scans = []
		self.sequencing = False

	def reset_channel(self):
		self.logger.info("resetting run data")
		self.run_data = copy.deepcopy(self.empty_run_data)
		self.run_data['minion_id'] = self.minion_id
		self.mux_scans = []
		self.sequencing = False

class WatchnchopScheduler(threading.Thread):
	def __init__(self, data_basedir, relative_path, experiment, sequencing_kit, fastq_reads_per_file,
				 bc_kws, stats_fp, channel, watchnchop_args, min_length, min_length_rna):
		threading.Thread.__init__(self)
		if getattr(self, 'daemon', None) is None:
			self.daemon = True
		else:
			self.setDaemon(True)
		self.stoprequest = threading.Event()	# set when joined without timeout (eg if terminated with ctr-c)
		self.exp_end = threading.Event()			# set when joined with timeout (eg if experiment ended)
		self.logger = logging.getLogger(name='gw.w{}.wcs'.format(channel+1))

		self.observed_dir = os.path.join(data_basedir, relative_path, 'fastq_pass')
		# define the command that is to be executed
		self.cmd = [which('perl'),
					which('watchnchop'),
					'-o', stats_fp,
					'-f', str(fastq_reads_per_file)]
		if watchnchop_args:
			self.cmd.extend(watchnchop_args)
		for kw in bc_kws:
			if kw.lower() in experiment.lower() or kw.lower() in sequencing_kit.lower():
				self.cmd.append('-b')
				break
		self.cmd.append('-l')
		if 'rna' in experiment.lower() or 'rna' in sequencing_kit.lower():
			self.cmd.append(str(min_length_rna))
		else:
			self.cmd.append(str(min_length))
		self.cmd.append(os.path.join(data_basedir, relative_path, ''))
		self.process = None

	def run(self):
		self.logger.info("STARTED watchnchop scheduler")
		while not (self.stoprequest.is_set() or self.exp_end.is_set()):
			if self.conditions_met():
				self.process = subprocess.Popen(self.cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, creationflags=subprocess.IDLE_PRIORITY_CLASS)
				self.logger.info("STARTED WATCHNCHOP with arguments: {}".format(self.cmd))
				break
			time.sleep(1)
		while not (self.stoprequest.is_set() or self.exp_end.is_set()):
			time.sleep(1)
		if self.process:
			try:
				self.process.terminate()
				self.logger.info("TERMINATED watchnchop process")
			except:
				self.logger.error("TERMINATING watchnchop process failed")
		else:
			if self.stoprequest.is_set():
				self.logger.error("watchnchop was NEVER STARTED: this thread was ordered to kill the watchnchop subprocess before it was started")
				return

			# try one last time to start watchnchop (necessary for runs with extremly low output, where all reads are buffered)
			self.logger.info("starting watchnchop in one minutes, then kill it after another 5 minutes")
			for i in range(60):
				if self.stoprequest.is_set():
					self.logger.error("watchnchop was NEVER STARTED: this thread was ordered to kill the watchnchop subprocess before it was started")
					return
				time.sleep(1)
			if self.conditions_met():
				self.process = subprocess.Popen(self.cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, creationflags=subprocess.IDLE_PRIORITY_CLASS)
				self.logger.info("STARTED WATCHNCHOP with arguments: {}".format(self.cmd))
			else:
				self.logger.error("watchnchop NOT STARTED: directory {} still does not exist or contains no fastq files".format(self.observed_dir))
				return
			for i in range(300):
				if self.stoprequest.is_set():
					break
				time.sleep(1)
			self.process.terminate()
			self.logger.info("TERMINATED watchnchop process")

	def conditions_met(self):
		if os.path.exists(self.observed_dir):
			if [fn for fn in os.listdir(self.observed_dir) if fn.endswith('.fastq')]:
				return True
		return False

	def join(self, timeout=None):
		if timeout:
			self.exp_end.set()
		else:
			self.stoprequest.set()
		super(WatchnchopScheduler, self).join(timeout)

class StatsparserScheduler(threading.Thread):

	def __init__(self, update_interval, sample_dir, statsparser_args, channel):
		threading.Thread.__init__(self)
		if getattr(self, 'daemon', None) is None:
			self.daemon = True
		else:
			self.setDaemon(True)
		self.stoprequest = threading.Event()	# set when joined without timeout (eg if terminated with ctr-c)
		self.exp_end = threading.Event()		# set when joined with timeout (eg if experiment ended)
		self.logger = logging.getLogger(name='gw.w{}.sps'.format(channel+1))
		self.channel = channel

		self.update_interval = update_interval
		self.sample_dir = sample_dir
		self.statsparser_args = statsparser_args
		self.page_opened = False

	def run(self):
		while not self.stoprequest.is_set() or self.exp_end.is_set():
			last_time = time.time()

			if self.conditions_met():
				self.update_report()

			this_time = time.time()
			while (this_time - last_time < self.update_interval) and not self.stoprequest.is_set() or self.exp_end.is_set():
				time.sleep(1)
				this_time = time.time()
		# start statsparser a last time if the experiment ended
		if not self.stoprequest.is_set() and self.conditions_met():
			self.update_report()

		SP_DIRS_LOCK.acquire()
		if self.sample_dir in SP_DIRS:
			if SP_DIRS[self.sample_dir] == self.channel:
				del SP_DIRS[self.sample_dir]
		SP_DIRS_LOCK.release()

	def conditions_met(self):
		conditions_met = False
		stats_fns = [fn for fn in os.listdir(os.path.abspath(self.sample_dir)) if fn.endswith('stats.csv')] if os.path.exists(os.path.abspath(self.sample_dir)) else []
		# assure that only one statsparser instance is running on a directory at a time
		SP_DIRS_LOCK.acquire()
		if not self.sample_dir in SP_DIRS:
			SP_DIRS[self.sample_dir] = self.channel
		if stats_fns and SP_DIRS[self.sample_dir] == self.channel:
			conditions_met = True
		SP_DIRS_LOCK.release()
		return conditions_met

	def update_report(self):
		self.logger.info("updating report...")
		cmd = [os.path.join(get_script_dir(),'statsparser'), #TODO: change to which() ?
			   self.sample_dir,
			   '-q']
		cmd.extend(self.statsparser_args)
		cp = subprocess.run(cmd) # waits for process to complete
		if cp.returncode == 0:
			if not self.page_opened:
				basedir = os.path.abspath(self.sample_dir)
				fp = os.path.join(basedir, 'report.html')
				self.logger.info("OPENING " + fp)
				try:
					webbrowser.open('file://' + os.path.realpath(fp))
				except:
					pass
				self.page_opened = True
		else:
			self.logger.warning("statsparser returned with errorcode {} for directory {}".format(cp.returncode, self.sample_dir))

	def join(self, timeout=None):
		if timeout:
			self.exp_end.set()
		else:
			self.stoprequest.set()
		super(StatsparserScheduler, self).join(timeout)

class Watcher():

	def __init__(self, minknow_log_basedir, channel, ignore_file_modifications, output_dir, data_basedir,
				 statsparser_args, update_interval, watchnchop_args, min_length, min_length_rna, bc_kws):
		self.q = queue.PriorityQueue()
		self.watchnchop_args = watchnchop_args
		self.min_length = min_length
		self.min_length_rna = min_length_rna
		self.channel = channel
		self.output_dir = output_dir
		self.data_basedir = data_basedir
		self.statsparser_args = statsparser_args
		self.update_interval = update_interval
		self.bc_kws = bc_kws
		self.observed_dir = os.path.join(minknow_log_basedir, "GA{}0000".format(channel+1))
		self.event_handler = LogFilesEventHandler(self.q, ignore_file_modifications, channel)
		self.observer = Observer()
		self.observer.schedule(self.event_handler,
							   self.observed_dir,
							   recursive=False)
		self.observer.start()
		self.channel_status = ChannelStatus("GA{}0000".format(channel+1), channel)
		self.spScheduler = None
		self.wcScheduler = []
		self.logger = logging.getLogger(name='gw.w{}'.format(channel+1))

		self.logger.info("...watcher for {} ready".format(self.observed_dir))

	def check_q(self):
		# checking sheduler queue
		if not self.q.empty():
			self.logger.debug("Queue content for {}:".format(self.observed_dir))
		while not self.q.empty():
			timestamp, origin, line = self.q.get()
			self.logger.debug("received '{}' originating from '{} log' at '{}'".format(line, origin, timestamp))

			if origin == 'server':
				self.parse_server_log_line(line)
			elif origin == 'bream':
				self.parse_bream_log_line(line)
			#elif origin == 'analyser':
			#	self.parse_analyser_log_line(line)

	def parse_server_log_line(self, line):
		dict_content = {}
		overwrite = False
		timestamp = line[:23]

		# fetch output_path, run_id, script_path, relative_path, protocol_start, flowcell_id [, experiment, sample]
		if   	"protocol_started"										in line:
			for m in re.finditer('([^\s,]+) = ([^\s,]+)', line):
				dict_content[m.group(1)] = m.group(2)
			overwrite = True
			dict_content['relative_path'] = dict_content['output_path'].split("/./")[1].strip("/")
			subdirs = dict_content['relative_path'].split('/')
			if len(subdirs) == 3:
				# case sequencing protocol
				dict_content['experiment'] = subdirs[0]
				dict_content['sample'] = subdirs[1]
				dict_content['flowcell_id'] = subdirs[2].split('_')[3]
			elif len(subdirs) == 1:
				# case qc protocol
				dict_content['flowcell_id'] = subdirs[0].split('_')[3]
			self.logger.info("PROTOCOL START")
			set_update_overview()
			self.channel_status.run_data['protocol_start'] = timestamp

		# fetch protocol_end
		elif	"protocol_finished" 									in line:
			self.logger.info("PROTOCOL END")
			set_update_overview()
			self.channel_status.run_data['protocol_end'] = timestamp
			if self.channel_status.mux_scans:
				self.save_logdata()
			self.channel_status.reset_channel()
			self.stop_statsparser()
			self.stop_watchnchop()

		#
		elif	"[engine/info]: : flowcell_discovered" 					in line:
			for m in re.finditer('([^\s,]+) = ([^\s,]+)', line):
				dict_content[m.group(1)] = m.group(2)
			overwrite = True
			self.logger.info("FLOWCELL DISCOVERED")
			set_update_overview()
			self.channel_status.flowcell_disconnected()
			self.stop_statsparser()
			self.stop_watchnchop()

		elif   	"[engine/info]: : data_acquisition_started"				in line:
			for m in re.finditer('([^\s,]+) = ([^\s,]+)', line):
				dict_content[m.group(1)] = m.group(2)
			overwrite = True

		elif	"flowcell_disconnected"									in line:
			self.logger.info("FLOWCELL DISCONNECTED")
			set_update_overview()
			self.channel_status.flowcell_disconnected()

		elif 	"pores available for sequencing" 						in line:
			active_pores = None
			in_use = None
			for m in re.finditer("has ([0-9]+) pores available for sequencing", line):
				active_pores = m.group(1)
			for m in re.finditer("Starting sequencing with ([0-9]+) pores", line):
				in_use = m.group(1)
			self.logger.info("new mux scan result: {} active, {} in use".format(active_pores, in_use))
			self.channel_status.add_mux_scan(timestamp, active_pores, in_use=in_use)
			set_update_overview()
			self.save_logdata()

		if dict_content:
			self.channel_status.update(dict_content, overwrite)

	def parse_bream_log_line(self, line):
		dict_content = {}
		overwrite = False
		timestamp = line.split(" - ")[1]

		if 		"INFO - Attribute"								in line:
			for m in re.finditer("([^\s,]+) set to (.+)", line):
				dict_content[m.group(1)] = m.group(2)

		elif 	"INFO - Asked to start protocol"				in line:
			for m in re.finditer("'--([^\s,]+)=([^\s,]+)'", line):
				dict_content[m.group(1)] = m.group(2)
				overwrite = True

		elif 	"INFO - Updating context tags in MinKNOW with" 	in line:
			for m in re.finditer("'([^\s,]+)'[:,] u?'([^\s,]+)'", line):
				dict_content[m.group(1)] = m.group(2)
			if 'sequencing_kit' in dict_content:
				dict_content['sequencing_kit'] = dict_content['sequencing_kit'].upper()

		elif	"platform_qc.report"			in line:
			self.logger.info("QC FINISHED")

		elif	"sequencing.start"										in line:
			dict_content["sequencing_start_time"] = timestamp
			self.logger.info("SEQUENCING STARTS")
			self.channel_status.sequencing = True
			set_update_overview()

			self.start_watchnchop()
			self.start_statsparser()

		if dict_content:
			self.channel_status.update(dict_content, overwrite)

	def check_attributes(self, attributes):
		for key in attributes:
			if key in self.channel_status.run_data:
				if self.channel_status.run_data[key]:
					continue
				else:
					return key
			elif key in self.channel_status.flowcell:
				if self.channel_status.flowcell[key]:
					continue
				else:
					return key
			else:
				return key
		return None

	def save_logdata(self):
		missing_key = self.check_attributes(['experiment_type', 'run_id', 'flowcell_id', 'asic_id_eeprom'])
		if missing_key:
			self.logger.warning("NOT SAVING REPORT for {} because the crucial attribute '{}' is missing".format(self.channel_status.run_data['run_id'], missing_key))
			return

		fn = []
		if "qc" in self.channel_status.run_data['experiment_type'].lower():
			missing_key = self.check_attributes(['experiment', 'sample'])
			if not missing_key:
				self.logger.warning("NOT SAVING REPORT for {} because it is not certain that this is a qc run".format(self.channel_status.run_data['run_id']))
				return
			fn.extend(["QC", self.channel_status.flowcell['flowcell_id'], self.channel_status.run_data['run_id']])
			target_dir = os.path.join(self.output_dir, 'qc')
		else:
			missing_key = self.check_attributes(['experiment', 'sample'])
			if missing_key:
				self.logger.warning("NOT SAVING REPORT for {} because the crucial attribute '{}' is missing".format(self.channel_status.run_data['run_id'], missing_key))
				return
			fn.extend([self.channel_status.run_data['run_id'], 'logdata'])
			target_dir = os.path.join(self.output_dir,
									  'runs',
									  self.channel_status.run_data['experiment'],
									  self.channel_status.run_data['sample'])
		fn = "_".join(fn) + ".json"

		self.logger.info("saving log data to file {}".format(os.path.join(target_dir, fn)))
		data = (self.channel_status.flowcell, self.channel_status.run_data, self.channel_status.mux_scans)
		if not os.path.exists(target_dir):
			os.makedirs(target_dir)
		with open( os.path.join(target_dir, fn), 'w') as f:
			print(json.dumps(data, indent=4), file=f)

		ALL_RUNS_LOCK.acquire()
		run_id = self.channel_status.run_data['run_id']
		asic_id_eeprom = self.channel_status.flowcell['asic_id_eeprom']
		if asic_id_eeprom in ALL_RUNS:
			ALL_RUNS[asic_id_eeprom][run_id] = {'flowcell': data[0],
												'run_data': data[1],
												'mux_scans': data[2]}
		else:
			ALL_RUNS[asic_id_eeprom] = {}
			ALL_RUNS[asic_id_eeprom][run_id] = {'flowcell': data[0],
												'run_data': data[1],
												'mux_scans': data[2]}
		ALL_RUNS_LOCK.release()

	def start_watchnchop(self):
		missing_key = self.check_attributes(['experiment', 'sample', 'sequencing_kit', 'run_id', 'fastq_reads_per_file', 'relative_path'])
		if missing_key:
			self.logger.warning("NOT executing watchnchop because the crucial attribute '{}' is missing".format(missing_key))
			return

		self.stop_watchnchop()

		stats_fp = os.path.join(self.output_dir,
								'runs',
								self.channel_status.run_data['experiment'],
								self.channel_status.run_data['sample'],
								"{}_stats.csv".format(self.channel_status.run_data['run_id']))
		self.wcScheduler.append(WatchnchopScheduler(self.data_basedir,
													self.channel_status.run_data['relative_path'],
													self.channel_status.run_data['experiment'],
													self.channel_status.run_data['sequencing_kit'],
													self.channel_status.run_data['fastq_reads_per_file'],
													self.bc_kws,
													stats_fp,
													self.channel,
													self.watchnchop_args,
													self.min_length,
													self.min_length_rna))
		self.wcScheduler[-1].start()
		return

	def stop_watchnchop(self, timeout=1.2):
		if self.wcScheduler[-1].is_alive() if self.wcScheduler else None:
			if timeout:
				self.wcScheduler[-1].join(timeout)
			else:
				self.wcScheduler[-1].join()

	def start_statsparser(self):
		missing_key = self.check_attributes(['experiment', 'sample'])
		if missing_key:
			self.logger.warning("NOT starting statsparser scheduler because the crucial attribute '{}' is missing".format(missing_key))
			return

		#start creation of plots at regular time intervals
		self.stop_statsparser()
		sample_dir = os.path.join(self.output_dir,
								  'runs',
								  self.channel_status.run_data['experiment'],
								  self.channel_status.run_data['sample'])
		self.logger.info('SCHEDULING update of report for sample {1} every {0:.1f} minutes'.format(self.update_interval/1000, sample_dir))
		self.spScheduler = StatsparserScheduler(self.update_interval,
												sample_dir,
												self.statsparser_args,
												self.channel)
		self.spScheduler.start()

	def stop_statsparser(self, timeout=1.2):
		if self.spScheduler.is_alive() if self.spScheduler else None:
			if timeout:
				self.spScheduler.join(timeout)
			else:
				self.spScheduler.join()

class OpenedFilesHandler():
	'''manages a set of opened files, reads their contents and
	processes them line by line. Incomplete lines are stored until
	they are "completed" by a newline character.'''
	def __init__(self, channel):
		self.logger = logging.getLogger(name='gw.w{}.ofh'.format(channel+1))
		self.open_files = {}

	def open_new_file(self, path):
		self.logger.info("Opening file {}".format(path))
		self.open_files[path] = [open(path, 'r'), ""]

	def close_file(self, path):
		self.logger.debug("Attempting to close file {}".format(path))
		try:
			self.open_files[path][0].close()
		except:
			self.logger.debug("File handle of file {} couldn't be closed".format(path))
		if path in self.open_files:
			del self.open_files[path]
			self.logger.debug("Deleted entry in open_files for file {}".format(path))

	def process_lines_until_EOF(self, process_function, path):
		file = self.open_files[path][0]
		while 1:
			line = file.readline()
			if line == "":
				break
			elif line.endswith("\n"):
				line = (self.open_files[path][1] + line).strip()
				if line:
					process_function(line)
				self.open_files[path][1] = ""
			else:
				#line potentially incomplete
				self.open_files[path][1] = self.open_files[path][1] + line

class LogFilesEventHandler(FileSystemEventHandler):
	control_server_log, bream_log = None, None

	def __init__(self, q, ignore_file_modifications, channel):
		super(LogFilesEventHandler, self).__init__()
		self.ignore_file_modifications = ignore_file_modifications
		self.file_handler = OpenedFilesHandler(channel)
		self.comm_q = q

		# while no server log file is opened, all lines read are buffered in a seperate Priority Queue
		self.buff_q = queue.PriorityQueue()
		self.q = self.buff_q
		self.logger = logging.getLogger(name='gw.w{}.lfeh'.format(channel+1))

	def on_moved(self, event):
		pass

	def on_created(self, event):
		if not event.is_directory:
			activate_q = False
			self.logger.debug("File {} was created".format(event.src_path))
			basename = os.path.basename(event.src_path)
			if basename.startswith("control_server_log"):
				if self.control_server_log:
					self.file_handler.close_file(event.src_path)
					self.logger.info("Replacing current control_server_log file {} with {}".format(self.control_server_log, event.src_path))
				else:
					# read lines of server file first, then activate the real communication q
					activate_q = True
				self.control_server_log = event.src_path
				self.logger.info("New control_server_log file {}".format(self.control_server_log))
				process_function = self.enqueue_server_log_line
			elif basename.startswith("bream") and basename.endswith(".log"):
				if self.bream_log:
					self.file_handler.close_file(event.src_path)
					self.logger.info("Replacing current bream_log file {} with {}".format(self.bream_log, event.src_path))
				self.bream_log = event.src_path
				self.logger.info("New bream_log file {}".format(self.bream_log))
				process_function = self.enqueue_bream_log_line
			else:
				self.logger.debug("File {} is not of concern for this tool".format(event.src_path))
				return
			self.file_handler.open_new_file(event.src_path)
			self.file_handler.process_lines_until_EOF(process_function, event.src_path)
			self.logger.info("approx. queue size: {}".format(self.q.qsize()))
			if activate_q:
				self.activate_q()

	def on_deleted(self, event):
		if not event.is_directory:
			self.logger.debug("File {} was deleted".format(event.src_path))
			#self.file_handler.close_file(event.src_path)
			if self.control_server_log == event.src_path:
				control_server_log = None
				self.logger.warning("Current control_server_log file {} was deleted!".format(event.src_path))
			elif self.bream_log == event.src_path:
				self.bream_log = None
				self.logger.warning("Current bream_log file {} was deleted".format(event.src_path))
			else:
				self.logger.debug("File {} is not opened and is therefore not closed.".format(event.src_path))
			self.file_handler.close_file(event.src_path)

	def on_modified(self, event):
		if not event.is_directory:
			self.logger.debug("File {} was modified".format(event.src_path))
			if event.src_path in self.file_handler.open_files:
				if self.control_server_log == event.src_path:
					self.file_handler.process_lines_until_EOF(self.enqueue_server_log_line, event.src_path)
				elif self.bream_log == event.src_path:
					self.file_handler.process_lines_until_EOF(self.enqueue_bream_log_line, event.src_path)
				else:
					self.logger.warning("case not handled")
					return
			else:
				if not self.ignore_file_modifications:
					self.on_created(event)
				else:
					self.logger.debug("File {} existed before this script was started".format(event.src_path))

	def activate_q(self):
		self.logger.info("activating communication queue")
		self.q = self.comm_q
		while not self.buff_q.empty():
			self.q.put(self.buff_q.get())

	def enqueue_server_log_line(self, line):
		try:
			self.q.put( (dateutil.parser.parse(line[:23]), 'server', line) )
		except:
			self.logger.debug("the timestamp of the following line in the server log file could not be parsed:\n{}".format(line))

	def enqueue_bream_log_line(self, line):
		try:
			self.q.put( (dateutil.parser.parse(line.split(' - ')[1]), 'bream', line) )
		except:
			self.logger.debug("the timestamp of the following line in the bream log file could not be parsed:\n{}".format(line))

class RunsDirsEventHandler(FileSystemEventHandler):

	def __init__(self, observed_dir):
		super(RunsDirsEventHandler, self).__init__()
		self.observed_dir = os.path.abspath(observed_dir)
		self.logger = logging.getLogger(name='gw.reh')

	def on_moved(self, event):
		if event.is_directory or (self.depth(event.src_path) == 3 and event.src_path.endswith('.json')):
			self.logger.debug("moved {}, depth {}, \ndest {}".format(event.src_path, self.depth(event.src_path), event.dest_path))
			if self.observed_dir in event.dest_path and self.depth(event.dest_path) == self.depth(event.src_path):
				self.reload_runs()
			else:
				self.on_deleted(event)

	def on_created(self, event):
		if event.is_directory:
			self.logger.debug("created directory {}, depth {}".format(event.src_path, self.depth(event.src_path)))
			if 1 <= self.depth(event.src_path) <= 2:
				self.reload_runs()
		elif self.depth(event.src_path) == 3 and event.src_path.endswith('.json'):
			self.logger.debug("created file {}, depth {}".format(event.src_path, self.depth(event.src_path)))
			self.reload_runs()

	def on_modified(self, event):
		if event.is_directory:
			self.logger.debug("modified directory {}, depth {}".format(event.src_path, self.depth(event.src_path)))

	def on_deleted(self, event):
		if event.is_directory:
			self.logger.debug("deleted directory {}, depth {}".format(event.src_path, self.depth(event.src_path)))
			if 1 <= self.depth(event.src_path) <= 2:
				self.reload_runs()
		elif self.depth(event.src_path) == 3 and event.src_path.endswith('.json'):
			self.logger.debug("deleted file {}, depth {}".format(event.src_path, self.depth(event.src_path)))
			self.reload_runs()

	def depth(self, src_path):
		src_path = os.path.abspath(src_path)
		return len(src_path.replace(self.observed_dir, '').strip('/').split('/'))

	def reload_runs(self):
		ALL_RUNS_LOCK.acquire()
		self.logger.info('deleting and re-importing all runs due to changes in the run directory')
		# delete sequencing runs
		to_delete = []
		for asic_id_eeprom in ALL_RUNS:
			for run_id in ALL_RUNS[asic_id_eeprom]:
				if 'qc' not in ALL_RUNS[asic_id_eeprom][run_id]['run_data']['experiment_type']:
					to_delete.append( (asic_id_eeprom, run_id) )
		for asic_id_eeprom, run_id in to_delete:
			del ALL_RUNS[asic_id_eeprom][run_id]
		#reload runs
		import_runs(self.observed_dir)
		ALL_RUNS_LOCK.release()
		set_update_overview()
		return

def standalone():
	args = parse_args()
	main(args)

if __name__ == "__main__":
	standalone()


import sys
import pymysql
import pymongo
import re
import itertools
import pickle as pkl
import pandas as pd
from pymongo import MongoClient
from collections import defaultdict

from util.preprocessing import *

client = MongoClient('localhost', 27017)
database = client['research']
ptt_posts = database['2018_ptt_posts']
ptt_objects = database['2018_ptt_objects']
ptt_comments = database['2018_ptt_comments']

if __name__ == "__main__":
    total_document_len = 0
    total_document_count = 0
    document_freq = defaultdict(int)

    cnt = 0
    for post in ptt_posts.find(no_cursor_timeout=True):
        sys.stdout.write(f'\r{cnt}')
        cnt += 1

        post['comments'] = list(ptt_comments.find({'parentID': post['uniID']}, no_cursor_timeout=True))
        post['comments_count'] = len(post['comments'])

        content = post['content']
        if str(content) != 'nan' and content != None:

            sentences = to_sentence(content)
            tokenized_content = tokenize(sentences, load_stopwords(), re.compile('[\Wa-zA-Z0-9]+'))
            post['sentence'] = " ".join(sentences)
            post['tokenized_content'] = tokenized_content
            post['keywords'] = defaultdict(int)
            for term in itertools.chain.from_iterable(tokenized_content):
                document_freq[term] += 1
                post['keywords'][term] += 1
            post['words_count'] = len(post['keywords'])

            total_document_len += post['words_count']
            total_document_count += 1

            ptt_posts.update_one({'_id': post['_id']}, {'$set': post}, upsert=False)

    with open('document_freq.pkl', 'wb') as f:
        pkl.dump(document_freq, f)

    with open('ptt_log.txt', 'w') as f:
        f.write(f'total_document_length: {total_document_len}\n')
        f.write(f'total_document_count: {total_document_count}\n')
        f.write(f'avg_document_length: {total_document_len / total_document_count}\n')


#!/usr/bin/env python
# -*- coding: utf-8 -*-
""" Test that the ``Oxentiel`` class loads dictionaries correctly. """
from typing import Dict, Any

from hypothesis import given

from oxentiel import Oxentiel
from oxentiel.tests import strategies

# pylint: disable=no-value-for-parameter

@given(strategies.settings_dicts())
def test_ox_adds_all_keys_from_nested_dicts(settings: Dict[str, Any]) -> None:
    """ Test that all keys are added when the dictionary is nested. """
    ox = Oxentiel(settings)

    def check_keys(mapping: Dict[str, Any], ox: Oxentiel) -> None:
        """ Recursively add all keys from a nested dictionary. """
        for key, value in mapping.items():
            if isinstance(value, dict):
                check_keys(value, getattr(ox, key))
            assert key in ox.keys()

    check_keys(settings, ox)

@given(strategies.settings_dicts())
def test_ox_attributes_get_set(settings: Dict[str, Any]) -> None:
    """ Test that all keys are set as attributes. """
    ox = Oxentiel(settings)

    def check_attributes(mapping: Dict[str, Any], ox: Oxentiel) -> None:
        """ Recursively add all keys from a nested dictionary. """
        for key, value in mapping.items():
            if isinstance(value, dict):
                check_attributes(value, getattr(ox, key))
            assert hasattr(ox, key)

    check_attributes(settings, ox)

def test_ox_settings_passed_by_value() -> None:
    """ Test that modifying ``Oxentiel.settings`` doesn't change the argument dict. """
    settings = {"key": {"subkey": [1, 2]}}
    ox = Oxentiel(settings)
    settings["key"]["subkey"].append(3)
    assert 3 not in ox.key.subkey

@given(strategies.settings_dicts())
def test_ox_repr_prints_everything(settings: Dict[str, Any]) -> None:
    """ Test that every key appears in the string representation. """
    ox_repr = repr(Oxentiel(settings))
    print(ox_repr)
    for key in settings:
        assert repr(key) in ox_repr


from aiogram import types
from aiogram.dispatcher.filters.builtin import CommandHelp

from loader import dp

@dp.message_handler(CommandHelp())
async def bot_help(message: types.Message):
    text = ("Список команд: ",
            "/start - Начать диалог",
            "/help - Получить справку",
            "/ref - Получить информацию о реферальной программе",
            "/cancel - Выйти из режима покупки товара, если что-то пошло не так",
            "\nДля выборов товаров вы можете использовать инлайн-режим.\n",
            "Для этого введите в любом диалоге: @имя_бота")

    await message.answer("\n".join(text))


import pandas as pd
import numpy as np
import random
import time
import os
import gc
import pickle

from sklearn.model_selection import StratifiedKFold
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import roc_auc_score

import lightgbm as lgb

import matplotlib.pyplot as plt
import seaborn as sns

import warnings
warnings.simplefilter('ignore')

N_SPLITS = 5
N_ESTIMATORS = 20000
EARLY_STOPPING_ROUNDS = 200
VERBOSE = 1000
SEED = 2021

def seed_everything(seed=42):
    random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    np.random.seed(seed)

seed_everything(SEED)

train = pd.read_csv("train.csv")
test = pd.read_csv("test.csv")
submission = pd.read_csv(INPUT + "sample_solution.csv")

features = [col for col in test.columns if 'f' in col]
TARGET = 'claim'

target = train[TARGET].copy()

train['n_missing'] = train[features].isna().sum(axis=1)
test['n_missing'] = test[features].isna().sum(axis=1)

train['std'] = train[features].std(axis=1)
test['std'] = test[features].std(axis=1)

features += ['n_missing', 'std']
n_missing = train['n_missing'].copy()

train[features] = train[features].fillna(train[features].mean())
test[features] = test[features].fillna(test[features].mean())

scaler = StandardScaler()
train[features] = scaler.fit_transform(train[features])
test[features] = scaler.transform(test[features])

lgb_params = {
    'objective': 'binary',
    'n_estimators': N_ESTIMATORS,
    'random_state': SEED,
    'learning_rate': 5e-3,
    'subsample': 0.6,
    'subsample_freq': 1,
    'colsample_bytree': 0.4,
    'reg_alpha': 10.0,
    'reg_lambda': 1e-1,
    'min_child_weight': 256,
    'min_child_samples': 20,
    'importance_type': 'gain',
}

lgb_oof = np.zeros(train.shape[0])
lgb_pred = np.zeros(test.shape[0])
lgb_importances = pd.DataFrame()

skf = StratifiedKFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)

for fold, (trn_idx, val_idx) in enumerate(skf.split(X=train, y=n_missing)):
    print(f"===== fold {fold} =====")
    X_train = train[features].iloc[trn_idx]
    y_train = target.iloc[trn_idx]
    X_valid = train[features].iloc[val_idx]
    y_valid = target.iloc[val_idx]
    X_test = test[features]

    start = time.time()
    model = lgb.LGBMClassifier(**lgb_params)
    model.fit(
        X_train,
        y_train,
        eval_set=[(X_valid, y_valid)],
        eval_metric='auc',
        early_stopping_rounds=EARLY_STOPPING_ROUNDS,
        verbose=VERBOSE,
    )

    fi_tmp = pd.DataFrame()
    fi_tmp['feature'] = model.feature_name_
    fi_tmp['importance'] = model.feature_importances_
    fi_tmp['fold'] = fold
    fi_tmp['seed'] = SEED
    lgb_importances = lgb_importances.append(fi_tmp)

    lgb_oof[val_idx] = model.predict_proba(X_valid)[:, -1]
    lgb_pred += model.predict_proba(X_test)[:, -1] / N_SPLITS

    elapsed = time.time() - start
    auc = roc_auc_score(y_valid, lgb_oof[val_idx])
    print(f"fold {fold} - lgb auc: {auc:.6f}, elapsed time: {elapsed:.2f}sec\n")

print(f"oof lgb roc = {roc_auc_score(target, lgb_oof)}")

np.save("lgb_oof.npy", lgb_oof)
np.save("lgb_pred.npy", lgb_pred)

submission[TARGET] = lgb_pred
submission.to_csv("submission.csv", index=False)

with open('model_LGBMClassifier.pkl','wb') as file:
    file.write(pickle.dumps(model))


import mock
import uuid
from contextlib import contextmanager

import webtest
from pyramid.config import Configurator

from cliquet.events import (ResourceChanged, AfterResourceChanged,
                            ResourceRead, AfterResourceRead, ACTIONS)
from cliquet.storage.exceptions import BackendError
from cliquet.tests.testapp import main as testapp
from cliquet.tests.support import unittest, BaseWebTest, get_request_class
from cliquet import statsd

@contextmanager
def notif_broken(app, event_cls):
    old = app.registry.notify

    def buggy(event):
        if not isinstance(event, event_cls):
            return old(event)
        raise Exception("boom")

    app.registry.notify = buggy
    yield
    app.registry.notify = old

class BaseEventTest(BaseWebTest):

    subscribed = tuple()

    def setUp(self):
        super(BaseEventTest, self).setUp()
        self.events = []
        self.body = {'data': {'name': 'de Paris'}}

    def tearDown(self):
        self.events = []
        super(BaseEventTest, self).tearDown()

    def listener(self, event):
        self.events.append(event)

    def make_app(self, settings=None):
        settings = self.get_app_settings(settings)
        self.config = Configurator(settings=settings)
        for event_cls in self.subscribed:
            self.config.add_subscriber(self.listener, event_cls)
        self.config.commit()
        app = testapp(config=self.config)
        app = webtest.TestApp(app)
        app.RequestClass = get_request_class(self.api_prefix)
        return app

class ResourceReadTest(BaseEventTest, unittest.TestCase):

    subscribed = (ResourceRead,)

    def test_get_sends_read_event(self):
        resp = self.app.post_json(self.collection_url, self.body,
                                  headers=self.headers, status=201)
        record_id = resp.json['data']['id']
        record_url = self.get_item_url(record_id)
        self.app.get(record_url, headers=self.headers)
        self.assertEqual(len(self.events), 1)
        self.assertEqual(self.events[0].payload['action'], ACTIONS.READ.value)
        self.assertEqual(len(self.events[0].read_records), 1)

    def test_collection_get_sends_read_event(self):
        self.app.get(self.collection_url, headers=self.headers)
        self.assertEqual(len(self.events), 1)
        self.assertEqual(self.events[0].payload['action'], ACTIONS.READ.value)
        self.assertEqual(len(self.events[0].read_records), 0)

    def test_post_sends_read_if_id_already_exists(self):
        resp = self.app.post_json(self.collection_url, self.body,
                                  headers=self.headers, status=201)
        record = resp.json['data']
        body = dict(self.body)
        body['data']['id'] = record['id']

        # a second post with the same record id
        self.app.post_json(self.collection_url, body, headers=self.headers,
                           status=200)
        self.assertEqual(len(self.events), 1)
        self.assertEqual(self.events[0].payload['action'], ACTIONS.READ.value)

class ResourceChangedTest(BaseEventTest, unittest.TestCase):

    subscribed = (ResourceChanged,)

    def test_post_sends_create_action(self):
        self.app.post_json(self.collection_url, self.body,
                           headers=self.headers, status=201)
        self.assertEqual(len(self.events), 1)
        self.assertEqual(self.events[0].payload['action'],
                         ACTIONS.CREATE.value)

    def test_put_sends_create_action(self):
        body = dict(self.body)
        body['data']['id'] = record_id = str(uuid.uuid4())
        record_url = self.get_item_url(record_id)
        self.app.put_json(record_url, body,
                          headers=self.headers, status=201)
        self.assertEqual(len(self.events), 1)
        self.assertEqual(self.events[0].payload['action'],
                         ACTIONS.CREATE.value)

    def test_not_triggered_on_failed_put(self):
        record_id = str(uuid.uuid4())
        record_url = self.get_item_url(record_id)
        self.app.put_json(record_url, self.body, headers=self.headers)
        headers = self.headers.copy()
        headers['If-Match'] = '"12345"'
        self.app.put_json(record_url, self.body, headers=headers, status=412)
        self.assertEqual(len(self.events), 1)
        self.assertEqual(self.events[0].payload['action'],
                         ACTIONS.CREATE.value)

    def test_patch_sends_update_action(self):
        resp = self.app.post_json(self.collection_url, self.body,
                                  headers=self.headers, status=201)
        record = resp.json['data']
        record_url = self.get_item_url(record['id'])

        self.app.patch_json(record_url, self.body, headers=self.headers,
                            status=200)
        self.assertEqual(len(self.events), 2)
        self.assertEqual(self.events[0].payload['action'],
                         ACTIONS.CREATE.value)
        self.assertEqual(self.events[1].payload['action'],
                         ACTIONS.UPDATE.value)

    def test_put_sends_update_action_if_record_exists(self):
        body = dict(self.body)
        body['data']['id'] = record_id = str(uuid.uuid4())
        record_url = self.get_item_url(record_id)
        self.app.put_json(record_url, body,
                          headers=self.headers, status=201)

        body['data']['more'] = 'stuff'
        self.app.put_json(record_url, body,
                          headers=self.headers, status=200)

        self.assertEqual(len(self.events), 2)
        self.assertEqual(self.events[0].payload['action'],
                         ACTIONS.CREATE.value)
        self.assertEqual(self.events[1].payload['action'],
                         ACTIONS.UPDATE.value)

    def test_delete_sends_delete_action(self):
        resp = self.app.post_json(self.collection_url, self.body,
                                  headers=self.headers, status=201)
        record = resp.json['data']
        record_url = self.get_item_url(record['id'])

        self.app.delete(record_url, headers=self.headers, status=200)
        self.assertEqual(len(self.events), 2)
        self.assertEqual(self.events[0].payload['action'],
                         ACTIONS.CREATE.value)
        self.assertEqual(self.events[1].payload['action'],
                         ACTIONS.DELETE.value)

    def test_collection_delete_sends_delete_action(self):
        self.app.post_json(self.collection_url, self.body,
                           headers=self.headers, status=201)
        self.app.post_json(self.collection_url, self.body,
                           headers=self.headers, status=201)

        self.app.delete(self.collection_url, headers=self.headers, status=200)

        self.assertEqual(len(self.events), 3)
        self.assertEqual(self.events[0].payload['action'],
                         ACTIONS.CREATE.value)
        self.assertEqual(self.events[1].payload['action'],
                         ACTIONS.CREATE.value)
        self.assertEqual(self.events[2].payload['action'],
                         ACTIONS.DELETE.value)

    def test_request_fails_if_notify_fails(self):
        with notif_broken(self.app.app, ResourceChanged):
            self.app.post_json(self.collection_url, self.body,
                               headers=self.headers, status=500)
        self.assertEqual(len(self.events), 0)

    def test_triggered_on_protected_resource(self):
        app = self.make_app(settings={
            'psilo_write_principals': 'system.Authenticated'
        })
        app.post_json('/psilos', self.body,
                      headers=self.headers, status=201)
        self.assertEqual(len(self.events), 1)
        self.assertEqual(self.events[0].payload['action'],
                         ACTIONS.CREATE.value)

    def test_permissions_are_stripped_from_event_on_protected_resource(self):
        app = self.make_app(settings={
            'psilo_write_principals': 'system.Authenticated'
        })
        resp = app.post_json('/psilos', self.body,
                             headers=self.headers, status=201)
        record = resp.json['data']
        record_url = '/psilos/' + record['id']
        app.patch_json(record_url, {"data": {"name": "De barcelona"}},
                       headers=self.headers)
        impacted_records = self.events[-1].impacted_records
        self.assertNotIn('__permissions__', impacted_records[0]['new'])
        self.assertNotIn('__permissions__', impacted_records[0]['old'])

class AfterResourceChangedTest(BaseEventTest, unittest.TestCase):

    subscribed = (AfterResourceChanged,)

    def test_request_succeeds_if_notify_fails(self):
        with notif_broken(self.app.app, AfterResourceChanged):
            self.app.post_json(self.collection_url, self.body,
                               headers=self.headers)

        self.assertEqual(len(self.events), 0)

class AfterResourceReadTest(BaseEventTest, unittest.TestCase):

    subscribed = (AfterResourceRead,)

    def test_request_succeeds_if_notify_fails(self):
        with notif_broken(self.app.app, AfterResourceChanged):
            self.app.post_json(self.collection_url, self.body,
                               headers=self.headers)

        self.assertEqual(len(self.events), 0)

class ImpactedRecordsTest(BaseEventTest, unittest.TestCase):

    subscribed = (ResourceChanged,)

    def test_create_has_new_record_and_no_old_in_payload(self):
        resp = self.app.post_json(self.collection_url, self.body,
                                  headers=self.headers)
        record = resp.json['data']
        impacted_records = self.events[-1].impacted_records
        self.assertEqual(len(impacted_records), 1)
        self.assertNotIn('old', impacted_records[0])
        self.assertEqual(impacted_records[0]['new'], record)

    def test_collection_delete_has_old_record_and_no_new_in_payload(self):
        resp = self.app.post_json(self.collection_url, self.body,
                                  headers=self.headers)
        record1 = resp.json['data']
        resp = self.app.post_json(self.collection_url, self.body,
                                  headers=self.headers)
        record2 = resp.json['data']

        self.app.delete(self.collection_url, headers=self.headers, status=200)

        impacted_records = self.events[-1].impacted_records
        self.assertEqual(len(impacted_records), 2)
        self.assertNotIn('new', impacted_records[0])
        self.assertNotIn('new', impacted_records[1])
        self.assertEqual(impacted_records[0]['old']['deleted'], True)
        self.assertEqual(impacted_records[1]['old']['deleted'], True)
        deleted_ids = {impacted_records[0]['old']['id'],
                       impacted_records[1]['old']['id']}
        self.assertEqual(deleted_ids, {record1['id'], record2['id']})

    def test_update_has_old_and_new_record(self):
        resp = self.app.post_json(self.collection_url, self.body,
                                  headers=self.headers, status=201)
        record = resp.json['data']
        record_url = self.get_item_url(record['id'])
        self.app.patch_json(record_url, {'data': {'name': 'en boite'}},
                            headers=self.headers)
        impacted_records = self.events[-1].impacted_records
        self.assertEqual(len(impacted_records), 1)
        self.assertEqual(impacted_records[0]['new']['id'], record['id'])
        self.assertEqual(impacted_records[0]['new']['id'],
                         impacted_records[0]['old']['id'])
        self.assertEqual(impacted_records[0]['old']['name'], 'de Paris')
        self.assertEqual(impacted_records[0]['new']['name'], 'en boite')

    def test_delete_has_old_record_and_no_new_in_payload(self):
        resp = self.app.post_json(self.collection_url, self.body,
                                  headers=self.headers, status=201)
        record = resp.json['data']
        record_url = self.get_item_url(record['id'])
        self.app.delete(record_url, headers=self.headers, status=200)

        impacted_records = self.events[-1].impacted_records
        self.assertEqual(len(impacted_records), 1)
        self.assertNotIn('new', impacted_records[0])
        self.assertEqual(impacted_records[0]['old']['id'], record['id'])
        self.assertEqual(impacted_records[0]['old']['deleted'], True)

class BatchEventsTest(BaseEventTest, unittest.TestCase):

    subscribed = (ResourceChanged, ResourceRead)

    def test_impacted_records_are_merged(self):
        record_id = str(uuid.uuid4())
        record_url = self.get_item_url(record_id)
        body = {
            "defaults": {
                "method": "PUT",
                "path": record_url
            },
            "requests": [
                {"body": {'data': {'name': 'foo'}}},
                {"body": {'data': {'name': 'bar'}}},
                {"body": {'data': {'name': 'baz'}}},
                {"method": "DELETE"}
            ]
        }
        self.app.post_json("/batch", body, headers=self.headers)
        self.assertEqual(len(self.events), 3)

        create_event = self.events[0]
        self.assertEqual(create_event.payload['action'], 'create')
        self.assertEqual(len(create_event.impacted_records), 1)
        self.assertNotIn('old', create_event.impacted_records[0])
        update_event = self.events[1]
        self.assertEqual(update_event.payload['action'], 'update')
        impacted = update_event.impacted_records
        self.assertEqual(len(impacted), 2)
        self.assertEqual(impacted[0]['old']['name'], 'foo')
        self.assertEqual(impacted[0]['new']['name'], 'bar')
        self.assertEqual(impacted[1]['old']['name'], 'bar')
        self.assertEqual(impacted[1]['new']['name'], 'baz')
        delete_event = self.events[2]
        self.assertEqual(delete_event.payload['action'], 'delete')
        self.assertEqual(len(delete_event.impacted_records), 1)
        self.assertNotIn('new', delete_event.impacted_records[0])

    def test_one_event_is_sent_per_resource(self):
        body = {
            "defaults": {
                "method": "POST",
                "body": self.body,
            },
            "requests": [
                {"path": '/mushrooms'},
                {"path": '/mushrooms'},
                {"path": '/psilos'},
            ]
        }
        self.app.post_json("/batch", body, headers=self.headers)
        self.assertEqual(len(self.events), 2)

    def test_one_event_is_sent_per_action(self):
        body = {
            "defaults": {
                "path": '/mushrooms',
            },
            "requests": [
                {"method": "POST", "body": self.body},
                {"method": "DELETE"},
                {"method": "GET"},
            ]
        }
        self.app.post_json("/batch", body, headers=self.headers)
        self.assertEqual(len(self.events), 3)

    def test_events_are_not_sent_if_subrequest_fails(self):
        patch = mock.patch.object(self.storage,
                                  'delete_all',
                                  side_effect=BackendError('boom'))
        patch.start()
        self.addCleanup(patch.stop)
        request_create = {
            "method": "POST",
            "body": self.body,
        }
        request_delete_all = {
            "method": "DELETE",
            "body": self.body,
        }
        body = {
            "defaults": {
                "path": self.collection_url
            },
            "requests": [request_create, request_delete_all]
        }
        self.app.post_json("/batch", body, headers=self.headers,
                           status=503)
        self.assertEqual(len(self.events), 0)

def load_from_config(config, prefix):
    class ClassListener(object):
        def __call__(self, event):
            pass
    return ClassListener()

@unittest.skipIf(not statsd.statsd_module, "statsd is not installed.")
class StatsDTest(BaseWebTest, unittest.TestCase):
    def get_app_settings(self, *args, **kwargs):
        settings = super(StatsDTest, self).get_app_settings(*args, **kwargs)
        if not statsd.statsd_module:
            return settings

        settings['statsd_url'] = 'udp://localhost:8125'
        this_module = 'cliquet.tests.resource.test_events'
        settings['event_listeners'] = 'test'
        settings['event_listeners.test.use'] = this_module
        return settings

    def test_statds_tracks_listeners_execution_duration(self):
        statsd_client = self.app.app.registry.statsd._client
        with mock.patch.object(statsd_client, 'timing') as mocked:
            self.app.post_json(self.collection_url,
                               {"data": {"name": "pouet"}},
                               headers=self.headers)
            timers = set(c[0][0] for c in mocked.call_args_list)
            self.assertIn('listeners.test', timers)


import yaml
from sigma.parser.condition import ConditionAND, ConditionOR
from sigma.config.exceptions import SigmaConfigParseError
from sigma.config.mapping import FieldMapping

# Configuration
class SigmaConfiguration:
    """Sigma converter configuration. Contains field mappings and logsource descriptions"""
    def __init__(self, configyaml=None):
        if configyaml == None:
            self.config = None
            self.fieldmappings = dict()
            self.logsources = dict()
            self.logsourcemerging = SigmaLogsourceConfiguration.MM_AND
            self.defaultindex = None
            self.backend = None
        else:
            config = yaml.safe_load(configyaml)
            self.config = config

            self.fieldmappings = dict()
            try:
                for source, target in config['fieldmappings'].items():
                    self.fieldmappings[source] = FieldMapping(source, target)
            except KeyError:
                pass
            if type(self.fieldmappings) != dict:
                raise SigmaConfigParseError("Fieldmappings must be a map")

            try:
                self.logsourcemerging = config['logsourcemerging']
            except KeyError:
                self.logsourcemerging = SigmaLogsourceConfiguration.MM_AND

            try:
                self.defaultindex = config['defaultindex']
            except KeyError:
                self.defaultindex = None

            self.logsources = list()
            self.backend = None

    def get_fieldmapping(self, fieldname):
        """Return mapped fieldname if mapping defined or field name given in parameter value"""
        try:
            return self.fieldmappings[fieldname]
        except KeyError:
            return FieldMapping(fieldname)

    def get_logsource(self, category, product, service):
        """Return merged log source definition of all logosurces that match criteria"""
        matching = [logsource for logsource in self.logsources if logsource.matches(category, product, service)]
        return SigmaLogsourceConfiguration(matching, self.defaultindex)

    def set_backend(self, backend):
        """Set backend. This is used by other code to determine target properties for index addressing"""
        self.backend = backend
        if self.config != None:
            if 'logsources' in self.config:
                logsources = self.config['logsources']
                if type(logsources) != dict:
                    raise SigmaConfigParseError("Logsources must be a map")
                for name, logsource in logsources.items():
                    self.logsources.append(SigmaLogsourceConfiguration(logsource, self.defaultindex, name, self.logsourcemerging, self.get_indexfield()))

    def get_indexfield(self):
        """Get index condition if index field name is configured"""
        if self.backend != None:
            return self.backend.index_field

class SigmaLogsourceConfiguration:
    """Contains the definition of a log source"""
    MM_AND = "and"  # Merge all conditions with AND
    MM_OR  = "or"   # Merge all conditions with OR

    def __init__(self, logsource=None, defaultindex=None, name=None, mergemethod=MM_AND, indexfield=None):
        self.name = name
        self.indexfield = indexfield
        if logsource == None:               # create empty object
            self.category = None
            self.product = None
            self.service = None
            self.index = list()
            self.conditions = None
        elif type(logsource) == list and all([isinstance(o, SigmaLogsourceConfiguration) for o in logsource]):      # list of SigmaLogsourceConfigurations: merge according to mergemethod
            # Merge category, product and service
            categories = set([ ls.category for ls in logsource if ls.category != None ])
            products = set([ ls.product for ls in logsource if ls.product != None ])
            services = set([ ls.service for ls in logsource if ls.service != None])
            if len(categories) > 1 or len(products) > 1 or len(services) > 1:
                raise ValueError("Merged SigmaLogsourceConfigurations must have disjunct categories (%s), products (%s) and services (%s)" % (str(categories), str(products), str(services)))

            try:
                self.category = categories.pop()
            except KeyError:
                self.category = None
            try:
                self.product = products.pop()
            except KeyError:
                self.product = None
            try:
                self.service = services.pop()
            except KeyError:
                self.service = None

            # Merge all index patterns
            self.index = list(set([index for ls in logsource for index in ls.index]))       # unique(flat(logsources.index))
            if len(self.index) == 0 and defaultindex is not None:   # if no index pattern matched and default index is present: use default index
                if type(defaultindex) == str:
                    self.index = [defaultindex]
                elif type(defaultindex) == list and all([type(i) == str for i in defaultindex]):
                    self.index = defaultindex
                else:
                    raise TypeError("Default index must be string or list of strings")

            # "merge" index field (should never differ between instances because it is provided by backend class
            indexfields = [ ls.indexfield for ls in logsource if ls.indexfield != None ]
            try:
                self.indexfield = indexfields[0]
            except IndexError:
                self.indexfield = None

            # Merge conditions according to mergemethod
            if mergemethod == self.MM_AND:
                cond = ConditionAND()
            elif mergemethod == self.MM_OR:
                cond = ConditionOR()
            else:
                raise ValueError("Mergemethod must be '%s' or '%s'" % (self.MM_AND, self.MM_OR))
            for ls in logsource:
                if ls.conditions != None:
                    cond.add(ls.conditions)
            if len(cond) > 0:
                self.conditions = cond
            else:
                self.conditions = None
        elif type(logsource) == dict:       # create logsource configuration from parsed yaml
            if 'category' in logsource and type(logsource['category']) != str \
                    or 'product' in logsource and type(logsource['product']) != str \
                    or 'service' in logsource and type(logsource['service']) != str:
                raise SigmaConfigParseError("Logsource category, product or service must be a string")
            try:
                self.category = logsource['category']
            except KeyError:
                self.category = None
            try:
                self.product = logsource['product']
            except KeyError:
                self.product = None
            try:
                self.service = logsource['service']
            except KeyError:
                self.service = None
            if self.category == None and self.product == None and self.service == None:
                raise SigmaConfigParseError("Log source definition will not match")

            if 'index' in logsource:
                index = logsource['index']
                if type(index) not in (str, list):
                    raise SigmaConfigParseError("Logsource index must be string or list of strings")
                if type(index) == list and not all([type(index) == str for index in logsource['index']]):
                    raise SigmaConfigParseError("Logsource index patterns must be strings")
                if type(index) == list:
                    self.index = index
                else:
                    self.index = [ index ]
            else:
                # no default index handling here - this branch is executed if log source definitions are parsed from
                # config and these must not necessarily contain an index definition. A valid index may later be result
                # from a merge, where default index handling applies.
                self.index = []

            if 'conditions' in logsource:
                if type(logsource['conditions']) != dict:
                    raise SigmaConfigParseError("Logsource conditions must be a map")
                cond = ConditionAND()
                for key, value in logsource['conditions'].items():
                    cond.add((key, value))
                self.conditions = cond
            else:
                self.conditions = None
        else:
            raise SigmaConfigParseError("Logsource definitions must be maps")

    def matches(self, category, product, service):
        """Match log source definition against given criteria, None = ignore"""
        searched = 0
        for searchval, selfval in zip((category, product, service), (self.category, self.product, self.service)):
            if searchval == None and selfval != None:
                return False
            if selfval != None:
                searched += 1
                if searchval != selfval:
                    return False
        if searched:
            return True

    def get_indexcond(self):
        """Get index condition if index field name is configured"""
        cond = ConditionOR()
        if self.indexfield:
            for index in self.index:
                cond.add((self.indexfield, index))
            return cond
        else:
            return None

    def __str__(self):
        return "[ LogSourceConfiguration: %s %s %s indices: %s ]" % (self.category, self.product, self.service, str(self.index))


from sqlalchemy import create_engine, MetaData, Table, Column, Integer, String, DateTime, Float, and_, or_
from sqlalchemy.dialects import postgresql
from geoalchemy2 import Geometry
from geoalchemy2.functions import GenericFunction, ST_AsMVTGeom,  ST_TileEnvelope
from sqlalchemy.dialects.postgresql import BYTEA, JSONB
import mercantile
from functools import partial
import pyproj
from shapely.ops import transform
from shapely.geometry import shape, MultiPolygon, MultiPoint, MultiLineString
from flask_gzip import Gzip
import os
import json
import numpy
from redis import Redis

from flask import Flask
from flask import request, make_response
from flask import jsonify

from flask_cors import CORS

import xxhash
import importlib

POSTGRES = os.environ.get('POSTGRES', '51.15.160.236:25432')
POSTGRES_USER = os.environ.get('POSTGRES_USER', 'admin')
POSTGRES_PASS = os.environ.get('POSTGRES_PASS', 'tgZWW3Dgze94FN9O')
POSTGRES_DBNAME = os.environ.get('POSTGRES_DBNAME', 'ohm')

REDIS_HOST = os.environ.get('REDIS_HOST', 'localhost')
REDIS_PORT = int(os.environ.get('REDIS_PORT', '6379'))
REDIS_DB = int(os.environ.get('REDIS_DB', '3'))

TILES_LAYERS=os.environ.get('TILES_LAYERS', 'boundary,boundary_label,culture,culture_label,waterway,water,building,industrial,landuse,transportation,place,religion')

EXPANDER = os.environ.get('EXPANDER', 'https://github.com/openhistorymap/mapstyles/blob/master/expander.json')

EPHEMERAL_STEP = float(1)/12/31
PERSIST_STEP = float(1)/12

def get_step(layer):
    if layer in TILES_LAYERS.split(','):
        return EPHEMERAL_STEP
    else:
        return EPHEMERAL_STEP

def get_month(x, year_step = EPHEMERAL_STEP):
    yr = [int(x)+y for y in [x*year_step for x in range(0,12*31)]]
    nyr = sorted(yr + [x])
    i = nyr.index(x)
    return yr[i-1]

class ST_AsMVT(GenericFunction):
    type = BYTEA

def float_to_date(f):
    y = int(f)
    m = int((f-y)*12)+1
    d = int(((f-y)*12-(m-1))*30)
    return '{}-{}-{}'.format(y,m,d)

def create_app(test_config=None):
    # create and configure the app
    app = Flask(__name__, instance_relative_config=True)
    # pool_pre_ping should help handle DB connection drops

    engine = create_engine('postgresql://{}:{}@{}/{}'.format(
            POSTGRES_USER, POSTGRES_PASS, POSTGRES, POSTGRES_DBNAME
        ), pool_size=20, max_overflow=0, pool_pre_ping=True,
        echo=False)
    db = engine.connect()

    r = Redis(REDIS_HOST, REDIS_PORT, REDIS_DB)

    exp = {
        "boundary": {
            "name": {
                "Rome": {
                    "color": "#8e001c"
                },
                "Lotharingia": {
                    "color": "#ddb318"
                },
                "Kingdom of Italy": {
                    "color": "#6397d0"
                }
            }
        }
    }

    metadata = MetaData()
    ohm_items = Table('items', metadata,
        Column('id', Integer, primary_key=True, autoincrement=True),
        Column('ohm_from', Float, index=True),
        Column('ohm_to', Float, index=True),
        Column('layer', String, index=True),
        Column('properties', JSONB),
        Column('geom', Geometry(geometry_type='GEOMETRY', srid=3857)),
        Column('author', String, default='ohm'),
    )
    ohm_rels = Table('relations', metadata,
        Column('id', Integer, primary_key=True, autoincrement=True),
        Column('ohm_from', Float, index=True),
        Column('ohm_to', Float, index=True),
        Column('layer', String, index=True),
        Column('properties', JSONB),
        Column('geom', Geometry(geometry_type='GEOMETRY', srid=3857)),
        Column('author', String, default='ohm'),
    )

    ohm_rel_members = Table('relation_members', metadata,
        Column('id', Integer, primary_key=True, autoincrement=True),
        Column('relation', Integer, index=True),
        Column('item', Integer, index=True),
        Column('role', String, index=True),
        Column('author', String, default='ohm'),
    )

    ohm_items_members = Table('item_node', metadata,
        Column('id', Integer, primary_key=True, autoincrement=True),
        Column('item', Integer, index=True),
        Column('node_id', Integer, index=True),
    )

    ohm_items_members = Table('item_arc', metadata,
        Column('id', Integer, primary_key=True, autoincrement=True),
        Column('item', Integer, index=True),
        Column('arc_id', Integer, index=True),
    )

    ohm_arcs = Table('arc', metadata,
        Column('id', Integer, primary_key=True, autoincrement=True),
        Column('node_1', Integer),
        Column('node_2', Integer),
        Column('direction', Integer),
    )

    ohm_points = Table('node', metadata,
        Column('id', Integer, primary_key=True, autoincrement=True),
        Column('geom', Geometry(geometry_type='POINT', srid=3857)),
    )

    project = partial(
        pyproj.transform,
        pyproj.Proj(init='epsg:4326'), # source coordinate system
        pyproj.Proj(init='epsg:3857')) # destination coordinate system

    @app.route('/')
    def index():
        return ''

    @app.route('/setup')
    def setup():
        metadata.create_all(engine)
        return jsonify({'result':'OK'})

    def pimp(data):
        ret = {}
        if data['layer'] in exp.keys():
            for k in exp[data['layer']]:
                if k in data:
                    ret = exp[data['layer']][k].get(data[k], {})
        out = ret.copy()
        out.update(data)
        return out

    def map_single(jdata):
        ss = shape(jdata['geometry'])
        ss = transform(project, ss)
        pdata = jdata['properties']
        #print(pdata)
        _from = jdata['from'] if 'from' in jdata else pdata['ohm:from:date']
        _to = jdata['to'] if 'to' in jdata else pdata['ohm:to:date']
        pdata = pimp(pdata)
        cs = [ss]
        if isinstance(ss, (MultiPolygon, MultiLineString, MultiPoint,)):
            cs = [its.buffer(0) for its in ss if its]
        ret = []
        pdata['$area'] = ss.area
        pdata['$length'] = ss.length
        if 'Poly' in ss.geom_type:
            ret.append(dict(
                ohm_from = _from,
                ohm_to = _to,
                layer = pdata['layer']+"_label",
                properties = pdata,
                geom = 'SRID=3857;' + ss.representative_point().wkt
            ))
        for s in cs:
            ret.append(dict(
                ohm_from = _from,
                ohm_to = _to,
                layer = pdata['layer'],
                properties = pdata,
                geom = 'SRID=3857;' + ss.wkt
            ))
        return ret

    @app.route('/items', methods=['POST'])
    def saveItem():
        data = request.data
        jdata = json.loads(data)
        #print(len(jdata))
        if not isinstance(jdata, list):
            jdata = [jdata]
        jdata = list(map(map_single, jdata))
        #print(len(jdata))

        flat_list = []
        for sublist in jdata:
            for item in sublist:
                flat_list.append(item)

        r.rpush('store', *[json.dumps(fi) for fi in flat_list])
        #map(lambda x: db.execute('ohm_storeitem(\'{layer}\', {ohm_from}, {ohm_to}, {properties}, {x})'.format**(x)), flat_list)
        #db.execute(ohm_items.insert(), flat_list)

        return jsonify({'result': 'OK', 'items_added': len(flat_list)})

    def out_rel_feat(r):
        rets = []

        f = r['itms']
        n = r['rel']
        pp = f[0].properties
        min_ = f[0].properties['ohm:from:date']
        max_ = f[-1].properties['ohm:to:date']
        pp['ohm:from:date'] = min_
        pp['ohm:from:date:year'] = int(min_)
        pp['ohm:to:date'] = max_
        pp['ohm:to:date:year'] = int(max_)
        pp['relation'] = n

        fp = []
        for fpo in f:
            fpop = fpo.properties
            fpop['name'] = float_to_date(fpop['ohm:from:date'])
            fpop['relation'] = n
            rets.append({
                "type": "Feature",
                "properties": fpop,
                "geometry": json.loads(fpo.gg)
            })
            fp.append(
                json.loads(fpo.gg).get('coordinates'),
            )

        rets.append({
            "type": "Feature",
            "properties": pp,
            "geometry": {
                "type": "LineString",
                "coordinates": fp
            }
        })

        return rets

    @app.route('/relation', methods=['POST'])
    def newRelation():
        data = request.data
        jdata = json.loads(data)
        if not isinstance(jdata, list):
            jdata = [jdata]
        jdata = list(map(map_single, jdata))

        flat_list = []
        for sublist in jdata:
            for item in sublist:
                flat_list.append(item)

        x = db.execute(ohm_items.insert(), flat_list)
        print(x)
        return jsonify({'result': 'OK', 'items_added': len(flat_list)})

    @app.route('/bots', methods=['GET'])
    @app.route('/bots/<bot>', methods=['GET'])
    def runBot(bot = 'movement'):
        r.rpush('bot', bot)
        #m = importlib.import_module("bot.{}".format(bot))
        #m.run(db, )

        return jsonify({'result': 'OK'})

    @app.route('/status', methods=['GET'])
    def status():
        ret = {
            'bots': r.llen('bot'),
            'store': r.llen('store')
        }
        return jsonify({'result': 'OK', 'status': ret})

    CORS(app, resources={r"*": {"origins": "*"}})
    Gzip(app)
    return app

app = create_app()

if __name__ == '__main__':
    app.run(host='0.0.0.0', port='9039', debug=True, threaded=True)