#!/usr/bin/env python
#searchKeys.py
"""
	Searches for specific keys or amino acids or positions and retrieves
	their details from triplets files
"""

import glob, os, csv, ntpath,socket,argparse, time, re
import pandas as pd, numpy as np
from collections import Counter
from joblib import Parallel, delayed, cpu_count
from os.path import expanduser
import itertools

__author__ = "Venkata Sarika Kondra"
__version__ = "1.0.1"
__maintainer__ = "Venkata Sarika Kondra"
__email__ = "c00219805@louisiana.edu"

parser = argparse.ArgumentParser(description='Search Keys.')
parser.add_argument('--sample_name', '-sample', metavar='sample_name', \
	default='sample_p_loop_10', \
	help='Name of the sample on which this script should be run.')
parser.add_argument('--path', '-path', metavar='path', \
	default=os.path.join(expanduser('~'),'Research', 'Protien_Database', \
		'extracted_new_samples', 'testing'), \
	help='Directory of input sample and other files.')
parser.add_argument('--keys', '-keys', metavar='keys', \
	default='6362130,6362129,6362128,6362131,6362132',\
	help='Key that is to be searched.')
parser.add_argument('--aacds', '-aacds', metavar='aacds', \
	default='gly,thr,lys',\
	help='Amino acids that are to be searched.')
parser.add_argument('--setting', '-setting', metavar='setting', \
	default='theta29_dist35', \
	help='Setting with theta and distance bin values.')
parser.add_argument('--exclude', '-exclude', metavar='exclude', \
	default=True, \
	help='Exclude certain keys from when extracting triplets details.')
parser.add_argument('--search_mode', '-search_mode', metavar='search_mode', \
	default=2, \
	help='0 if Key search, 1 if amino acid search and 2 if sequence identification.')

def search_by_key(file,keys):
	print('inside search key..filtering started..')
	search_records = []
	line_count = len(file)
	df = file[file['key'].isin(keys)]
	# for line in file:
	# 	line_count += 1
	# 	line_splitted=line.split('\t')
	# 	if line_splitted[0].strip() in keys:
	# 		search_records.append(line_splitted)
	#return pd.DataFrame(search_records,columns = column_names)
	return df, line_count, len(set(df['key'].values))

def search_aacd(file,aacds, req_pos):
	search_records = []
	pattern_line = []
	for line in file:
		line_aacds = {}
		line_splitted=line.split('\t')
		line_aacd_arr = [line_splitted[1].strip().upper(), \
			line_splitted[3].strip().upper(), line_splitted[5].strip().upper()]
		if sorted(line_aacd_arr) == sorted(aacds):
			search_records.append(line_splitted)
			if req_pos:
				line_pos_arr = [line_splitted[2].strip().upper(), \
					line_splitted[4].strip().upper(), \
					line_splitted[6].strip().upper()]
				if all(elem in req_pos for elem in line_pos_arr):
					pattern_line.append(line)

	return pd.DataFrame(search_records,columns = column_names), pattern_line

def identify_pattern_by_regex(files,column_names, patterns):
	matched = []
	all_pattern = generate_sequences_from_triplets_by_FR(file)
	for pattern in patterns:
		print(re.findall(pattern, all_pattern))
		matched.append(pattern)
	return ntpath.basename(file)[:4], matched

def identify_pattern_by_normal(file, pattern,count_of_x):
	matched = []
	# If file size is greater than 5 GB pandas is throwing Memory error
	# But if you always read line-by-line it is very slow.
	# Hence using two way reading
	if float(os.path.getsize(file)/ 2**30) > 4 :
		print('File Size too big')
		return ntpath.basename(file)[:4], ['File Size too big']
		#all_pattern = generate_sequences_from_triplets_by_FR(file)
	all_pattern = generate_sequences_from_triplets_by_DF(file)
	i = 0
	while i < len(all_pattern):
		if (all_pattern[i].split('_')[0] == pattern[0]) and ( i + count_of_x + len(pattern) < len(all_pattern)):
			#print 'checking this: {}'.format(all_pattern[i:i + count_of_x + len(pattern)])
			if (all_pattern[i + count_of_x +1].split('_')[0] == pattern[1]) \
				and (all_pattern[i+count_of_x +2].split('_')[0] == pattern[2]) \
				and (all_pattern[i+count_of_x +3].split('_')[0] == pattern[3]):
				print( 'found', all_pattern[i:i + count_of_x + len(pattern)])
				matched.append(all_pattern[i:i + count_of_x + len(pattern)])
		i += 1
	return (ntpath.basename(file)[:4], matched)

def generate_sequences_from_triplets_by_DF(file):
	print(ntpath.basename(file)[:4])
	all_pattern = []
	df = pd.read_table(file, delimiter = '\t', names = column_names)
	a_0 = pd.Series(df.aa0.values,index=df.pos0).to_dict()
	a_1 = pd.Series(df.aa1.values,index=df.pos1).to_dict()
	a_2 = pd.Series(df.aa2.values,index=df.pos2).to_dict()
	a_0.update(a_1)
	a_0.update(a_2)
	pos_aa_dict = sorted(a_0.items())
	for key, value in pos_aa_dict:
		all_pattern .append(str(value) + '_' + str(key))
	return all_pattern
def generate_sequences_from_triplets_by_FR(file):
	print( ntpath.basename(file)[:4])
	all_pattern = []
	pos_aa = {}
	for line in open(file,'r'):
		line_splitted=line.split('\t')
		if line_splitted[2].strip() not in pos_aa.keys():
			pos_aa[line_splitted[2].strip()] = line_splitted[1].strip().upper()
		if line_splitted[4].strip() not in pos_aa.keys():
			pos_aa[line_splitted[4].strip()] = line_splitted[3].strip().upper()
		if line_splitted[6].strip() not in pos_aa.keys():
			pos_aa[line_splitted[6].strip()] = line_splitted[5].strip().upper()
	for key, value in sorted(pos_aa.items()):
		all_pattern .append(str(value) + '_' + str(key))
	return all_pattern

def common_keys(files):
	print( 'Common Keys Calculation started.')
	common_keys = []
	start = time.time()
	for file in files:
		print( ntpath.basename(file)[:4])
		keys = []
		for line in open(file, 'r'):
			keys.append(line.split('\t')[0])
		if common_keys:
			common_keys = list(set(common_keys) & set(keys))
		else:
			common_keys = list(set(keys))

	print( 'Time taken for Common Keys Calculation: ', (time.time() - start)/60)
	return common_keys

def calculate_low_less15_freq_from_common_keys():
	writer = pd.ExcelWriter(args.path + args.sample_name + args.setting + \
		'common_keys/'+'//all_common_key_distribution.xlsx', engine='xlsxwriter')
	writer_low_freq = pd.ExcelWriter(args.path + args.sample_name + args.setting + \
		'common_keys/'+'//only_low_key_distribution.xlsx', engine='xlsxwriter')
	writer_dist_less15 = pd.ExcelWriter(args.path + args.sample_name + args.setting + \
		'common_keys/'+'//dist_less15_triplets.xlsx', engine='xlsxwriter')
	summary = []
	commom_key_freqs = Counter()
	common_keys_files = glob.glob(args.path + args.sample_name + args.setting + \
		'common_keys/' +'*_key_search_common_keys.csv')
	for file in common_keys_files:
		print( ntpath.basename(file)[:4])
		df = pd.read_csv(file, delimiter = ',')
		x = Counter(df['key'].values)
		ckeys_freqs_df = pd.DataFrame(sorted(x.items(), key=lambda pair: pair[1], \
			 reverse=True), columns = ['key', 'freq'])
		ckeys_freqs_df.to_excel(writer,sheet_name=ntpath.basename(file)[:4])
		#Low Frequency Common Keys
		low_freqs = ckeys_freqs_df[ckeys_freqs_df['freq'] < 3]
		low_freqs.to_excel(writer_low_freq,sheet_name=ntpath.basename(file)[:4])
		df_low_freqs_triplets = df[ df['key'].isin(set(low_freqs['key'].values))]
		#Distance less than 15
		df_distance_less15 = df_low_freqs_triplets[df_low_freqs_triplets['distance'] <= 15]
		df_distance_less15.to_excel(writer_dist_less15,sheet_name=ntpath.basename(file)[:4])
		summary.append((ntpath.basename(file)[:4], len(set(low_freqs['key'].values)), \
			len(set(df_distance_less15['key'].values))))
		print (ntpath.basename(file)[:4], len(set(low_freqs['key'].values)), \
			len(set(df_distance_less15['key'].values)))
		#Overall Common Keys
		if commom_key_freqs:
			commom_key_freqs = commom_key_freqs + x
		else:
			commom_key_freqs = x

	pd.DataFrame(sorted(commom_key_freqs.items(), key=lambda pair: pair[1], reverse=True)). \
		to_excel(writer,sheet_name='Summary')

	df_merge = pd.read_csv(args.path + args.sample_name + args.setting +'common_keys/'+ \
		'summary_common_keys-7-31-2018.csv').merge(pd.DataFrame(summary, \
			columns = ['fileName', '# of low freq common keys', '# of low freq common keys with maxdist <= 15']),on = 'fileName')
	df_merge.to_csv(args.path + args.sample_name + args.setting + 'common_keys/' + 'summary_common_keys-8-02-2018.csv')

def process_files(file, req_aacds):
	print(ntpath.basename(file)[:4])
	pattern_theta_1 = ''
	pattern_dist_1 = ''
	pattern_theta_2 = ''
	pattern_dist_2 = ''

	samples_file = pd.read_csv(os.path.join(args.path, \
		args.sample_name, 'sample_details.csv'))
	req_pos = re.findall(r'\d+', samples_file.set_index('protein').\
		loc[ntpath.basename(file)[:4],  'pattern'])
	records, patterns = search_aacd(open(file,'r'),req_aacds, req_pos)
	if patterns:
		# pattern_theta_1 = [pattern.split('\t')[8] for pattern in patterns]
		# pattern_dist_1 = [pattern.split('\t')[10] for pattern in patterns]
		pattern_theta_1 = patterns[0].split('\t')[8]
		pattern_dist_1 = patterns[0].split('\t')[10]
		if len(patterns) >1:
			pattern_theta_2 = patterns[1].split('\t')[8]
			pattern_dist_2 = patterns[1].split('\t')[10]
	records.to_csv(os.path.join(args.path, args.sample_name, \
			args.setting, 'common_keys', \
	 		ntpath.basename(file)[:4] + '_'+ "_".join(req_aacds) + '.csv'))
	print (ntpath.basename(file)[:4], pd.to_numeric(records['theta']).mean(),\
	 pd.to_numeric(records['distance']).mean(), pattern_theta_1, pattern_theta_2,\
	 pattern_dist_1, pattern_dist_2, patterns)
	return (ntpath.basename(file)[:4], pd.to_numeric(records['theta']).mean(),\
	 pd.to_numeric(records['distance']).mean(), pattern_theta_1, pattern_theta_2,\
	 pattern_dist_1, pattern_dist_2, patterns)

if __name__ == '__main__':
	start = time.time()
	args = parser.parse_args()
	files = glob.glob(os.path.join(args.path,\
					 args.sample_name, args.setting, '*.triplets*'))
	column_names = ['key', 'aa0', 'pos0', 'aa1', 'pos1', 'aa2', 'pos2', \
	'classT', 'theta', 'classL', 'distance', 'x0', 'y0', 'z0', 'x1', 'y1',\
	 'z1', 'x2', 'y2', 'z2']

	#files = glob.glob(args.path + args.sample_name + args.setting + '//*.keys*')
	#column_names = ['key','freq']

	#results_key_search = pd.DataFrame(columns = column_names)
	#summary = []

	#Common Keys
	# keys = common_keys(files)

	# #Keys Search
	if args.search_mode == 0:
		keys = [ str(key) for key in args.keys.split(',')]
		for file in files:
			print( ntpath.basename(file)[:4])
			start = time.time()
			print('inside common keys. Started reading file..')
			df_all = pd.read_table(file, delimiter = '\t', names = column_names)
			print('done reading file', (time.time() - start)/60)
			#records, line_count, matched_keys_count = search_key(open(file,'r'),keys)
			records, line_count, matched_keys_count = search_key(df_all,keys)

			records['fileName'] = ntpath.basename(file)[:4]
			summary.append([ntpath.basename(file)[:4], matched_keys_count, line_count])
			#results_key_search = results_key_search.append(records)
			records.to_csv(args.path + args.sample_name + args.setting +'common_keys/'+ \
				ntpath.basename(file)[:4]+'_key_search_common_keys.csv')
		pd.DataFrame(summary, columns = ['fileName','common_keys','all_keys']) \
				.to_csv(args.path + args.sample_name + args.setting +'summary_common_keys.csv')

	#calculate_low_less15_freq_from_common_keys()

	# #Amino Acids Search
	if args.search_mode == 1:
		all_list = ['GLY', 'LYS','SER', 'GLY']
		already_completed = []
		for req_aacds in itertools.combinations(all_list, 3):
			if sorted(req_aacds) not in already_completed:
				already_completed.append(sorted(req_aacds))
				print('Started AminoAcid serach for: {}'.format("_".join(req_aacds)))
				means_theta = []
				means_distance = []
				writer = pd.ExcelWriter(os.path.join(args.path, args.sample_name, \
					args.setting, 'means_' + "_".join(req_aacds) + '.xlsx'), \
					engine = 'xlsxwriter')

				means_theta_distance  = Parallel(n_jobs=cpu_count() - 1, verbose=10, \
					backend="multiprocessing", batch_size="auto")(\
					delayed(process_files)(file, req_aacds) for file in files)
				pd.DataFrame(means_theta_distance, \
					columns = ['file', 'all_theta_per_protein', \
							'all_dist_per_protein', 'motif_theta_1','motif_theta_2', \
							'motif_dist_1', 'motif_dist_2','motif'])\
					.to_excel(writer,sheet_name='means_theta_distance')
				writer.close()

	#Sequence Identification
	if args.search_mode == 2:
		writer_pattern = pd.ExcelWriter(os.path.join(args.path, args.sample_name, \
					args.setting, 'patterns{}'.format('.xlsx')), \
					engine = 'xlsxwriter')
		patterns = [['GLY','GLY','LYS','SER'], ['GLY','GLY','LYS','THR']]
		for pattern in patterns:
			matched_list = []
			# sequences  = Parallel(n_jobs=cpu_count() - 1, verbose=10, \
			# 			backend="multiprocessing", batch_size="auto")(\
			# 			delayed(identify_sequence)(file, column_names, \
			#  	[r'GLY_\d+[A-Z]{3}_[\d]+[A-Z]{3}_[\d]+[A-Z]{3}_[\d]+[A-Z]{3}_[\d]+GLY_\d+LYS_\d+SER_\d+', \
			#  	r'GLY_\d+[A-Z]{3}_[\d]+[A-Z]{3}_[\d]+[A-Z]{3}_[\d]+[A-Z]{3}_[\d]+GLY_\d+LYS_\d+THR_\d+']) for file in files)
			# identify_sequence(files,column_names, \
			# 	[r'GLY_\d+[A-Z]{3}_[\d]+[A-Z]{3}_[\d]+[A-Z]{3}_[\d]+[A-Z]{3}_[\d]+GLY_\d+LYS_\d+SER_\d+', \
			# 	r'GLY_\d+[A-Z]{3}_[\d]+[A-Z]{3}_[\d]+[A-Z]{3}_[\d]+[A-Z]{3}_[\d]+GLY_\d+LYS_\d+THR_\d+'])
			for file in files:
				matched_list.append(identify_pattern_by_normal(file,pattern, 4))
			pd.DataFrame(matched_list, columns = ['file','pattern_matched'])\
				.to_excel(writer_pattern, sheet_name = "_".join(pattern))

	end = time.time()
	print("Key Search Completed. Total time taken: {} mins".format((end - start)/60))


"""
The object graph is the core unit of microcosm-based applications.

Object graphs wire together independently defined `components` using a set
of factory functions and application configuration. Components are bound
to the graph lazily (or via `graph.use()`) and are cached for reuse.

"""
from __future__ import annotations

from contextlib import contextmanager
from typing import (
    Any,
    Callable,
    Iterable,
    List,
    Optional,
    Tuple,
    Type,
)

from microcosm.caching import Cache, create_cache
from microcosm.config.api import configure
from microcosm.config.model import Configuration
from microcosm.constants import RESERVED
from microcosm.errors import CyclicGraphError, LockedGraphError
from microcosm.hooks import invoke_resolve_hook
from microcosm.loaders import load_from_environ
from microcosm.metadata import Metadata
from microcosm.profile import NoopProfiler
from microcosm.registry import Registry, _registry
from microcosm.typing import Component

Factory = Callable[['ObjectGraph'], Component]

class ObjectGraph:
    """
    An object graph contains all of the instantiated components for a microservice.

    Because components can reference each other a-cyclically, this collection of
    components forms a directed acyclic graph.

    """
    def __init__(self, metadata: Metadata, config, registry, profiler, cache, loader) -> None:
        self.metadata = metadata
        self.config = config
        self._locked = False
        self._registry = registry
        self._profiler = profiler
        self._cache = cache
        self.loader = loader

    def use(self, *keys: str) -> List[Component]:
        """
        Explicitly initialize a set of components by their binding keys.

        """
        return [
            getattr(self, key)
            for key in keys
        ]

    def assign(self, key: str, value: Component) -> Component:
        """
        Explicitly assign a graph binding to a value.

        In general, graph values should only be derived from registered factories
        and the graph should not be assigned to; however, there can be exceptions including
        testing and "virtual" bindings, so assign can be used when circumventing setattr.

        """
        self._cache[key] = value
        return value

    def lock(self) -> ObjectGraph:
        """
        Lock the graph so that new components cannot be created.

        """
        self._locked = True
        return self

    def unlock(self) -> ObjectGraph:
        """
        Unlock the graph so that new components can created.

        """
        self._locked = False
        return self

    def factory_for(self, key: str) -> Factory:
        return self._registry.resolve(key)

    def get(self, key: str) -> Component:
        return self._cache.get(key)

    def __getattr__(self, key: str) -> Component:
        """
        Access a component by its binding key.

        If the component is not present, it will be lazily created.

        :raises CyclicGraphError: if the factory function requires a cycle
        :raises LockedGraphError: if the graph is locked

        """
        try:
            component = self._cache[key]
            if component is RESERVED:
                raise CyclicGraphError(key)
            return component
        except KeyError:
            pass

        if self._locked:
            raise LockedGraphError(key)
        return self._resolve_key(key)

    def __setattr__(self, key: str, value: Component) -> None:
        if not key.startswith("_") and key not in ("metadata", "config", "loader"):
            raise Exception("Cannot setattr on ObjectGraph for key: {}".format(key))
        super(ObjectGraph, self).__setattr__(key, value)

    @contextmanager
    def _reserve(self, key: str) -> Any:
        """
        Reserve a component's binding temporarily.

        Protects against cycles.

        """
        self.assign(key, RESERVED)
        try:
            yield
        finally:
            del self._cache[key]

    def _resolve_key(self, key: str) -> Component:
        """
        Attempt to lazily create a component.

        :raises NotBoundError: if the component does not have a bound factory
        :raises CyclicGraphError: if the factory function requires a cycle
        :raises LockedGraphError: if the graph is locked
        """
        with self._reserve(key):
            factory = self.factory_for(key)
            with self._profiler(key):
                component = factory(self)
            invoke_resolve_hook(component)

        return self.assign(key, component)

    def items(self) -> Iterable[Tuple[str, Component]]:
        """
        Iterates over tuples of (key, component) for all bound components.
        """
        yield from self._cache.items()

    __getitem__ = __getattr__

def create_object_graph(
    name: str,
    debug: bool = False,
    testing: bool = False,
    import_name: Optional[str] = None,
    root_path: Optional[str] = None,
    loader: Callable[[Metadata], Configuration] = load_from_environ,
    registry: Registry = _registry,
    profiler: Any = None,
    cache: Optional[Type[Cache]] = None
) -> ObjectGraph:
    """
    Create a new object graph.

    :param name: the name of the microservice
    :param debug: is development debugging enabled?
    :param testing: is unit testing enabled?
    :param import_name: the import name to use for resource loading
    :param root_path: the root path for resource loading
    :param loader: the configuration loader to use
    :param registry: the registry to use (defaults to the global)
    :param profiler:
    :param cache:

    """
    metadata = Metadata(
        name=name,
        debug=debug,
        testing=testing,
        import_name=import_name,
        root_path=root_path,
    )

    defaults = registry.defaults
    config = configure(defaults, metadata, loader)

    if profiler is None:
        profiler = NoopProfiler()

    if cache is None or isinstance(cache, str):
        cache = create_cache(cache)

    return ObjectGraph(
        metadata=metadata,
        config=config,
        registry=registry,
        profiler=profiler,
        cache=cache,
        loader=loader,
    )

def get_component_name(graph: ObjectGraph, component: Component) -> str:
    """
    Given an object that is attached to the graph, it returns the object name.

    """
    return next(
        key
        for key, possible_component in graph.items()
        if possible_component == component
    )


"""Copied and modified from, https://github.com/runwayml/stable-diffusion/blob/main/ldm/models/diffusion/ddpm.py.
"""

from contextlib import contextmanager
from functools import partial

import numpy as np
import pytorch_lightning as pl
import torch
import torch.nn as nn
from distillsd.ldm.ddim import DDIMSampler
from distillsd.ldm.diffusion_utils import make_beta_schedule
from distillsd.ldm.ema import LitEma
from distillsd.ldm.util import exists, default, count_params, \
    instantiate_from_config
from distillsd.vae.autoencoder import VQModelInterface
from distillsd.vae.distributions import DiagonalGaussianDistribution
from einops import rearrange, repeat
from omegaconf import ListConfig

__conditioning_keys__ = {
    'concat': 'c_concat',
    'crossattn': 'c_crossattn',
    'adm': 'y'
}

def disabled_train(self, mode=True):
    """Overwrite model. train with this function to make sure train/eval mode
    does not change anymore."""
    return self

class DDPM(pl.LightningModule):
    # classic DDPM with Gaussian diffusion, in image space
    def __init__(
            self,
            unet_config,
            timesteps=1000,
            beta_schedule="linear",
            loss_type="l2",
            ckpt_path=None,
            ignore_keys=[],
            load_only_unet=False,
            monitor="val/loss",
            use_ema=True,
            first_stage_key="image",
            image_size=256,
            channels=3,
            log_every_t=100,
            clip_denoised=True,
            linear_start=1e-4,
            linear_end=2e-2,
            cosine_s=8e-3,
            given_betas=None,
            original_elbo_weight=0.,
            v_posterior=0.,  # weight for choosing posterior variance as sigma = (1-v) * beta_tilde + v * beta
            l_simple_weight=1.,
            conditioning_key=None,
            parameterization="eps",  # all assuming fixed variance schedules
            scheduler_config=None,
            use_positional_encodings=False,
            learn_logvar=False,
            logvar_init=0.,
    ):
        super().__init__()
        assert parameterization in ["eps", "x0"], 'currently only supporting "eps" and "x0"'
        self.parameterization = parameterization
        print(f"{self.__class__.__name__}: Running in {self.parameterization}-prediction mode")
        self.cond_stage_model = None
        self.clip_denoised = clip_denoised
        self.log_every_t = log_every_t
        self.first_stage_key = first_stage_key
        self.image_size = image_size  # try conv?
        self.channels = channels
        self.use_positional_encodings = use_positional_encodings
        self.model = DiffusionWrapper(unet_config, conditioning_key)
        count_params(self.model, verbose=True)
        self.use_ema = use_ema
        if self.use_ema:
            self.model_ema = LitEma(self.model)
            print(f"Keeping EMAs of {len(list(self.model_ema.buffers()))}.")

        self.use_scheduler = scheduler_config is not None
        if self.use_scheduler:
            self.scheduler_config = scheduler_config

        self.v_posterior = v_posterior
        self.original_elbo_weight = original_elbo_weight
        self.l_simple_weight = l_simple_weight

        if monitor is not None:
            self.monitor = monitor
        if ckpt_path is not None:
            self.init_from_ckpt(ckpt_path, ignore_keys=ignore_keys, only_model=load_only_unet)

        self.register_schedule(given_betas=given_betas, beta_schedule=beta_schedule, timesteps=timesteps,
                               linear_start=linear_start, linear_end=linear_end, cosine_s=cosine_s)

        self.loss_type = loss_type

        self.learn_logvar = learn_logvar
        self.logvar = torch.full(fill_value=logvar_init, size=(self.num_timesteps,))
        if self.learn_logvar:
            self.logvar = nn.Parameter(self.logvar, requires_grad=True)

    def register_schedule(
            self, given_betas=None, beta_schedule="linear", timesteps=1000,
            linear_start=1e-4, linear_end=2e-2, cosine_s=8e-3
    ):
        if exists(given_betas):
            betas = given_betas
        else:
            betas = make_beta_schedule(
                beta_schedule, timesteps, linear_start=linear_start, linear_end=linear_end,
                cosine_s=cosine_s
            )

        alphas = 1. - betas
        alphas_cumprod = np.cumprod(alphas, axis=0)
        alphas_cumprod_prev = np.append(1., alphas_cumprod[:-1])

        timesteps, = betas.shape
        self.num_timesteps = int(timesteps)
        self.linear_start = linear_start
        self.linear_end = linear_end
        assert alphas_cumprod.shape[0] == self.num_timesteps, 'alphas have to be defined for each timestep'

        to_torch = partial(torch.tensor, dtype=torch.float32)

        self.register_buffer('betas', to_torch(betas))
        self.register_buffer('alphas_cumprod', to_torch(alphas_cumprod))
        self.register_buffer('alphas_cumprod_prev', to_torch(alphas_cumprod_prev))

        # calculations for diffusion q(x_t | x_{t-1}) and others
        self.register_buffer('sqrt_alphas_cumprod', to_torch(np.sqrt(alphas_cumprod)))
        self.register_buffer('sqrt_one_minus_alphas_cumprod', to_torch(np.sqrt(1. - alphas_cumprod)))
        self.register_buffer('log_one_minus_alphas_cumprod', to_torch(np.log(1. - alphas_cumprod)))
        self.register_buffer('sqrt_recip_alphas_cumprod', to_torch(np.sqrt(1. / alphas_cumprod)))
        self.register_buffer('sqrt_recipm1_alphas_cumprod', to_torch(np.sqrt(1. / alphas_cumprod - 1)))

        # calculations for posterior q(x_{t-1} | x_t, x_0)
        posterior_variance = (1 - self.v_posterior) * betas * (1. - alphas_cumprod_prev) / (
                1. - alphas_cumprod) + self.v_posterior * betas
        # above: equal to 1. / (1. / (1. - alpha_cumprod_tm1) + alpha_t / beta_t)
        self.register_buffer('posterior_variance', to_torch(posterior_variance))
        # below: log calculation clipped because the posterior variance is 0 at the beginning of the diffusion chain
        self.register_buffer('posterior_log_variance_clipped', to_torch(np.log(np.maximum(posterior_variance, 1e-20))))
        self.register_buffer(
            'posterior_mean_coef1', to_torch(betas * np.sqrt(alphas_cumprod_prev) / (1. - alphas_cumprod))
        )
        self.register_buffer(
            'posterior_mean_coef2', to_torch((1. - alphas_cumprod_prev) * np.sqrt(alphas) / (1. - alphas_cumprod))
        )

    @contextmanager
    def ema_scope(self, context=None):
        if self.use_ema:
            self.model_ema.store(self.model.parameters())
            self.model_ema.copy_to(self.model)
            if context is not None:
                print(f"{context}: Switched to EMA weights")
        try:
            yield None
        finally:
            if self.use_ema:
                self.model_ema.restore(self.model.parameters())
                if context is not None:
                    print(f"{context}: Restored training weights")

    def init_from_ckpt(self, path, ignore_keys=list(), only_model=False):
        sd = torch.load(path, map_location="cpu")
        if "state_dict" in list(sd.keys()):
            sd = sd["state_dict"]
        keys = list(sd.keys())
        for k in keys:
            for ik in ignore_keys:
                if k.startswith(ik):
                    print("Deleting key {} from state_dict.".format(k))
                    del sd[k]
        missing, unexpected = self.load_state_dict(sd, strict=False) if not only_model else self.model.load_state_dict(
            sd, strict=False)
        print(f"Restored from {path} with {len(missing)} missing and {len(unexpected)} unexpected keys")
        if len(missing) > 0:
            print(f"Missing Keys: {missing}")
        if len(unexpected) > 0:
            print(f"Unexpected Keys: {unexpected}")

    def forward(self, x, *args, **kwargs):
        # b, c, h, w, device, img_size, = *x.shape, x.device, self.image_size
        # assert h == img_size and w == img_size, f'height and width of image must be {img_size}'
        t = torch.randint(0, self.num_timesteps, (x.shape[0],), device=self.device).long()
        return self.p_losses(x, t, *args, **kwargs)

class LatentDiffusion(DDPM):
    """main class"""

    def __init__(
            self,
            # first_stage_config,
            # cond_stage_config,
            num_timesteps_cond=None,
            cond_stage_key="image",
            cond_stage_trainable=False,
            concat_mode=True,
            cond_stage_forward=None,
            conditioning_key=None,
            scale_factor=1.0,
            scale_by_std=False,
            *args, **kwargs
    ):
        self.num_timesteps_cond = default(num_timesteps_cond, 1)
        self.scale_by_std = scale_by_std
        assert self.num_timesteps_cond <= kwargs['timesteps']
        # for backwards compatibility after implementation of DiffusionWrapper
        if conditioning_key is None:
            conditioning_key = 'concat' if concat_mode else 'crossattn'
        # if cond_stage_config == '__is_unconditional__':
        #     conditioning_key = None
        ckpt_path = kwargs.pop("ckpt_path", None)
        ignore_keys = kwargs.pop("ignore_keys", [])
        super().__init__(conditioning_key=conditioning_key, *args, **kwargs)
        self.concat_mode = concat_mode
        self.cond_stage_trainable = cond_stage_trainable
        self.cond_stage_key = cond_stage_key
        # try:
        #     self.num_downs = len(first_stage_config.params.ddconfig.ch_mult) - 1
        # except:
        #     self.num_downs = 0
        if not scale_by_std:
            self.scale_factor = scale_factor
        else:
            self.register_buffer('scale_factor', torch.tensor(scale_factor))
        # self.instantiate_first_stage(first_stage_config)
        # self.instantiate_cond_stage(cond_stage_config)
        self.cond_stage_forward = cond_stage_forward
        self.clip_denoised = False
        self.bbox_tokenizer = None

        self.restarted_from_ckpt = False
        if ckpt_path is not None:
            self.init_from_ckpt(ckpt_path, ignore_keys)
            self.restarted_from_ckpt = True

    def make_cond_schedule(self, ):
        self.cond_ids = torch.full(size=(self.num_timesteps,), fill_value=self.num_timesteps - 1, dtype=torch.long)
        ids = torch.round(torch.linspace(0, self.num_timesteps - 1, self.num_timesteps_cond)).long()
        self.cond_ids[:self.num_timesteps_cond] = ids

    def register_schedule(
            self,
            given_betas=None, beta_schedule="linear", timesteps=1000,
            linear_start=1e-4, linear_end=2e-2, cosine_s=8e-3
    ):
        super().register_schedule(given_betas, beta_schedule, timesteps, linear_start, linear_end, cosine_s)

        self.shorten_cond_schedule = self.num_timesteps_cond > 1
        if self.shorten_cond_schedule:
            self.make_cond_schedule()

    def instantiate_first_stage(self, config):
        model = instantiate_from_config(config)
        self.first_stage_model = model.eval()
        self.first_stage_model.train = disabled_train
        for param in self.first_stage_model.parameters():
            param.requires_grad = False

    def instantiate_cond_stage(self, config):
        if not self.cond_stage_trainable:
            if config == "__is_first_stage__":
                print("Using first stage also as cond stage.")
                self.cond_stage_model = self.first_stage_model
            elif config == "__is_unconditional__":
                print(f"Training {self.__class__.__name__} as an unconditional model.")
                self.cond_stage_model = None
                # self.be_unconditional = True
            else:
                model = instantiate_from_config(config)
                self.cond_stage_model = model.eval()
                self.cond_stage_model.train = disabled_train
                for param in self.cond_stage_model.parameters():
                    param.requires_grad = False
        else:
            assert config != '__is_first_stage__'
            assert config != '__is_unconditional__'
            model = instantiate_from_config(config)
            self.cond_stage_model = model

    def get_first_stage_encoding(self, encoder_posterior):
        if isinstance(encoder_posterior, DiagonalGaussianDistribution):
            z = encoder_posterior.sample()
        elif isinstance(encoder_posterior, torch.Tensor):
            z = encoder_posterior
        else:
            raise NotImplementedError(f"encoder_posterior of type '{type(encoder_posterior)}' not yet implemented")
        return self.scale_factor * z

    def get_learned_conditioning(self, c):
        if self.cond_stage_forward is None:
            if hasattr(self.cond_stage_model, 'encode') and callable(self.cond_stage_model.encode):
                c = self.cond_stage_model.encode(c)
                if isinstance(c, DiagonalGaussianDistribution):
                    c = c.mode()
            else:
                c = self.cond_stage_model(c)
        else:
            assert hasattr(self.cond_stage_model, self.cond_stage_forward)
            c = getattr(self.cond_stage_model, self.cond_stage_forward)(c)
        return c

    def meshgrid(self, h, w):
        y = torch.arange(0, h).view(h, 1, 1).repeat(1, w, 1)
        x = torch.arange(0, w).view(1, w, 1).repeat(h, 1, 1)

        arr = torch.cat([y, x], dim=-1)
        return arr

    def delta_border(self, h, w):
        """
        :param h: height
        :param w: width
        :return: normalized distance to image border,
         wtith min distance = 0 at border and max dist = 0.5 at image center
        """
        lower_right_corner = torch.tensor([h - 1, w - 1]).view(1, 1, 2)
        arr = self.meshgrid(h, w) / lower_right_corner
        dist_left_up = torch.min(arr, dim=-1, keepdims=True)[0]
        dist_right_down = torch.min(1 - arr, dim=-1, keepdims=True)[0]
        edge_dist = torch.min(torch.cat([dist_left_up, dist_right_down], dim=-1), dim=-1)[0]
        return edge_dist

    def get_weighting(self, h, w, Ly, Lx, device):
        weighting = self.delta_border(h, w)
        weighting = torch.clip(weighting, self.split_input_params["clip_min_weight"],
                               self.split_input_params["clip_max_weight"], )
        weighting = weighting.view(1, h * w, 1).repeat(1, 1, Ly * Lx).to(device)

        if self.split_input_params["tie_braker"]:
            L_weighting = self.delta_border(Ly, Lx)
            L_weighting = torch.clip(L_weighting,
                                     self.split_input_params["clip_min_tie_weight"],
                                     self.split_input_params["clip_max_tie_weight"])

            L_weighting = L_weighting.view(1, 1, Ly * Lx).to(device)
            weighting = weighting * L_weighting
        return weighting

    def get_fold_unfold(self, x, kernel_size, stride, uf=1, df=1):  # todo load once not every time, shorten code
        """
        :param x: img of size (bs, c, h, w)
        :return: n img crops of size (n, bs, c, kernel_size[0], kernel_size[1])
        """
        bs, nc, h, w = x.shape

        # number of crops in image
        Ly = (h - kernel_size[0]) // stride[0] + 1
        Lx = (w - kernel_size[1]) // stride[1] + 1

        if uf == 1 and df == 1:
            fold_params = dict(kernel_size=kernel_size, dilation=1, padding=0, stride=stride)
            unfold = torch.nn.Unfold(**fold_params)

            fold = torch.nn.Fold(output_size=x.shape[2:], **fold_params)

            weighting = self.get_weighting(kernel_size[0], kernel_size[1], Ly, Lx, x.device).to(x.dtype)
            normalization = fold(weighting).view(1, 1, h, w)  # normalizes the overlap
            weighting = weighting.view((1, 1, kernel_size[0], kernel_size[1], Ly * Lx))

        elif uf > 1 and df == 1:
            fold_params = dict(kernel_size=kernel_size, dilation=1, padding=0, stride=stride)
            unfold = torch.nn.Unfold(**fold_params)

            fold_params2 = dict(kernel_size=(kernel_size[0] * uf, kernel_size[0] * uf),
                                dilation=1, padding=0,
                                stride=(stride[0] * uf, stride[1] * uf))
            fold = torch.nn.Fold(output_size=(x.shape[2] * uf, x.shape[3] * uf), **fold_params2)

            weighting = self.get_weighting(kernel_size[0] * uf, kernel_size[1] * uf, Ly, Lx, x.device).to(x.dtype)
            normalization = fold(weighting).view(1, 1, h * uf, w * uf)  # normalizes the overlap
            weighting = weighting.view((1, 1, kernel_size[0] * uf, kernel_size[1] * uf, Ly * Lx))

        elif df > 1 and uf == 1:
            fold_params = dict(kernel_size=kernel_size, dilation=1, padding=0, stride=stride)
            unfold = torch.nn.Unfold(**fold_params)

            fold_params2 = dict(kernel_size=(kernel_size[0] // df, kernel_size[0] // df),
                                dilation=1, padding=0,
                                stride=(stride[0] // df, stride[1] // df))
            fold = torch.nn.Fold(output_size=(x.shape[2] // df, x.shape[3] // df), **fold_params2)

            weighting = self.get_weighting(kernel_size[0] // df, kernel_size[1] // df, Ly, Lx, x.device).to(x.dtype)
            normalization = fold(weighting).view(1, 1, h // df, w // df)  # normalizes the overlap
            weighting = weighting.view((1, 1, kernel_size[0] // df, kernel_size[1] // df, Ly * Lx))

        else:
            raise NotImplementedError

        return fold, unfold, normalization, weighting

    @torch.no_grad()
    def decode_first_stage(self, z, predict_cids=False, force_not_quantize=False):
        if predict_cids:
            if z.dim() == 4:
                z = torch.argmax(z.exp(), dim=1).long()
            z = self.first_stage_model.quantize.get_codebook_entry(z, shape=None)
            z = rearrange(z, 'b h w c -> b c h w').contiguous()

        z = 1. / self.scale_factor * z

        if hasattr(self, "split_input_params"):
            if self.split_input_params["patch_distributed_vq"]:
                ks = self.split_input_params["ks"]  # eg. (128, 128)
                stride = self.split_input_params["stride"]  # eg. (64, 64)
                uf = self.split_input_params["vqf"]
                bs, nc, h, w = z.shape
                if ks[0] > h or ks[1] > w:
                    ks = (min(ks[0], h), min(ks[1], w))
                    print("reducing Kernel")

                if stride[0] > h or stride[1] > w:
                    stride = (min(stride[0], h), min(stride[1], w))
                    print("reducing stride")

                fold, unfold, normalization, weighting = self.get_fold_unfold(z, ks, stride, uf=uf)

                z = unfold(z)  # (bn, nc * prod(**ks), L)
                # 1. Reshape to img shape
                z = z.view((z.shape[0], -1, ks[0], ks[1], z.shape[-1]))  # (bn, nc, ks[0], ks[1], L )

                # 2. apply model loop over last dim
                if isinstance(self.first_stage_model, VQModelInterface):
                    output_list = [
                        self.first_stage_model.decode(
                            z[:, :, :, :, i],
                            force_not_quantize=predict_cids or force_not_quantize
                        ) for i in range(z.shape[-1])
                    ]
                else:
                    output_list = [
                        self.first_stage_model.decode(z[:, :, :, :, i]) for i in range(z.shape[-1])
                    ]

                o = torch.stack(output_list, axis=-1)  # # (bn, nc, ks[0], ks[1], L)
                o = o * weighting
                # Reverse 1. reshape to img shape
                o = o.view((o.shape[0], -1, o.shape[-1]))  # (bn, nc * ks[0] * ks[1], L)
                # stitch crops together
                decoded = fold(o)
                decoded = decoded / normalization  # norm is shape (1, 1, h, w)
                return decoded
            else:
                if isinstance(self.first_stage_model, VQModelInterface):
                    return self.first_stage_model.decode(z, force_not_quantize=predict_cids or force_not_quantize)
                else:
                    return self.first_stage_model.decode(z)

        else:
            if isinstance(self.first_stage_model, VQModelInterface):
                return self.first_stage_model.decode(z, force_not_quantize=predict_cids or force_not_quantize)
            else:
                return self.first_stage_model.decode(z)

    @torch.no_grad()
    def encode_first_stage(self, x):
        if hasattr(self, "split_input_params"):
            if self.split_input_params["patch_distributed_vq"]:
                ks = self.split_input_params["ks"]  # eg. (128, 128)
                stride = self.split_input_params["stride"]  # eg. (64, 64)
                df = self.split_input_params["vqf"]
                self.split_input_params['original_image_size'] = x.shape[-2:]
                bs, nc, h, w = x.shape
                if ks[0] > h or ks[1] > w:
                    ks = (min(ks[0], h), min(ks[1], w))
                    print("reducing Kernel")

                if stride[0] > h or stride[1] > w:
                    stride = (min(stride[0], h), min(stride[1], w))
                    print("reducing stride")

                fold, unfold, normalization, weighting = self.get_fold_unfold(x, ks, stride, df=df)
                z = unfold(x)  # (bn, nc * prod(**ks), L)
                # Reshape to img shape
                z = z.view((z.shape[0], -1, ks[0], ks[1], z.shape[-1]))  # (bn, nc, ks[0], ks[1], L )

                output_list = [self.first_stage_model.encode(z[:, :, :, :, i])
                               for i in range(z.shape[-1])]

                o = torch.stack(output_list, axis=-1)
                o = o * weighting

                # Reverse reshape to img shape
                o = o.view((o.shape[0], -1, o.shape[-1]))  # (bn, nc * ks[0] * ks[1], L)
                # stitch crops together
                decoded = fold(o)
                decoded = decoded / normalization
                return decoded

            else:
                return self.first_stage_model.encode(x)
        else:
            return self.first_stage_model.encode(x)

    def forward(self, x, c, *args, **kwargs):
        t = torch.randint(0, self.num_timesteps, (x.shape[0],), device=self.device).long()
        if self.model.conditioning_key is not None:
            assert c is not None
            if self.cond_stage_trainable:
                c = self.get_learned_conditioning(c)
            if self.shorten_cond_schedule:  # TODO: drop this option
                tc = self.cond_ids[t].to(self.device)
                c = self.q_sample(x_start=c, t=tc, noise=torch.randn_like(c.float()))
        return self.p_losses(x, c, t, *args, **kwargs)

    def apply_model(self, x_noisy, t, cond, return_ids=False):

        if isinstance(cond, dict):
            # hybrid case, cond is exptected to be a dict
            pass
        else:
            if not isinstance(cond, list):
                cond = [cond]
            key = 'c_concat' if self.model.conditioning_key == 'concat' else 'c_crossattn'
            cond = {key: cond}

        if hasattr(self, "split_input_params"):
            assert len(cond) == 1  # todo can only deal with one conditioning atm
            assert not return_ids
            ks = self.split_input_params["ks"]  # eg. (128, 128)
            stride = self.split_input_params["stride"]  # eg. (64, 64)

            h, w = x_noisy.shape[-2:]

            fold, unfold, normalization, weighting = self.get_fold_unfold(x_noisy, ks, stride)

            z = unfold(x_noisy)  # (bn, nc * prod(**ks), L)
            # Reshape to img shape
            z = z.view((z.shape[0], -1, ks[0], ks[1], z.shape[-1]))  # (bn, nc, ks[0], ks[1], L )
            z_list = [z[:, :, :, :, i] for i in range(z.shape[-1])]

            if self.cond_stage_key in ["image", "LR_image", "segmentation",
                                       'bbox_img'] and self.model.conditioning_key:  # todo check for completeness
                c_key = next(iter(cond.keys()))  # get key
                c = next(iter(cond.values()))  # get value
                assert (len(c) == 1)  # todo extend to list with more than one elem
                c = c[0]  # get element

                c = unfold(c)
                c = c.view((c.shape[0], -1, ks[0], ks[1], c.shape[-1]))  # (bn, nc, ks[0], ks[1], L )

                cond_list = [{c_key: [c[:, :, :, :, i]]} for i in range(c.shape[-1])]

            elif self.cond_stage_key == 'coordinates_bbox':
                assert 'original_image_size' in self.split_input_params, 'BoudingBoxRescaling is missing original_image_size'

                # assuming padding of unfold is always 0 and its dilation is always 1
                n_patches_per_row = int((w - ks[0]) / stride[0] + 1)
                full_img_h, full_img_w = self.split_input_params['original_image_size']
                # as we are operating on latents, we need the factor from the original image size to the
                # spatial latent size to properly rescale the crops for regenerating the bbox annotations
                num_downs = self.first_stage_model.encoder.num_resolutions - 1
                rescale_latent = 2 ** (num_downs)

                # get top left postions of patches as conforming for the bbbox tokenizer, therefore we
                # need to rescale the tl patch coordinates to be in between (0,1)
                tl_patch_coordinates = [(rescale_latent * stride[0] * (patch_nr % n_patches_per_row) / full_img_w,
                                         rescale_latent * stride[1] * (patch_nr // n_patches_per_row) / full_img_h)
                                        for patch_nr in range(z.shape[-1])]

                # patch_limits are tl_coord, width and height coordinates as (x_tl, y_tl, h, w)
                patch_limits = [(x_tl, y_tl,
                                 rescale_latent * ks[0] / full_img_w,
                                 rescale_latent * ks[1] / full_img_h) for x_tl, y_tl in tl_patch_coordinates]
                # patch_values = [(np.arange(x_tl,min(x_tl+ks, 1.)),np.arange(y_tl,min(y_tl+ks, 1.))) for x_tl, y_tl in tl_patch_coordinates]

                # tokenize crop coordinates for the bounding boxes of the respective patches
                patch_limits_tknzd = [torch.LongTensor(self.bbox_tokenizer._crop_encoder(bbox))[None].to(self.device)
                                      for bbox in patch_limits]  # list of length l with tensors of shape (1, 2)
                print(patch_limits_tknzd[0].shape)
                # cut tknzd crop position from conditioning
                assert isinstance(cond, dict), 'cond must be dict to be fed into model'
                cut_cond = cond['c_crossattn'][0][..., :-2].to(self.device)
                print(cut_cond.shape)

                adapted_cond = torch.stack([torch.cat([cut_cond, p], dim=1) for p in patch_limits_tknzd])
                adapted_cond = rearrange(adapted_cond, 'l b n -> (l b) n')
                print(adapted_cond.shape)
                adapted_cond = self.get_learned_conditioning(adapted_cond)
                print(adapted_cond.shape)
                adapted_cond = rearrange(adapted_cond, '(l b) n d -> l b n d', l=z.shape[-1])
                print(adapted_cond.shape)

                cond_list = [{'c_crossattn': [e]} for e in adapted_cond]

            else:
                cond_list = [cond for i in range(z.shape[-1])]  # Todo make this more efficient

            # apply model by loop over crops
            output_list = [self.model(z_list[i], t, **cond_list[i]) for i in range(z.shape[-1])]
            assert not isinstance(output_list[0],
                                  tuple)  # todo cant deal with multiple model outputs check this never happens

            o = torch.stack(output_list, axis=-1)
            o = o * weighting
            # Reverse reshape to img shape
            o = o.view((o.shape[0], -1, o.shape[-1]))  # (bn, nc * ks[0] * ks[1], L)
            # stitch crops together
            x_recon = fold(o) / normalization

        else:
            x_recon = self.model(x_noisy, t, **cond)

        if isinstance(x_recon, tuple) and not return_ids:
            return x_recon[0]
        else:
            return x_recon

    @torch.no_grad()
    def sample(
            self, cond, batch_size=16, return_intermediates=False, x_T=None,
            verbose=True, timesteps=None, quantize_denoised=False,
            mask=None, x0=None, shape=None, **kwargs
    ):
        if shape is None:
            shape = (batch_size, self.channels, self.image_size, self.image_size)
        if cond is not None:
            if isinstance(cond, dict):
                cond = {key: cond[key][:batch_size] if not isinstance(cond[key], list) else
                list(map(lambda x: x[:batch_size], cond[key])) for key in cond}
            else:
                cond = [c[:batch_size] for c in cond] if isinstance(cond, list) else cond[:batch_size]
        return self.p_sample_loop(
            cond,
            shape,
            return_intermediates=return_intermediates, x_T=x_T,
            verbose=verbose, timesteps=timesteps, quantize_denoised=quantize_denoised,
            mask=mask, x0=x0
        )

    @torch.no_grad()
    def sample_log(self, cond, batch_size, ddim, ddim_steps, **kwargs):

        if ddim:
            ddim_sampler = DDIMSampler(self)
            shape = (self.channels, self.image_size, self.image_size)
            samples, intermediates = ddim_sampler.sample(ddim_steps, batch_size, shape, cond, verbose=False, **kwargs)

        else:
            samples, intermediates = self.sample(cond=cond, batch_size=batch_size, return_intermediates=True, **kwargs)

        return samples, intermediates

    @torch.no_grad()
    def get_unconditional_conditioning(self, batch_size, null_label=None):
        if null_label is not None:
            xc = null_label
            if isinstance(xc, ListConfig):
                xc = list(xc)
            if isinstance(xc, dict) or isinstance(xc, list):
                c = self.get_learned_conditioning(xc)
            else:
                if hasattr(xc, "to"):
                    xc = xc.to(self.device)
                c = self.get_learned_conditioning(xc)
        else:
            # todo: get null label from cond_stage_model
            raise NotImplementedError()
        c = repeat(c, "1 ... -> b ...", b=batch_size).to(self.device)
        return c

    @torch.no_grad()
    def to_rgb(self, x):
        x = x.float()
        if not hasattr(self, "colorize"):
            self.colorize = torch.randn(3, x.shape[1], 1, 1).to(x)
        x = nn.functional.conv2d(x, weight=self.colorize)
        x = 2. * (x - x.min()) / (x.max() - x.min()) - 1.
        return x

class DiffusionWrapper(pl.LightningModule):
    def __init__(self, diff_model_config, conditioning_key):
        super().__init__()
        self.diffusion_model = instantiate_from_config(diff_model_config)
        self.conditioning_key = conditioning_key
        assert self.conditioning_key in [None, 'concat', 'crossattn', 'hybrid', 'adm']

    def forward(self, x, t, c_concat: list = None, c_crossattn: list = None):
        if self.conditioning_key is None:
            out = self.diffusion_model(x, t)
        elif self.conditioning_key == 'concat':
            xc = torch.cat([x] + c_concat, dim=1)
            out = self.diffusion_model(xc, t)
        elif self.conditioning_key == 'crossattn':
            cc = torch.cat(c_crossattn, 1)
            out = self.diffusion_model(x, t, context=cc)
        elif self.conditioning_key == 'hybrid':
            xc = torch.cat([x] + c_concat, dim=1)
            cc = torch.cat(c_crossattn, 1)
            out = self.diffusion_model(xc, t, context=cc)
        elif self.conditioning_key == 'adm':
            cc = c_crossattn[0]
            out = self.diffusion_model(x, t, y=cc)
        else:
            raise NotImplementedError()

        return out


import random
import time

def native_search(list,target):
    for i in range(len(list)):
        if list[i] == target:
            return i
    return -1
import time

def binary_search(list, target, left = None, right=None):
    if left is None:
        left = 0
    if right is None:
        right = len(list) - 1
    # The target not in the list
    if right < left:
        return -1
    # Middle index of the list
    midpoint = (left + right) // 2
    if list[midpoint] == target:
        return midpoint
    elif list[midpoint] > target:
        return binary_search(list, target, left, midpoint -1)
    else:
        #list[midpoint] < target
        return binary_search(list, target, midpoint + 1 , right)

if __name__ =='__main__':

    length = int(input('Input length of list: '))
    print("Length of list:", length)
    list = set()
    #build a random list an then sorted it
    while len(list)  < length:
        # Add random int number in range (-length, length)
        list.add(random.randint(-length, length))
    sorted_list =sorted(list)

    start = time.time()
    #Search every element in sorted_list
    for target in sorted_list:
        native_search(sorted_list,target)
    end = time.time()
    print("Native search time: ", (end - start) / length, "seconds")

    start = time.time()
    #Search every element in sorted_list
    for target in sorted_list:
        binary_search(sorted_list,target)
    end = time.time()
    print("Binary search time: ", (end - start) / length, "seconds")


# coding = utf-8
import shutil
from multiprocessing import Process
import docx
import os
import glob

# 定义自动创建文件夹的函数
def make_folder(p):
    if os.path.exists(p):  # 判断文件夹是否存在
        shutil.rmtree(p)  # 如果存在删除原有目录的文件夹以及其中所有文件
    os.mkdir(p)

def clear_all_user_information(t):
    if t.cell(0, 0).text == '编号':  # 判断是否为含有编号的单元格，含有即为用户信息表
        m = 0  # 创建变量m收集行数，每张表进行一次初始化
        # 遍历行
        for i in range(len(t.rows)):
            n = 0  # 创建变量n收集列数，每行进行一次初始化
            m += 1  # 统计行数
            # 每遍历一次行遍历该行中所有列
            for j in range(len(t.rows[i].cells)):
                n += 1  # 统计列数
                if t.cell(i, j).text in ['政治面貌', '班级', '电子邮件', '联系电话']:  # 判断如果为第二行则开始清洗内容
                    run = t.cell(i, j).paragraphs[0].clear()  # 清空单元格段落数据
                    t.cell(i, j).merge(t.cell(i, len(t.rows[i].cells) - 1))  # 合并单元格
                    run = t.cell(i, j).paragraphs[0].clear()  # 清空单元格段落数据

if __name__ == '__main__':
    base_path = os.getcwd()
    readfile_path = base_path + '/tihuanmingzi'
    savefile_path = base_path + '/qingxixinxi'
    make_folder(savefile_path)
    for file_name in glob.glob(os.path.join(readfile_path, '*.docx')):
        file = docx.Document(file_name)
        tables = file.tables  # 读取所有表格
        # 遍历所有表，迭代得出每一张表
        for i in range(len(tables)):
            t = tables[i]  # 将每一张表传递给参数t
            p = Process(target=clear_all_user_information(t))  # Process 对象只是一个子任务，运行该任务时系统会自动创建一个子进程
            # 启动一个子进程来运行子任务
            p.start()
            p.join()
            # 子进程完成后，继续运行主进程,保存文件
        if base_path[0] == '/':
            new_name = file_name.strip(readfile_path)  # 去掉文件路径只保留文件名
            file.save(savefile_path + '/' + new_name)  # 保存文件
        else:
            new_name = file_name.split('\\')[-1]  # 去掉文件路径只保留文件名
            file.save(savefile_path + '\\' + new_name)  # 保存文件


import numpy as np

from utils.constants import (
    DEFAULT_GRAD_SMOOTHING_VAL,
    DEFAULT_CLASSIFIER_LOSS,
)

def find_gradient(
    loss_type: str,
    predictions: np.ndarray,
    targets: np.ndarray,
) -> np.ndarray:
    """
    Computes the gradient for the given loss function w.r.t the prediction target
    ex) gradient for cross-entropy loss:
        d_loss_d_pred = -target/pred

    :return: the gradient matrix of size len(targets)
    """
    if loss_type == DEFAULT_CLASSIFIER_LOSS:
        return -(targets + DEFAULT_GRAD_SMOOTHING_VAL) / (
            predictions + DEFAULT_GRAD_SMOOTHING_VAL
        )
    else:
        NotImplementedError("Invalid choice of loss function")

def find_hessian(
    loss_type: str, predictions: np.ndarray, targets: np.ndarray
) -> np.ndarray:
    """
    Computes the hessian for the given loss function w.r.t the prediction target
    ex) hessian for cross-entropy loss:
        d_loss_d_pred = target/pred^2

    :return: the gradient matrix of size len(targets)
    """
    if loss_type == DEFAULT_CLASSIFIER_LOSS:
        return (targets + DEFAULT_GRAD_SMOOTHING_VAL) / (
            np.square(predictions) + DEFAULT_GRAD_SMOOTHING_VAL
        )
    else:
        NotImplementedError("Invalid choice of loss function")

def get_next_targets(
    loss_type: str,
    is_classification,
    targets: np.ndarray,
    predictions: np.ndarray,
) -> np.ndarray:
    """
    Updates the targets for the next iteration of boosting. For classification, the resulting new training set will
    look like {X, -grad/hessian} and for regression, it will look like {X, ensemble_residuals}. This function assumes
    the tree is already fitted.

    :return: the new targets
    """
    if is_classification:
        return -find_gradient(loss_type, predictions, targets) / find_hessian(
            loss_type, predictions, targets
        )
    else:
        return targets - predictions


import discord
from discord.ext import commands
from discord.ext.tasks import loop
from jotform import JotformAPIClient
import json

client = commands.Bot(command_prefix='key ')

def load_data():
    with open('keys.json', 'r') as f:
        return json.load(f)

@client.command()
async def enter(ctx, code):
    success_role = ctx.guild.get_role(690400019282526229)
    data = load_data()
    if data.get(ctx.author.name, None) == code or data.get(str(ctx.author), None) == code or data.get(str(ctx.author.nick), None) == code:
        await ctx.send('https://66.media.tumblr.com/6924ed150fd01ad56327c46d29d05285/tumblr_nxv9vnr00V1s3uawvo1_400.gif')
        await ctx.send('Access granted, you may enter the castle.')
        await ctx.author.add_roles(success_role)
    else:
        await ctx.send('https://thumbs.gfycat.com/IdenticalBackJaguarundi-size_restricted.gif')
        await ctx.send('Access denied, you\'re not getting in. Try again in 5 minutes.')

client.run('')


import sys
import xml.etree.ElementTree as ET
from symbol_classifier import *

#=====================================================================
#  Submission for task1 of CROHME 2014
#
#  Created by:
#      - Kenny Davila (Apr 13, 2014)
#
#======================================================================

#the current XML namespace prefix...
INKML_NAMESPACE = '{http://www.w3.org/2003/InkML}'
#the number of results to show
TOP_N = 10

def load_inkml_traces(file_name):
    #first load the tree...
    tree = ET.parse(file_name)
    root = tree.getroot()

    #get all annotations...
    annotations = root.findall(INKML_NAMESPACE + 'annotation')
    #find UI annotation...
    UI = None
    for annotation in annotations:
        if "type" in annotation.attrib:
            if annotation.attrib["type"] == "UI":
                UI = annotation.text
                break

    #extract all the traces first...
    traces_list = []
    for trace in root.findall(INKML_NAMESPACE + 'trace'):
        #text contains all points as string, parse them and put them
        #into a list of tuples...
        points_s = trace.text.split(",");
        points_f = []
        for p_s in points_s:
            #split again...
            coords_s = p_s.split()
            #add...
            points_f.append( (float(coords_s[0]), float(coords_s[1])) )

        trace_id = int(trace.attrib['id'])

        trace_info = (trace_id, points_f)
        traces_list.append(trace_info)

    return UI, traces_list

def main():
    #usage check
    if len(sys.argv) != 2:
        print("Usage: python task1_submission.py list_filename")
        print("Where")
        print("\tlist_filename\t= Path to list of .inkml files to process")
        return

    classifier = SymbolClassifier("./task1_classifier.rsvm", "./task1_parameters.dat")

    #now, load the file with the parameters
    list_filename = sys.argv[1]
    list_file = open(list_filename, 'r')
    all_filenames = list_file.readlines()
    list_file.close()

    for filename in all_filenames:
        #process inkml file....
        UI, traces = load_inkml_traces(filename.strip())

        #now, classify...
        confidences = classifier.classify(traces)

        top_classes = classifier.topNLabels(confidences, TOP_N)

        class_str = ""
        scores_str = ""
        for label, score in top_classes:
            class_str += "," + label
            scores_str += "," + str(round(score, 4))

        print(UI + class_str)
        print("scores" + scores_str)
        #print(top_10)

main()


import tensorflow as tf, pandas as pd, numpy as np
import os
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM,Dense,Bidirectional,GRU, Dropout
from keras.callbacks import EarlyStopping , ReduceLROnPlateau
from tensorflow.keras.optimizers import Adam

def split_xy_yr(dataset, time_steps, y_time_steps):
    x,y = list(), list()

    for i in range(len(dataset)):
        x_end_number = i + time_steps
        y_end_number = x_end_number + y_time_steps

        if y_end_number > len(dataset):
            break

        tmp_x = dataset[i:x_end_number,0:-1]
        tmp_y = dataset[x_end_number:y_end_number,-1] # , -1
        x.append(tmp_x)
        y.append(tmp_y)

    return np.array(x),np.array(y)

path = os.path.dirname(os.path.realpath(__file__))

bitcoin = pd.read_csv(path + '/BTC_USD.csv',header=0).iloc[:370,:]\
        .sort_values(['날짜'],ascending=True,ignore_index=True)\
        .rename(columns={'오픈':'시가','변동 %':'타겟'})\
        [['날짜','시가','고가','저가','종가','거래량','타겟']]

price_list = ['시가','고가','저가','종가']

for p in price_list:
    for i,price in enumerate(bitcoin[p]):
        price = float(price.replace(',',''))
        bitcoin[p][i] = price

for i,deal in enumerate(bitcoin['거래량']):
    deal = deal.replace('K','000').replace('.','')
    bitcoin['거래량'][i] = deal

for i,target in enumerate(bitcoin['타겟']):
    target = float(target.replace('%',''))
    if target > 0: bitcoin['타겟'][i] = 1
    elif target < 0: bitcoin['타겟'][i] = 0

data = np.array(bitcoin[['시가','종가','타겟']])

x,y = split_xy_yr(data,3,1)
split_time = 50

x_train = x[:split_time]
x_test = x[split_time:]
y_train = y[:split_time]
y_test = y[split_time:]

x_train=np.asarray(x_train).astype(np.float64)
x_test=np.asarray(x_test).astype(np.float64)
y_train=np.asarray(y_train).astype(np.float64)
y_test=np.asarray(y_test).astype(np.float64)

model = Sequential()
model.add(LSTM(100,input_shape=(3,2)))
model.add(LSTM(128, activation='relu'))
model.add(Dropout(0.2))
model.add(LSTM(64, activation='relu'))
model.add(LSTM(2, activation='relu'))
model.add(Dense(2,activation='softmax'))

model.summary()
# exit()
optimizer = Adam(learning_rate=0.0001)  # 1e-4
lr=ReduceLROnPlateau(monitor= "val_acc", patience = 60, mode='max',factor = 0.1, min_lr=0.000001,verbose=False)
es = EarlyStopping(monitor ="val_acc", patience=15, mode='max',verbose=1,restore_best_weights=True)

model.compile(loss='sparse_categorical_crossentropy', optimizer=optimizer, metrics=['acc'])

model.fit(x_train,y_train,batch_size=2,epochs=1000,validation_data=(x_test,y_test),callbacks=[lr,es], verbose=1)

y_pred = model.predict(x_test)
y_pred_int  = np.argmax(y_pred,axis=1)
print(y_pred[-1:])
print(y_pred_int[-1:])


from fdfs_client.client import Fdfs_client
from ihome import constants
import sys

def upload_image(image_data):
    """
    上传图片
    :param image_data: 图片的二进制数据
    :return: 返回图片的访问路径
    """

    # 创建Fdfs_client对象

    client = Fdfs_client(sys.path[0] + constants.FDFS_CLIENT_CONFIG_PATH)
    # 上传文件的流
    res = client.upload_by_buffer(image_data)
    # return dict
    # {
    #     'Group name': group_name,
    #     'Remote file_id': remote_file_id,
    #     'Status': 'Upload successed.',
    #     'Local file name': '',
    #     'Uploaded size': upload_size,
    #     'Storage IP': storage_ip
    # }
    if res.get('Status') != 'Upload successed.':
        # 上传失败
        raise Exception('上传文件到fast dfs失败')
    # 获取返回文件的ID
    filename = res.get("Remote file_id")

    return filename

if __name__ == '__main__':
    upload_image("")


import mwparserfromhell
from mwparserfromhell import nodes

def parseTemplate(node):
    result = {}
    name = str(node.name).strip().lower()
    result['NAME'] = name
    for param in node.params:
        if param.showkey:
            key = str(param.name).strip().lower()
            value = str(param.value).strip()
            result[key] = value
        else:
            key = 'UNNAMED'
            value = str(param.value).strip()
            result[key] = value
    return (result)

def normaliseTemplate(template, askGoogle):
    # First use DOI in template
    if 'doi' in template and template['doi'] != '':
        return 'DOI:' + template['doi']
    if template['NAME'] == 'doi' and 'UNNAMED' in template and template['UNNAMED'] != '':
        return 'DOI:' + template['UNNAMED']
    # Use ISBN, if present
    if 'isbn' in template and template['isbn'] != '':
        return 'ISBN:' + template['isbn'].replace('-', ' ').replace(' ', '')
    if template['NAME'] == 'isbn' and 'UNNAMED' in template and template['UNNAMED'] != '':
        return 'ISBN:' + template['UNNAMED'].replace('-', ' ').replace(' ', '')
    # Use arXiv ID for arXiv templates:
    if template['NAME'] == 'cite arxiv':
        for key in ['arxiv', 'eprint']:
            if key in template and template[key] != '':
                return 'ARXIV:' + template[key]
    # Use URL for all citation templates
    if template['NAME'].startswith('cite ') or template['NAME'] == 'citation':
        for key in ['url', 'chapter-url', 'chapterurl', 'contribution-url', 'contributionurl', 'section-url',
                    'sectionurl']:
            if key in template and template[key] != '':
                return 'URL:' + template[key]
    # Ask google for ISBN for books
    if askGoogle is not None:
        if template['NAME'] in ['cite book', 'citation']:
            title = ''
            if 'title' in template and template['title'] != '':
                title = template['title']
            author = ''
            for key in ['last', 'last1']:
                if key in template and template[key] != '':
                    author = template[key]
            if title != '':
                isbn = askGoogle.askGoogleForISBN(title, author)
                if isbn != '':
                    return 'ISBN:' + isbn.replace('-', ' ').replace(' ', '')
    return ''

def normalise(wikitext, askGoogle=None):
    wiki = mwparserfromhell.parse(wikitext)
    templates = []
    for node in wiki.nodes:
        if isinstance(node, nodes.template.Template):
            parsed = parseTemplate(node)
            if parsed['NAME'] in ['dead link', 'webarchive']:
                continue
            templates.append(parsed)
    result = []
    # Find IDs by templates
    for template in templates:
        norm = normaliseTemplate(template, askGoogle)
        if norm != '' and '{{' not in norm:
            result.append(norm)
    # Find free-standing URLs
    for node in wiki.nodes:
        if isinstance(node, nodes.external_link.ExternalLink):
            if str(node.url) != '' and '{{' not in str(node.url):
                result.append('URL:' + str(node.url))
        elif isinstance(node, nodes.tag.Tag) and str(node.tag) in ['b', 'i']:
            for node2 in node.contents.nodes:
                if isinstance(node2, nodes.external_link.ExternalLink):
                    if str(node2.url) != '' and '{{' not in str(node2.url):
                        result.append('URL:' + str(node2.url))

    return result


#!usr/bin/env python3
#-*- coding: utf-8 -*-
"""
'labels.py' (relation_extractor/resources)

Class label mappings for relation extraction datasets

2019 Steve Neale <steveneale3000@gmail.com>

"""

semeval_classes = { "Other": 0,
                    "Message-Topic(e1,e2)": 1,
                    "Message-Topic(e2,e1)": 2,
                    "Product-Producer(e1,e2)": 3,
                    "Product-Producer(e2,e1)": 4,
                    "Instrument-Agency(e1,e2)": 5,
                    "Instrument-Agency(e2,e1)": 6,
                    "Entity-Destination(e1,e2)": 7,
                    "Entity-Destination(e2,e1)": 8,
                    "Cause-Effect(e1,e2)": 9,
                    "Cause-Effect(e2,e1)": 10,
                    "Component-Whole(e1,e2)": 11,
                    "Component-Whole(e2,e1)": 12,
                    "Entity-Origin(e1,e2)": 13,
                    "Entity-Origin(e2,e1)": 14,
                    "Member-Collection(e1,e2)": 15,
                    "Member-Collection(e2,e1)": 16,
                    "Content-Container(e1,e2)": 17,
                    "Content-Container(e2,e1)": 18 }


import collections
import re
from typing import List, Set, MutableMapping, cast, Iterator

from icontract import require, ensure, DBC

# crosshair: on
from python_by_contract_corpus.common import Lines

class Point(DBC):
    """Represent a cube in 3D."""

    def __new__(cls, x: int, y: int, z: int) -> "Point":
        """Create the point as tuple with the given coordinates."""
        return cast(Point, (x, y, z))

    def __getitem__(self, index: int) -> int:
        """Get the item at the given integer index."""
        raise NotImplementedError("Only for type annotations")

class Activity(DBC):
    """Represent the current active cubes in the energy source"""

    def __new__(cls, active_cubes: Set[Point]) -> "Activity":
        """Create an activity as a set of active cubes."""
        return cast(Activity, active_cubes)

    def __iter__(self) -> Iterator[Point]:
        """Iterate over the lines."""
        raise NotImplementedError("Only for type annotations")

    def __contains__(self, item: Point) -> bool:
        """Return True if the cube is activated."""
        raise NotImplementedError("Only for type annotations")

    def __len__(self) -> int:
        """Return the number of active cubes."""
        raise NotImplementedError("Only for type annotations")

def are_neighbours(point: Point, another: Point) -> bool:
    """Check whether the ``point`` and ``another`` are adjacent cubes."""
    return (
        abs(point[0] - another[0]) <= 1
        and abs(point[1] - another[1]) <= 1
        and abs(point[2] - another[2]) <= 1
    )

@ensure(lambda result: len(result) == 26)
@ensure(lambda point, result: point not in result)
# fmt: off
@ensure(
    lambda point, result:
    all(
        are_neighbours(point=point, another=neighbour)
        for neighbour in result
    )
)
# fmt: on
def list_neighbourhood(point: Point) -> List[Point]:
    """List all neighbouring cubes w.r.t. the ``point`` cube."""
    result = []  # type: List[Point]
    for x in [point[0] - 1, point[0], point[0] + 1]:
        for y in [point[1] - 1, point[1], point[1] + 1]:
            for z in [point[2] - 1, point[2], point[2] + 1]:
                neighbour = Point(x=x, y=y, z=z)
                if neighbour != point:
                    result.append(neighbour)

    return result

def apply(activity: Activity) -> Activity:
    """
    Perform a single cycle of the initialization starting from ``activity``.

    :return: the new state of the energy source
    """
    # Count the number of active neighbours
    votes = collections.defaultdict(lambda: 0)  # type: MutableMapping[Point, int]

    for point in activity:
        for neighbour in list_neighbourhood(point=point):
            votes[neighbour] += 1

    next_activity = set()  # type: Set[Point]
    for point, active_neighbours in votes.items():
        is_active = point in activity

        if is_active:
            if active_neighbours in (2, 3):
                next_activity.add(point)
            else:
                pass
        else:
            if active_neighbours == 3:
                next_activity.add(point)

    return Activity(next_activity)

# fmt: off
@require(
    lambda lines:
    len(lines) >= 1
    and all(
        len(line) == len(lines[0])
        for line in lines
    )
)
# fmt: on
@require(lambda lines: all(re.match(r'^[.#]+\Z', line) for line in lines))
@ensure(lambda lines, result: sum(line.count('#') for line in lines) == len(result))
def parse_initial(lines: Lines) -> Activity:
    """Parse ``lines`` into the state of the energy source."""
    result = set()  # type: Set[Point]

    z = 0
    for y, line in enumerate(lines):
        for x, symbol in enumerate(line):
            if symbol == '#':
                result.add(Point(x, y, z))

    return Activity(result)

def repr_activity(activity: Activity) -> str:
    """Represent the activity as a string for easier testing/debugging."""
    z_set = {point[2] for point in activity}

    xmin = min(point[0] for point in activity)
    xmax = max(point[0] for point in activity)
    ymin = min(point[1] for point in activity)
    ymax = max(point[1] for point in activity)

    z_blocks = []  # type: List[str]

    for z in sorted(z_set):
        block = [f'z={z}']
        for y in range(ymin, ymax + 1):
            line = []  # type: List[str]
            for x in range(xmin, xmax + 1):
                point = Point(x, y, z)
                if point in activity:
                    line.append('#')
                else:
                    line.append('.')

            block.append(''.join(line))

        z_blocks.append('\n'.join(block))

    return '\n\n'.join(z_blocks)

@ensure(lambda result: result >= 0)
def count_active(activity: Activity) -> int:
    """Count number of active cells in the ``activity``."""
    return len(activity)


from typing import List
from data_models.data_row import DataRow
from utils import get_logger

class ResultsInfo:
    def __init__(self, accuracy, recall, precision, false_positive_rate):
        self.accuracy = accuracy
        self.recall = recall
        self.precision = precision
        self.false_positive_rate = false_positive_rate

    def __str__(self):
        return f"Accuracy : {(self.accuracy * 100):.2f}%\n" \
            f"Recall : {self.recall * 100:.2f}% -> low = not enough anomalies\n" \
            f"Precision : {self.precision * 100:.2f}% -> how much detected_anomalies are really anomalies\n" \
            f"False Positive Rate : {self.false_positive_rate * 100:.2f}%"

class ResultsAnalyzer:
    def __init__(self):
        self._log = get_logger(self.__class__.__name__)

    def analyze_all(self, results: dict):
        results_info_lst = []
        for file_name, findings in results.items():
            file_content = findings[0]
            anomalies = findings[1]
            self._log.info(f"Analyzing {file_name}")
            results_info_lst.append(self.analyze_single_file(file_content, anomalies))
        return results_info_lst

    def analyze_single_file(self, file_content, anomalies):
        self._log.info(f"Analyzing results...")

        anomalies_found = self.get_results_anomaly_points(anomalies)
        anomalies_in_data = self.get_anomalies_in_data(file_content)
        data_set = set(i for i in range(len(file_content)))
        non_anomalies_int_data = data_set - anomalies_in_data

        true_positive = anomalies_found & anomalies_in_data
        false_positive = anomalies_found - anomalies_in_data

        true_negative = non_anomalies_int_data - anomalies_found
        false_negative = anomalies_in_data - anomalies_found
        self._log.info(f"Raw data:")
        self._log.info(f"Total anomalies found : {len(anomalies_found)}")
        self._log.info(f"Total anomalies in data : {len(anomalies_in_data)}")

        self._log.info(f"True Positives total : {len(true_positive)}")
        self._log.info(f"False Positives total : {len(false_positive)}")

        self._log.info(f"True Negatives total : {len(true_negative)}")
        self._log.info(f"False Negatives total : {len(false_negative)}")
        self._log.info(f"")

        accuracy = (len(true_positive) + len(true_negative)) / (
                (len(true_positive) + len(false_negative)) + (len(true_negative) + len(false_positive)))
        recall = len(true_positive) / (len(true_positive) + len(false_negative)) if (len(true_positive) + len(false_negative))!=0 else 0
        precision = len(true_positive) / (len(true_positive) + len(false_positive)) if (len(true_positive) + len(false_positive))!=0 else 0
        false_positive_rate = len(false_positive) / (len(false_positive) + len(true_negative)) if (len(false_positive) + len(true_negative))!=0 else 0

        self._log.info(f"Analyzed results:")
        self._log.info(f"Accuracy : {(accuracy * 100):.2f}%")
        self._log.info(f"Recall : {recall * 100:.2f}% -> low = not enough anomalies")
        self._log.info(f"Precision : {precision * 100:.2f}% -> how much detected_anomalies are really anomalies")
        self._log.info(f"False Positive Rate : {false_positive_rate * 100:.2f}%")
        return ResultsInfo(
            accuracy=accuracy,
            recall=recall,
            precision=precision,
            false_positive_rate=false_positive_rate
        )

    def get_results_anomaly_points(self, anomalies):
        _anomalies = set()
        for k, v in anomalies.items():
            _anomalies.add(k)
        return _anomalies

    def get_anomalies_in_data(self, file_content):
        data_anomalies = set()
        for i, row in enumerate(file_content):
            if "false" in row.command_fault:
                data_anomalies.add(i)
        return data_anomalies

    @staticmethod
    def get_avg_results_info(results_info_lst: List[ResultsInfo]):
        avg_accuracy = 0
        avg_recall = 0
        avg_precision = 0
        avg_false_positive_rate = 0
        for result_info in results_info_lst:
            avg_accuracy += result_info.accuracy
            avg_recall += result_info.recall
            avg_precision += result_info.precision
            avg_false_positive_rate += result_info.false_positive_rate
        avg_accuracy /= len(results_info_lst)
        avg_recall /= len(results_info_lst)
        avg_precision /= len(results_info_lst)
        avg_false_positive_rate /= len(results_info_lst)
        return ResultsInfo(
            accuracy=avg_accuracy,
            recall=avg_recall,
            precision=avg_precision,
            false_positive_rate=avg_false_positive_rate
        )

    def _test(self, results):
        total_data = 0
        total_anomalies = 0

        for file_name, findings in results.items():
            file_content = findings[0]
            total_data+= len(file_content)
            anomalies_in_data = self.get_anomalies_in_data(file_content)
            total_anomalies += len(anomalies_in_data)
        print("TOTALS")
        print(total_data)
        print(total_anomalies)


import json

from note import Note

class Notebook:

    def load_from_file(self, file):
        with open(file, 'r') as json_file:
            notes_to_load = json.load(json_file)
        for id in notes_to_load:
            json_note = notes_to_load[id]
            note = Note(json_note['text'], json_note['title'], json_note['group'], json_note['tags'])
            note.date = json_note['date']
            note.update_date = json_note['update']
            note.id = json_note['id']
            self.notes[id] = note

    def save_to_file(self, file):
        notes_to_save = {}
        for note in self.notes:
            notes_to_save[note] = self.notes[note].save()
        with open(file, 'w') as json_file:
            json.dump(notes_to_save, json_file)

    def add_note(self, note):
        self.notes[note.id] = note

    def __iter__(self):
        return (self.notes[note] for note in self.notes.__iter__())

    def __init__(self):
        self.notes = {}


import configuracion
import random

class Equipo():
	def __init__(self, pais, ranking, formacion, grupo):
		super(Equipo, self).__init__()
		self.pais=pais
		self.grupo=grupo
		self.formacion=formacion
		self._obtener_probabilidades(ranking)

#Recupera las probabilidades de acertar pase, anotar gol y encajar gol segun su ranking FIFA.
	def _obtener_probabilidades(self,ranking):
		categoria=""
		if ranking<=10:
			categoria="1-10"
		elif ranking<=20:
			categoria="11-20"
		elif ranking<=30:
			categoria="21-30"
		elif ranking<=40:
			categoria="31-40"
		elif ranking<=50:
			categoria="41-50"
		elif ranking<=60:
			categoria="51-60"
		elif ranking<=70:
			categoria="61-70"
		elif ranking<=80:
			categoria="71-80"
		elif ranking<=90:
			categoria="81-90"
		else:
			categoria="91-100"
		self.categoria=categoria
		self.probabilidad=configuracion.TABLA_PROBABILIDADES[categoria]

#Determina si se acierta o falla un pase.
	def _pasar(self):
		p_pase=random.randrange(100)+self.probabilidad["pase"]
		if p_pase>100:
			paso=True
		else:
			paso=False
		return paso

#Determina si se acierta o falla un remate a porteria (intento de gol).
	def _chutar(self, encajar_equipo_contrario):
		p_anotacion=random.randrange(100)+self.probabilidad["anotar"]+ encajar_equipo_contrario
		if p_anotacion>150:
			entro= True
		else:
			entro= False
		return entro

#Secuencia de ataque del equipo. Si se logran 4 pases seguidos, se remata al arco, si no se pierde el balon.
	def atacar(self, encajar_equipo_contrario):
		pases=0
		while pases<4 and self._pasar():
			pases = pases+1
		if pases==4:
			 gol=self._chutar(encajar_equipo_contrario)
		else:
			 gol=False
		return (pases, gol)


def plot_hpd(
    x,
    y,
    credible_interval=0.94,
    color="C1",
    circular=False,
    smooth=True,
    smooth_kwargs=None,
    fill_kwargs=None,
    plot_kwargs=None,
    ax=None,
):
    """
    Plot hpd intervals for regression data.

    Parameters
    ----------
    x : array-like
        Values to plot
    y : array-like
        values from which to compute the hpd
    credible_interval : float, optional
        Credible interval to plot. Defaults to 0.94.
    color : str
        Color used for the limits of the HPD interval and fill. Should be a valid matplotlib color
    circular : bool, optional
        Whether to compute the hpd taking into account `x` is a circular variable
        (in the range [-np.pi, np.pi]) or not. Defaults to False (i.e non-circular variables).
    smooth : boolean
        If True the result will be smoothed by first computing a linear interpolation of the data
        over a regular grid and then applying the Savitzky-Golay filter to the interpolated data.
        Defaults to True.
    smooth_kwargs : dict, optional
        Additional keywords modifying the Savitzky-Golay filter. See Scipy's documentation for
        details
    fill_kwargs : dict
        Keywords passed to `fill_between` (use fill_kwargs={'alpha': 0} to disable fill).
    plot_kwargs : dict
        Keywords passed to HPD limits
    ax : matplotlib axes

    Returns
    -------
    ax : matplotlib axes
    """
    if plot_kwargs is None:
        plot_kwargs = {}
    plot_kwargs.setdefault("color", color)
    plot_kwargs.setdefault("alpha", 0)

    if fill_kwargs is None:
        fill_kwargs = {}
    fill_kwargs.setdefault("color", color)
    fill_kwargs.setdefault("alpha", 0.5)

    if ax is None:
        ax = gca()

    hpd_ = hpd(y, credible_interval=credible_interval, circular=circular)

    if smooth:
        if smooth_kwargs is None:
            smooth_kwargs = {}
        smooth_kwargs.setdefault("window_length", 55)
        smooth_kwargs.setdefault("polyorder", 2)
        x_data = np.linspace(x.min(), x.max(), 200)
        hpd_interp = griddata(x, hpd_, x_data)
        y_data = savgol_filter(hpd_interp, axis=0, **smooth_kwargs)
    else:
        idx = np.argsort(x)
        x_data = x[idx]
        y_data = hpd_[idx]

    ax.plot(x_data, y_data, **plot_kwargs)
    ax.fill_between(x_data, y_data[:, 0], y_data[:, 1], **fill_kwargs)

    return ax


import torch
import torch.nn as nn
import torch.nn.functional as F
from mamujoco_maddpg.common.utils import make_env
from multiagent.multi_discrete import MultiDiscrete
import copy

class Centralized_q(nn.Module):
    def __init__(self, args, task_sampler):
        super(Centralized_q, self).__init__()
        self.args = args
        self.device = args.device
        self.max_action = args.high_action if hasattr(args, "high_action") else 1
        self.input_shape = []
        for s in task_sampler.scenarios_names:
            args = copy.copy(self.args)
            if s == "2_Agent_Ant":
                scenario_name = "Ant-v2"
                args.scenario_name = scenario_name
                args.agent_conf = "2x4"
                args.agent_obsk = 1
                _, args = make_env(args=args)
                input_shape = sum(args.obs_shape) + sum(args.action_shape)
                self.input_shape.append(input_shape)
            elif s == "2_Agent_HalfCheetah":
                scenario_name = "HalfCheetah-v2"
                args.scenario_name = scenario_name
                args.agent_conf = "2x3"
                args.agent_obsk = 1
                _, args = make_env(args=args)
                input_shape = sum(args.obs_shape) + sum(args.action_shape)
                self.input_shape.append(input_shape)
            elif s == "2_Agent_Swimmer":
                scenario_name = "Swimmer-v2"
                args.scenario_name = scenario_name
                args.agent_conf = "2x1"
                args.agent_obsk = 1
                _, args = make_env(args=args)
                input_shape = sum(args.obs_shape) + sum(args.action_shape)
                self.input_shape.append(input_shape)
            elif s == "2_Agent_Humanoid":
                scenario_name = "Humanoid-v2"
                args.scenario_name = scenario_name
                args.agent_conf = "9|8"
                args.agent_obsk = 1
                _, args = make_env(args=args)
                input_shape = sum(args.obs_shape) + sum(args.action_shape)
                self.input_shape.append(input_shape)

        self.input_shape = max(self.input_shape)

        self.fc1 = nn.Linear(self.input_shape, 64)
        self.fc2 = nn.Linear(64, 64)
        self.fc3 = nn.Linear(64, 64)
        self.q_out = nn.Linear(64, 1)

    def forward(self, state, action):
        state = torch.cat(state, dim=1)
        for i in range(len(action)):
            action[i] /= self.max_action
        action = torch.cat(action, dim=1)
        x = torch.cat([state, action], dim=1)
        while True:
            if len(x[0]) == self.input_shape:
                break
            x = torch.cat((x, torch.tensor([[0]]*256).to(self.device)), dim=1)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = F.relu(self.fc3(x))
        q_value = self.q_out(x)
        return q_value


#!/usr/bin/env python2
# -*- coding: utf-8 -*-
# file: client.py

from macroses import BUSNAME, OBJNAME
import dbus
import sys
import readline

bus = dbus.SessionBus()
provider = bus.get_object(BUSNAME, OBJNAME)
tweets = []

def valid_status(status):
    if len(status) > 140:
        print("-- Error: Status length ({0}) is more than 140 symbols.".format(str(len(status))))
        return False
    return True

def updateStatus(status):
    if valid_status(status):
        try:
            provider.updateStatus(status, dbus_interface = BUSNAME)
        except Exception as e:
            print("-- Error:", str(e))

def usage():
    print("Use it like this:")
    print("{0} {1}".format(sys.argv[0], "\"status\""))

def inputAndUpdate():
    status = raw_input("updateStatus> ")
    if status :
        updateStatus(status)

def mainloop():
    while True:
        inputAndUpdate()

if __name__ == "__main__":
    timeline()
    if len(sys.argv) == 1 :
        try:
            mainloop()
        except KeyboardInterrupt:
            sys.exit(0)
    else:
        updateStatus(sys.argv[1])


import apache_beam as beam
from apache_beam.runners.dataflow import DataflowRunner
from CSVToDict import CSVToDictFn
import argparse

def mapToCSVRow(elem: dict) -> str:
    return ",".join(f'"{v}"' for _, v in elem.items())

def run_beam():
    parser = argparse.ArgumentParser()
    parser.add_argument("--input_file", type=str)
    parser.add_argument("--output_file", type=str)
    args, _ = parser.parse_known_args()
    input_file = args.input_file
    output_file = args.output_file
    schema = [
        ("id", int),
        ("first_name", str),
        ("last_name", str),
        ("gender", str),
        ("occupation", str),
    ]
    options = beam.options.pipeline_options.PipelineOptions(
        streaming=False,
        save_main_session=False,
    )
    with beam.Pipeline(
        runner=DataflowRunner(),
        options=options,
    ) as pipe:
        (
            pipe
            | beam.io.ReadFromText(input_file, skip_header_lines=1)
            | beam.ParDo(CSVToDictFn(schema))
            | beam.Filter(lambda l: l["id"] % 2 == 1)
            | beam.Map(mapToCSVRow)
            | beam.io.WriteToText(output_file, shard_name_template="")  # write to file
        )

if __name__ == "__main__":
    run_beam()


from datetime import datetime

import requests
import iota
from loguru import logger

from config import API_URI, API_OPEN

class Iota:
    def __init__(self):
        self.api_uri = API_URI
        self.api_open = API_OPEN
        self.api_entry = self.get_api_entry()

    def get_api_entry(self):
        """return usable api entry"""
        available_nodes = list()
        for node in self.api_uri:
            logger.info(f"[CHECK NODES] Testing {node}")
            try:
                api = iota.Iota(iota.HttpAdapter(node, timeout=5))
                # Check node alive
                node_info = api.get_node_info()
                # Show Node Info
                logger.debug(node_info)
                # Check node milestone is latest
                assert node_info["latestMilestone"] == node_info["latestSolidSubtangleMilestone"]
                logger.success(f"[CHECK NODES] Node is alive. URI: {node}")
                available_nodes.append(node)
            except AssertionError:
                logger.warning(f"[CHECK NODES] Node is not up to date. URI: {node}")
            except requests.exceptions.ConnectionError:
                logger.error(f"[CHECK NODES] Node is down. URI: {node}")
            except requests.exceptions.ReadTimeout:
                logger.error(f"[CHECK NODES] Node timeout. URI: {node}")
        if available_nodes:
            logger.info(f"Using IOTA Node: {available_nodes[0]}")
            api_entry = iota.Iota(available_nodes[0])
        else:
            logger.warning("ALL NODES DOWN\nUsing Open Nodes.")
            api_entry = iota.Iota(self.api_open)
        return api_entry

    @staticmethod
    def get_tag(text):
        """transfer tag to IOTA TAG which accept A-Z and 9

        Arguments:
            text {string} -- type of data

        Returns:
            string -- IOTA TAG
        """
        # add timestamp
        text += f"_{chr(ord('A') + datetime.utcnow().hour)}"
        # tranfer to uppercase and use `9` to replace `_`
        tag = text.upper().replace("_", "9")
        return tag

    def send_to_iota(self, send_data, target_address):
        """transfer data to the Tangle

        Arguments:
            send_data {list} -- five pair of tag and upload data with string type
            target_address {string} -- iota receive address

        Returns:
            iota.transaction.types.BundleHash -- BundleHash of the transaction
            datetime.timedelta -- cost time
        """
        # preparing transactions
        transactions = []
        for tag, data in send_data:
            transactions.append(
                iota.ProposedTransaction(
                    address=iota.Address(target_address),
                    message=iota.TryteString.from_bytes(data),
                    tag=iota.Tag(self.get_tag(tag).encode()),
                    value=0,
                )
            )

        start_time = datetime.now()
        final_bundle = self.api_entry.send_transfer(transfers=transactions)["bundle"]
        end_time = datetime.now()

        return final_bundle.hash, end_time - start_time


import os
import sys
import time

sys.path.append(os.path.join(sys.path[0], '..'))    # noqa: E402

import OTBMGenerator

def main(example):
    otbm_file = None    # Input file
    json_file = None    # Output file
    if example == 1:
        otbm_file = "./otbmparser_files/example1.otbm"
        json_file = "./output/example1_output.json"
    elif example == 2:
        otbm_file = "./otbmparser_files/example2.otbm"
        json_file = "./output/example2_output.json"
    elif example == 3:
        otbm_file = "./otbmparser_files/example3.otbm"
        json_file = "./output/example3_output.json"
    else:
        print("Please provide a valid example value (1, 2 3).")
        return

    start_time = time.process_time()    # Time measurement

    # Create OTBMGenerator instance and set input & output files path
    otbm_generator = OTBMGenerator.OTBMGenerator()
    otbm_generator.otbm2json_parser.otbm_file_path = otbm_file
    otbm_generator.otbm2json_parser.json_file_path = json_file

    # Process OTBM file, parsing its data to json
    otb2json.process_file()

    # Save generated json with map data.
    otb2json.generate_json()

    elapsed = time.process_time() - start_time
    path = os.path.abspath(json_file)
    print(f"Parsing finished in {elapsed} seconds!, saved in {path}")

if __name__ == "__main__":
    main(example=1)    # Valid values: 1, 2, 3


def loselose(s):
    hash = 0
    for ch in s:
        hash = hash + ord(ch)
    return hash

def rs(s):
    hash = 0
    a = 63689
    b = 378551

    for ch in s:
        hash = hash * a + ord(ch)
        a = a*b
    return hash

def djb2(s):
    hash_seed = 5381
    hashCode = hash_seed
    hash_mask = 0x7FFFFFFF
    hash_multiplier = 33
    for ch in s:
        hashCode = hash_multiplier * hashCode + ord(ch)
    return hashCode & hash_mask

def sdbm(s):
    hash = 0
    for ch in s:
        hash = ord(ch) + (hash << 6) + (hash << 16) -hash
    return hash

def main():
    first = loselose('Unicode')
    print('Resultado LoseLose Hash: {}'.format(first))
    print(hex(first))
    second = rs('Unicode')
    print('Resultado RS Hash: {}'.format(second))
    print(hex(second))
    third = djb2('Unicode')
    print('Resultado DJB2 Hash: {}'.format(third))
    print(hex(third))
    fourth = sdbm('Unicode')
    print('Resultado SDBM Hash: {}'.format(fourth))
    print(hex(fourth))

if __name__ == '__main__':
    main()


from time import time
from flask import Response
import flask
import json
import cv2
from PIL import Image
import numpy as np
import time
import os
from skimage import io
from firebase_admin import credentials, initialize_app, storage

app = flask.Flask(__name__)

cred = credentials.Certificate(
    "<firebase credentials>")
initialize_app(
    cred, {'storageBucket': 'edge-detector.appspot.com'})

def get_file_names(filename):
    return "1" + str(int(time.time())) + filename, "2" + str(int(time.time())) + filename

def delete_file(filename):
    os.remove("D:\\Projects\\sample_flask\\" + filename)

def upload_file(bucket, filename, text):
    blob = bucket.blob(filename)
    blob.upload_from_filename(filename)
    blob.make_public()
    print(text, blob.public_url)

def save_image(filename, file_content):
    file = Image.fromarray(file_content)
    file.save(filename)

def upload_images_to_storage(normal_image, edge_detected_image, filename):
    filename1, filename2 = get_file_names(filename)
    save_image(filename1, normal_image)
    save_image(filename2, edge_detected_image)

    bucket = storage.bucket()
    upload_file(bucket, filename1, "Normal image URL - ")
    upload_file(bucket, filename2, "Edge detected image URL - ")
    delete_file(filename1)
    delete_file(filename2)

@app.route('/get_all_images',methods=['GET'])
def fn():
    bucket = storage.bucket()
    blob_iter = bucket.list_blobs(delimiter='/')
    result = list()
    for res in blob_iter:
        if res.name[0] == '1':
            continue
        url = "https://firebasestorage.googleapis.com/v0/b/edge-detector.appspot.com/o/" + res.name + "?alt=media"
        dic = {
            "name" : res.name,
            "url" : url
        }
        result.append(dic)
    return Response(json.dumps(result),  mimetype='application/json')

@app.route('/detect_edges_image_from_phone', methods=['POST'])
def detect_edges_of_image_from_phone():
    image_file = flask.request.files['image']
    print("\nReceived image File name : " + image_file.filename)
    normal_image = cv2.cvtColor(cv2.imdecode(np.frombuffer(
        image_file.read(), np.uint8), cv2.IMREAD_UNCHANGED), cv2.COLOR_BGR2RGB)
    edge_detected_image = cv2.Canny(normal_image, 100, 100)
    upload_images_to_storage(
        normal_image, edge_detected_image, image_file.filename)

    return "Image Uploaded Successfully"

@app.route('/detect_edges_image_from_url', methods=['POST'])
def detect_edges_of_image_from_url():
    url = flask.request.form.get('url')
    print(url)
    filename = url.split('/')[-1]
    normal_image = io.imread(url)
    edge_detected_image = cv2.Canny(normal_image, 100, 100)
    upload_images_to_storage(
        normal_image, edge_detected_image, filename)

    return "Image Uploaded Successfully"

if __name__ == '__main__':
    app.run(host='127.0.0.1', port=5000, debug=True)