#!/usr/bin/env python
import cPickle
import heapq
import os
import struct
import sys

import dictionary_saver
import doc2words
import docreader
import simple9
import varbyte

def build_index_files(paths, max_doc_ids_per_index_file=10 ** 6):
    reader = docreader.DocumentStreamReader(paths)

    doc_urls = dict()

    next_file_id = 0
    last_doc_ids = 0
    file_names = []

    index = dict()
    for doc_id, doc in enumerate(reader):
        doc_urls[doc_id] = doc.url
        words = doc2words.extract_words(doc.text)
        for word in words:
            doc_ids = index.get(word, set())
            index[word] = doc_ids
            if doc_id not in doc_ids:
                last_doc_ids += 1
            doc_ids.add(doc_id)

            if last_doc_ids >= max_doc_ids_per_index_file:
                index_file_name = __get_index_file_name(next_file_id)
                file_names.append(index_file_name)
                dump_index_to_file(index, index_file_name)
                next_file_id += 1
                last_doc_ids = 0
                index.clear()

    if index:
        index_file_name = __get_index_file_name(next_file_id)
        file_names.append(index_file_name)
        dump_index_to_file(index, index_file_name)
        next_file_id += 1
        index.clear()

    return doc_urls, file_names

def dump_index_to_file(index, file_name):
    with open(file_name, 'wb') as f:
        sorted_words = sorted(index.keys())
        for next_word in sorted_words:
            posting_list = sorted(index[next_word])
            append_list_to_file(f, next_word, posting_list)

def append_list_to_file(file, word, ints_or_bytearray):
    encoded_word = word.encode('utf-8')
    result = bytearray()
    int_encoder = struct.Struct('I')
    result.extend(int_encoder.pack(len(encoded_word)))
    result.extend(encoded_word)

    if isinstance(ints_or_bytearray, bytearray):
        result.extend(int_encoder.pack(len(ints_or_bytearray)))
        result.extend(ints_or_bytearray)
    else:
        bytes_ = bytearray()
        for a in ints_or_bytearray:
            bytes_.extend(int_encoder.pack(a))
        result.extend(int_encoder.pack(len(bytes_)))
        result.extend(bytes_)
    file.write(result)

def read_next(file, as_ints_array=False):
    word_len = file.read(4)
    if word_len == '':
        return None
    struct_decoder = struct.Struct('I')
    word_len = struct_decoder.unpack(word_len)[0]
    word = file.read(word_len)
    word = word.decode('utf-8')
    size = file.read(4)
    if size == '':
        raise RuntimeError('The word is present, but the records are empty')
    size = struct_decoder.unpack(size)[0]

    if as_ints_array:
        all_bytes = file.read(size)
        result = []
        for i in range(0, size, 4):
            result.append(struct_decoder.unpack(all_bytes[i:i + 4])[0])
        return word, result
    else:
        byte_array = bytearray(file.read(size))
        return word, byte_array

def merge_two_lists(first_list, second_list):
    if not first_list:
        return second_list
    if not second_list:
        return first_list
    n = len(first_list)
    m = len(second_list)
    i = 0
    j = 0
    result = []
    while i < n and j < m:
        if first_list[i] < second_list[j]:
            result.append(first_list[i])
            i += 1
        else:
            result.append(second_list[j])
            j += 1
    while i < n:
        result.append(first_list[i])
        i += 1
    while j < m:
        result.append(second_list[j])
        j += 1
    return result

def merge_index_files(file_names):
    opened_files = []
    next_word_blocks = dict()

    min_words_queue = []
    min_words_set = set()
    for i, file_name in enumerate(file_names):
        f = open(file_name, 'rb')
        opened_files.append(f)
        block = read_next(f, True)
        if block is not None:
            word = block[0]
            if word not in min_words_set:
                heapq.heappush(min_words_queue, word)
                min_words_set.add(word)
            if word not in next_word_blocks:
                next_word_blocks[word] = []
            next_word_blocks[word].append((i, block))

    dictionary = dict()
    try:
        with open('indices/main.idx', 'wb') as out:
            while True:
                if not any(min_words_queue): break
                next_min_word = heapq.heappop(min_words_queue)
                min_words_set.remove(next_min_word)

                if next_min_word in next_word_blocks:
                    blocks = next_word_blocks.pop(next_min_word)
                    if blocks:
                        resulting_list = None
                        for (i, block) in blocks:
                            block_list = block[1]
                            if resulting_list is None:
                                resulting_list = block_list
                            else:
                                resulting_list = merge_two_lists(resulting_list, block_list)

                            next_i_block = read_next(opened_files[i], True)
                            if next_i_block is not None:
                                next_word, next_list = next_i_block
                                if next_word not in next_word_blocks:
                                    next_word_blocks[next_word] = []
                                next_word_blocks[next_word].append((i, next_i_block))

                                if next_word not in min_words_set:
                                    heapq.heappush(min_words_queue, next_word)
                                    min_words_set.add(next_word)

                        assert (resulting_list is not None)
                        diffs = transform_posting_list_to_diffs(resulting_list)
                        encoded_diffs = list_encoder(diffs)
                        dictionary[next_min_word] = out.tell()
                        append_list_to_file(out, next_min_word, encoded_diffs)
        return dictionary
    finally:
        for f in opened_files:
            f.close()

def transform_posting_list_to_diffs(posting_list):
    if not posting_list:
        return []
    last_num = posting_list[0]
    for i in range(1, len(posting_list)):
        delta = posting_list[i] - last_num
        last_num = posting_list[i]
        posting_list[i] = delta
    return posting_list

def detransform_posting_list_from_diffs(compressed_list):
    if not compressed_list:
        return []
    result = list(compressed_list)
    for i in range(1, len(result)):
        result[i] += result[i - 1]
    return result

indices_base_dir = 'indices'

def save_encoding_type(encoding_type):
    with open(__get_encoding_type_file(), 'w') as f:
        f.write(encoding_type)

def read_encoding_type():
    with open(__get_encoding_type_file(), 'r') as f:
        return f.read()

def __get_encoding_type_file():
    return indices_base_dir + '/encoding_type.txt'

def __get_doc_urls_file():
    return indices_base_dir + '/doc_urls.idx'

def __get_index_file_name(id):
    return indices_base_dir + '/index' + str(id) + '.idx'

def save_doc_urls(doc_urls):
    with open(__get_doc_urls_file(), 'wb') as f:
        cPickle.dump(doc_urls, f, protocol=cPickle.HIGHEST_PROTOCOL)

def read_doc_urls():
    with open(__get_doc_urls_file(), 'rb') as f:
        return cPickle.load(f)

def save_bucket_offsets(offsets):
    with open('indices/bucket_offsets.idx', 'wb') as f:
        cPickle.dump(offsets, f, protocol=cPickle.HIGHEST_PROTOCOL)

def read_bucket_offsets():
    with open('indices/bucket_offsets.idx', 'rb') as f:
        return cPickle.load(f)

def save_buckets_cnt(buckets_cnt):
    with open('indices/buckets_cnt.txt', 'w') as f:
        f.write(str(buckets_cnt))

def read_buckets_cnt():
    with open('indices/buckets_cnt.txt', 'r') as f:
        return int(f.read())

def remove_files(files_names):
    for name in files_names:
        os.remove(name)

if __name__ == '__main__':
    if len(sys.argv) < 3:
        raise RuntimeError('Usage: ./indexer.py (varbyte|simple9) path_to_gzip')
    encoding_type = sys.argv[1]
    if encoding_type == 'varbyte':
        list_encoder = varbyte.varbyte_encode
    elif encoding_type == 'simple9':
        list_encoder = simple9.simple9_encode
    else:
        raise RuntimeError('Unknown encoding type: %s' % encoding_type)

    if not os.path.exists(indices_base_dir):
        os.mkdir(indices_base_dir)
    save_encoding_type(encoding_type)
    doc_urls, files_names = build_index_files(sys.argv[2:])
    save_doc_urls(doc_urls)

    dictionary = merge_index_files(files_names)
    remove_files(files_names)

    buckets_cnt, bucket_offsets = dictionary_saver.save_dictionary('indices/dict.idx', dictionary)
    save_buckets_cnt(buckets_cnt)
    save_bucket_offsets(bucket_offsets)


# Implementar la funcion agregar_peso, que inserte un registro en la tabla PersonaPeso.
# Debe validar:
# - que el ID de la persona ingresada existe (reutilizando las funciones ya implementadas).
# - que no existe de esa persona un registro de fecha posterior al que queremos ingresar.

# Debe devolver:
# - ID del peso registrado.
# - False en caso de no cumplir con alguna validacion.

import datetime

from sqlalchemy import create_engine, func
from sqlalchemy.orm import sessionmaker
from sqlalchemy.sql import select

from practico_03.ejercicio_02 import agregar_persona
from practico_03.ejercicio_06 import reset_tabla, PersonaPeso
from practico_03.ejercicio_04 import buscar_persona

engine = create_engine('sqlite:///sqlalchemy_ejemplo.db', echo=False) #cambiar a True para ver el log

def agregar_peso(id_persona, fecha, peso):

    rta = False
    Session = sessionmaker(bind=engine)

    # create a Session
    session = Session()

    if buscar_persona(id_persona) != False:
        try:
            personasPesos = session.query(PersonaPeso).filter(PersonaPeso.IdPersona == id_persona, PersonaPeso.Fecha > fecha).all()
            if personasPesos == []:
                # work with sess
                personaPeso =  PersonaPeso(IdPersona=id_persona, Fecha=fecha,Peso=peso)
                session.add(personaPeso)
                session.commit()
                session.refresh(personaPeso)
                return personaPeso.Id
            else:
                return False

        except:
            session.rollback()
            rta = False
        finally:
            session.close()

    else:
        return False

@reset_tabla
def pruebas():
    id_juan = agregar_persona('juan perez', datetime.datetime(1988, 5, 15), 32165498, 180)
    assert agregar_peso(id_juan, datetime.datetime(2018, 5, 26), 80) > 0
    # id incorrecto
    assert agregar_peso(200, datetime.datetime(1988, 5, 15), 80) == False
    # registro previo al 2018-05-26
    assert agregar_peso(id_juan, datetime.datetime(2018, 5, 16), 80) == False

if __name__ == '__main__':
    pruebas()


class Reflector(object):

	def __init__(self, reflector_type, reflector_characters):

		self._reflector_type = reflector_type
		self._reflector_characters = self.validate_characters(reflector_characters)
		super(Reflector, self).__init__()

	def format_character(self, character):

		if character.isalpha():
			return character.upper()
		else:
			raise ValueError("invalid input. must be a character")

	def validate_characters(self, reflector_characters):
		valid_characters = []
		for character in reflector_characters:
			valid_characters.append(self.format_character(character))
		unique = set(valid_characters)
		if len(unique) == 26:
			return valid_characters
		else:
			raise ValueError(valid_characters, " not a valid reflector list")

	def reflector_output(self, index):

		return self._reflector_characters.index(chr(index + 65))

	def reflector_type(self):

		return self._reflector_type

	def reflector_list(self):

		return self._reflector_list

	def get_reflector_dict(self):

		reflector_dict = {}
		reflector_dict["REFLECTOR_CHARACTERS"] = self._reflector_characters.copy()
		reflector_dict["REFLECTOR_TYPE"] = self._reflector_type
		return reflector_dict

class UKWD_reflector(Reflector):

	def __init__(self, reflector_characters):
		super().__init__("UKWD", reflector_characters)

	def change_reflector_characters(self, new_characters):

		self._reflector_characters = self.validate_characters(new_characters)

if __name__ == "__main__":

	letters = [chr(i) for i in range(65, 91)]
	letters2 = letters[::-1]
	ref_type = "REFLECTOR"
	reflector = Reflector(ref_type, letters)
	for i in range(26):
		print(reflector.reflector_output(i))
	print(reflector.reflector_type())

	reflector2 = UKWD_reflector(letters2)
	for i in range(26):
		print(reflector2.reflector_output(i))
	print(reflector2.reflector_type())


# Script to plot longitude/latitude points on a shapefile
import pandas as pd
import matplotlib.pyplot as plt
import descartes
import geopandas as gpd
from shapely.geometry import Point, Polygon

map_file = '/home/jon/PycharmProjects/jon-insight-project/data/external/PA/tl_2017_42_place.shp'

file_name = '/home/jon/PycharmProjects/jon-insight-project/data/processed/pa_course_database_processed.plk'
df = pd.read_pickle(file_name)

crs = {'init': 'eosg:4326'}

street_map = gpd.read_file(map_file)

fig, ax = plt.subplots(figsize = (15, 15))
street_map.plot(ax = ax)

plt.show()

geometry = [Point(xy) for xy in zip(df['longitude'], df['latitude'])]

geo_df = gpd.GeoDataFrame(df, crs = crs, geometry= geometry)

geo_df.head()

fig, ax = plt.subplots(figsize = (15, 15))
street_map.plot(ax = ax, alpha = 0.4, color = 'grey')
geo_df.plot(ax=ax, markersize = 20, color = 'blue', marker = 'o')
ax.set_xlabel('Longitude')
ax.set_ylabel('Latitude')
ax.set_title('Disc golf courses in Pennsylvania')
plt.show()


import brian2 as b2
from neurodynex.adex_model import AdEx
b2.defaultclock.dt = 1 * b2.ms

from neurodynex.tools import plot_tools, input_factory
import io
import math
import pdb
from numba import jit
import numpy as np
from .base import *
import quantities as qt
from quantities import mV, ms, s, us, ns, V
import matplotlib as mpl
SLOW_ZOOM = True
from neuronunit.capabilities import spike_functions as sf
mpl.use('Agg')
import matplotlib.pyplot as plt

getting_started = False
try:
    import asciiplotlib as apl
    fig = apl.figure()
    fig.plot([1,0], [0,1])
    ascii_plot = True
    import gc

except:
    ascii_plot = False
import numpy
try:
    brian2.clear_cache('cython')
except:
    pass
    #I_stim = stim, simulation_time=st)

from scipy.interpolate import interp1d

# This function implement Adaptive Exponential Leaky Integrate-And-Fire neuron model
def simulate_AdEx_neuron_local(
        tau_m=None,
        R=None,
        v_rest=None,#
        v_reset=None,#V_RESET,
        v_rheobase=None,#RHEOBASE_THRESHOLD_v_rh,
        a=None,#ADAPTATION_VOLTAGE_COUPLING_a,
        b=None,#SPIKE_TRIGGERED_ADAPTATION_INCREMENT_b,
        v_spike=None,#FIRING_THRESHOLD_v_spike,
        delta_T=None,#SHARPNESS_delta_T,
        tau_w=None,#ADAPTATION_TIME_CONSTANT_tau_w,
        I_stim=None,#input_factory.get_zero_current(),
        simulation_time=None):
    r"""
    code is from:
    /neurodynex/adex_model/AdEx.py

    Implementation of the AdEx model with a single adaptation variable w.

    The Brian2 model equations are:

    .. math::

        \frac{dv}{dt} = (-(v-v_rest) +delta_T*exp((v-v_rheobase)/delta_T)+ R * I_stim(t,i) - R * w)/(tau_m) : volt \\
        \frac{dw}{dt} = (a*(v-v_rest)-w)/tau_w : amp

    Args:
        tau_m (Quantity): membrane time scale
        R (Quantity): membrane restistance
        v_rest (Quantity): resting potential
        v_reset (Quantity): reset potential
        v_rheobase (Quantity): rheobase threshold
        a (Quantity): Adaptation-Voltage coupling
        b (Quantity): Spike-triggered adaptation current (=increment of w after each spike)
        v_spike (Quantity): voltage threshold for the spike condition
        delta_T (Quantity): Sharpness of the exponential term
        tau_w (Quantity): Adaptation time constant
        I_stim (TimedArray): Input current
        simulation_time (Quantity): Duration for which the model is simulated

    Returns:
        (state_monitor, spike_monitor):
        A b2.StateMonitor for the variables "v" and "w" and a b2.SpikeMonitor
    """
    v_spike_str = "v>{:f}*mvolt".format(v_spike / b2.mvolt)

    # EXP-IF
    eqs = """
        dv/dt = (-(v-v_rest) +delta_T*exp((v-v_rheobase)/delta_T)+ R * I_stim(t,i) - R * w)/(tau_m) : volt
        dw/dt=(a*(v-v_rest)-w)/tau_w : amp
        """

    neuron = b2.NeuronGroup(1, model=eqs, threshold=v_spike_str, reset="v=v_reset;w+=b", method="euler")

    # initial values of v and w is set here:
    neuron.v = v_rest
    neuron.w = 0.0 * b2.pA

    # Monitoring membrane voltage (v) and w
    state_monitor = b2.StateMonitor(neuron, ["v", "w"], record=True)
    spike_monitor = b2.SpikeMonitor(neuron)

    # running simulation
    b2.run(simulation_time)
    return state_monitor, spike_monitor

getting_started = False
class ADEXPBackend(Backend):

    name = 'adexp'

    def get_spike_count(self):
        return int(self.spike_monitor.count[0])
    def init_backend(self, attrs=None, cell_name='thembi',
                     current_src_name='spanner', DTC=None,
                     debug = False):
        super(ADEXPBackend,self).init_backend()
        self.debug = None
        self.model._backend.use_memory_cache = False
        self.current_src_name = current_src_name
        self.cell_name = cell_name
        self.vM = None
        self.attrs = attrs
        self.debug = debug
        self.temp_attrs = None
        self.n_spikes = None
        self.spike_monitor = None
        self.peak_v = 0.02
        self.verbose = False
        self.model.get_spike_count = self.get_spike_count

        if type(attrs) is not type(None):
            self.set_attrs(**attrs)
            self.sim_attrs = attrs

        if type(DTC) is not type(None):
            if type(DTC.attrs) is not type(None):
                self.set_attrs(**DTC.attrs)
            if hasattr(DTC,'current_src_name'):
                self._current_src_name = DTC.current_src_name
            if hasattr(DTC,'cell_name'):
                self.cell_name = DTC.cell_name

    def set_stop_time(self, stop_time = 650*pq.ms):
        """Sets the simulation duration
        stopTimeMs: duration in milliseconds
        """
        self.tstop = float(stop_time.rescale(pq.ms))

    def get_membrane_potential(self):
        """Must return a neo.core.AnalogSignal.
        And must destroy the hoc vectors that comprise it.
        """
        return self.vM

    def set_attrs(self, **attrs):

        self.AdEx = None
        self.AdEx = AdEx

        if len(attrs):
            self.AdEx.ADAPTATION_TIME_CONSTANT_tau_w = attrs['ADAPTATION_TIME_CONSTANT_tau_w']*AdEx.b2.units.ms
            self.AdEx.ADAPTATION_VOLTAGE_COUPLING_a = attrs['ADAPTATION_VOLTAGE_COUPLING_a']*AdEx.b2.units.nS
            self.AdEx.FIRING_THRESHOLD_v_spike = attrs['FIRING_THRESHOLD_v_spike']*AdEx.b2.units.mV*100
            self.AdEx.MEMBRANE_RESISTANCE_R = attrs['MEMBRANE_RESISTANCE_R']*AdEx.b2.units.Gohm
            self.AdEx.MEMBRANE_TIME_SCALE_tau_m = attrs['MEMBRANE_TIME_SCALE_tau_m']*AdEx.b2.units.ms
            self.AdEx.RHEOBASE_THRESHOLD_v_rh = attrs['RHEOBASE_THRESHOLD_v_rh']*AdEx.b2.units.mV*100
            self.AdEx.SHARPNESS_delta_T = attrs['SHARPNESS_delta_T']*AdEx.b2.units.mV*100
            self.AdEx.SPIKE_TRIGGERED_ADAPTATION_INCREMENT_b = attrs['SPIKE_TRIGGERED_ADAPTATION_INCREMENT_b']*AdEx.b2.units.pA
            self.AdEx.V_RESET = attrs['V_RESET']*AdEx.b2.units.mV
            self.AdEx.V_REST = attrs['V_REST']*AdEx.b2.units.mV*100

            if str('peak_v') in attrs:
                self.peak_v = attrs['peak_v']

            self.model.attrs.update(attrs)
        if attrs is None:
            #from neurodynex.adex_model import AdEx
            b2.defaultclock.dt = 1 * b2.ms

            self.AdEx =AdEx
    def finalize(self):
        '''
        Necessary for imputing missing sampling, simulating at high sample frequency is prohibitevely slow, with
        out significant difference in behavior.
        '''
        transform_function = interp1d([float(t) for t in self.vM.times],[float(v) for v in self.vM.magnitude])
        xnew = np.linspace(0, float(np.max(self.vM.times)), num=1004001, endpoint=True)
        vm_new = transform_function(xnew) #% generate the y values for all x values in xnew
        self.vM = AnalogSignal(vm_new,units = V,sampling_period = float(xnew[1]-xnew[0]) * pq.s)
        if self.verbose:

            print(len(self.vM))
        self.vM = AnalogSignal(vm_new,units = V,sampling_period = float(xnew[1]-xnew[0]) * pq.s)
        return self.vM
    def inject_square_current(self, current):#, section = None, debug=False):
        """Inputs: current : a dictionary with exactly three items, whose keys are: 'amplitude', 'delay', 'duration'
        Example: current = {'amplitude':float*pq.pA, 'delay':float*pq.ms, 'duration':float*pq.ms}}
        where \'pq\' is a physical unit representation, implemented by casting float values to the quanitities \'type\'.
        Description: A parameterized means of applying current injection into defined
        Currently only single section neuronal models are supported, the neurite section is understood to be simply the soma.

        """
        b2.defaultclock.dt = 1 * b2.ms
        self.state_monitor = None
        self.spike_monitor = None
        self.AdEx = None
        self.AdEx = AdEx
        attrs = copy.copy(self.model.attrs)
        if self.model.attrs is None or not len(self.model.attrs):
            self.AdEx = AdEx
        else:
            self.set_attrs(**attrs)
        if 'injected_square_current' in current.keys():
            c = current['injected_square_current']
        else:
            c = current

        #amplitude = c['amplitude'].simplified

        duration = int(c['duration'])#/dt#/dt.rescale('ms')
        delay = int(c['delay'])#/dt#.rescale('ms')
        pre_current = int(duration)+100
        amp = c['amplitude']#.rescale('uA')
        amplitude = amp.simplified
        if getting_started == False:
            stim = input_factory.get_step_current(delay, delay+duration, b2.ms, 1000.0*amplitude*b2.A)
            st = (duration+delay+200)* b2.ms
        else:
            stim = input_factory.get_step_current(10, 45, b2.ms, 7.2 * b2.uA)
            st = 70 * b2.ms

        if self.model.attrs is None or not len(self.model.attrs):
            #from neurodynex.adex_model import AdEx
            b2.defaultclock.dt = 1 * b2.ms

            self.AdEx = AdEx
            self.state_monitor, self.spike_monitor = simulate_AdEx_neuron_local(I_stim = stim, simulation_time=st)

        else:
            if self.verbose:
                #print(attrs)
                print(attrs['ADAPTATION_TIME_CONSTANT_tau_w'])

            if getting_started == True:

                stim = input_factory.get_step_current(10, 200, 1. * b2.ms, 65.0 * b2.pA)
                st = 300 * b2.ms
            self.set_attrs(**attrs)
            self.state_monitor, self.spike_monitor = self.AdEx.simulate_AdEx_neuron(
            tau_m = attrs['MEMBRANE_TIME_SCALE_tau_m']*AdEx.b2.units.ms,
            R = np.abs(attrs['MEMBRANE_RESISTANCE_R'])*AdEx.b2.units.Gohm,
            v_rest = attrs['V_REST']*AdEx.b2.units.V,
            v_reset = attrs['V_RESET']*AdEx.b2.units.V,
            v_rheobase = attrs['RHEOBASE_THRESHOLD_v_rh']*AdEx.b2.units.V,
            a = attrs['ADAPTATION_VOLTAGE_COUPLING_a']*AdEx.b2.units.nS,
            b =  attrs['b']*b2.pA,
            v_spike=attrs['FIRING_THRESHOLD_v_spike']*AdEx.b2.units.V,
            delta_T = attrs['SHARPNESS_delta_T']*AdEx.b2.units.V,
            tau_w = attrs['ADAPTATION_TIME_CONSTANT_tau_w']*AdEx.b2.units.ms,
            I_stim = stim, simulation_time=st)

        #self.state_monitor.clock.dt = 1 *b2.ms
        self.dt = self.state_monitor.clock.dt

        state_dic = self.state_monitor.get_states()
        vm = state_dic['v']
        vm = [ float(i) for i in vm ]

        self.vM = AnalogSignal(vm,units = mV,sampling_period = float(0.0001) * pq.s)

        tdic = self.spike_monitor.spike_trains()
        if str('peak_v') in attrs.keys():
            self.peak_v = attrs['peak_v']
        else:
            self.peak_v = 2
        if self.verbose:

            print(self.peak_v)
        for key,value in tdic.items():

            if len(value)==1:
                i = int(float(value)/0.001)
                self.vM[i] = self.peak_v*qt.mV
            else:
                for v in value:
                    i = int(float(v)/0.001)
                    self.vM[i] = self.peak_v*qt.mV

        self.n_spikes = int(self.spike_monitor.count[0])
        self.attrs = attrs

        if ascii_plot:
            if SLOW_ZOOM and self.get_spike_count()>=1 :
                from neuronunit.capabilities.spike_functions import get_spike_waveforms
                vm = get_spike_waveforms(self.vM)
            else:
                vm = self.vM
            try:
                t = [float(f) for f in vm.times]
                v = [float(f) for f in vm.magnitude]
                fig = apl.figure()
                fig.plot(t, v, label=str('brain adexp: ')+str(vm.units), width=100, height=20)
                fig.show()
                gc.collect()
                fig = None
            except:
                print('strange error')
        return self.vM

    def _backend_run(self):
        results = None
        results = {}

        results['vm'] = self.vM

        results['t'] = self.vM.times
        results['run_number'] = results.get('run_number',0) + 1
        return results


from ...tools.Logger import Logger
from bitarray import bitarray
import numpy as np
from ..PointDescriptor.PointDescriptor import initiatePointDescriptor
import copy

class PointMatcher_QuantizedPosition:

  @staticmethod
  def findMatches(previous_matches, previous_points, current_points, volume:np.ndarray, config, pointSelector=None, pointDescriptor=None, previous_volume=None):
    pointDescriptor = initiatePointDescriptor(config)
    current_points = copy.deepcopy(previous_points)
    # calculate descripiton for force calculation
    current_points = pointDescriptor.getDescriptionForPoints(volume, current_points)
    for previous_match in previous_matches:

  	  # get previous point
      previous_point = previous_points[previous_match[len(previous_match)-1]["id"]]
      # check coordinates in current volume
      current_point = current_points[previous_match[len(previous_match)-1]["id"]]

      # initialize output
      best_match = {
        "id": previous_match[len(previous_match)-1]["id"],
        "distance_to_previous":0
      }
      force = np.array([0,0,0])

      max_index = len(current_point["description"])
      for index, current_description in enumerate(current_point["description"]):
        previous_descripiton = previous_point["description"][index]
        # increase quantization, until color of the point is not the same anymore
        if current_description["color"] != previous_descripiton["color"]:
          # save quantization depth as matching precision
          best_match["distance_to_previous"] = 1/current_description["quantization"]
          max_index = current_description["quantization"] - 1
          break

        # calculate force for point movement
        force[0] += ((previous_descripiton["x"]["high"] - current_description["x"]["high"]) + (previous_descripiton["x"]["low"] - current_description["x"]["low"])) / current_description["quantization"]
        force[1] += ((previous_descripiton["y"]["high"] - current_description["y"]["high"]) + (previous_descripiton["y"]["low"] - current_description["y"]["low"])) / current_description["quantization"]
        force[2] += ((previous_descripiton["z"]["high"] - current_description["z"]["high"]) + (previous_descripiton["z"]["low"] - current_description["z"]["low"])) / current_description["quantization"]

      # normalize the force with ln approximation for harmonic row
      # -1 because first element of harmonic row is not added up
      # divide by 2 because force from high and low
      force = (force / 2) / (np.log(max_index + 1) + np.euler_gamma - 1)

      force = force * config["impuls_attenuation"]

      if config["momentum"]:
        if "momentum" in previous_point:
          force = np.add(previous_point["momentum"] * 0.5, force * 0.5)
        current_point["momentum"] = force

      current_point["coordinates"] = np.add(current_point["coordinates"], force)
      current_point["coordinates"] = list(np.clip(np.add(current_point["coordinates"], force), 0, min(volume.shape)-1))
      previous_match.append(best_match)

    # describe points new position with defined depth
    current_points = pointDescriptor.getDescriptionForPoints(volume, current_points)

    return [previous_matches, current_points]


# uncompyle6 version 3.6.7
# Python bytecode 2.4 (62061)
# Decompiled from: Python 3.8.2 (tags/v3.8.2:7b3ab59, Feb 25 2020, 23:03:10) [MSC v.1916 64 bit (AMD64)]
# Embedded file name: build/bdist.linux-i686/egg/pydbgr/processor/command/show_subcmd/aliases.py
# Compiled at: 2013-03-18 05:57:53
import columnize
from import_relative import import_relative
Mbase_subcmd = import_relative('base_subcmd', '..', 'pydbgr')

class ShowAliases(Mbase_subcmd.DebuggerShowIntSubcommand):
    """**show aliases** [*alias* ...| *]

Show command aliases. If parameters are given a list of all aliases and
the command they run are printed. Alternatively one can list specific
alias names for the commands those specific aliases are attached to.
If instead of an alias `*` appears anywhere as an alias then just a list
of aliases is printed, not what commands they are attached to.
"""
    __module__ = __name__
    min_abbrev = len('al')
    short_help = 'Show command aliases'
    run_cmd = False

    def _alias_header(self):
        self.section('%-10s : %s' % ('Alias', 'Command'))
        self.msg('%-10s : %s' % ('-' * 10, '-' * 11))

    def _alias_line(self, alias):
        self.msg('%-10s : %s' % (alias, self.proc.aliases[alias]))

    def run(self, args):
        aliases = list(self.proc.aliases.keys())
        aliases.sort()
        if len(args) == 0:
            self._alias_header()
            for alias in aliases:
                self._alias_line(alias)

            return
        if '*' in args:
            self.section('Current aliases:')
            self.msg(columnize.columnize(aliases, lineprefix='    '))
        else:
            self._alias_header()
            for alias in args:
                if alias in aliases:
                    self._alias_line(alias)
                else:
                    self.errmsg('%s is not an alias' % alias)

            return

if __name__ == '__main__':
    Mhelper = import_relative('__demo_helper__', '.', 'pydbgr')
    sub = Mhelper.demo_run(ShowAliases)
    sub.run(['*'])
    sub.run(['s+', 'n+'])


# import datetime
import math

def matrixmultiply(matrix1, matrix2):
    a = (matrix1[0][0] * matrix2[0][0]) + (matrix1[0][1] * matrix2[1][0])
    b = (matrix1[0][0] * matrix2[0][1]) + (matrix1[0][1] * matrix2[1][1])
    c = (matrix1[1][0] * matrix2[0][0]) + (matrix1[1][1] * matrix2[1][0])
    d = (matrix1[1][0] * matrix2[0][1]) + (matrix1[1][1] * matrix2[1][1])
    return [[a, b], [c, d]]

def twobytwopower(exponent):
    matrix = [[1, 0], [0, 1]]
    left = exponent

    while left > 0:
        power = int(math.log(left, 2))
        tempmatrix = [[0, 1], [1, 1]]
        for x in range(power):
            tempmatrix = matrixmultiply(tempmatrix, tempmatrix)

        matrix = matrixmultiply(matrix, tempmatrix)
        left = left - (math.pow(2, power))

    return matrix

def fibmatrix(num):
    matrix = twobytwopower(num)
    return matrix[0][0] + matrix[0][1]

def fibrecursive(num):
    if num == 1 or num == 0:
        return 1

    return fibrecursive(num - 1) + fibrecursive(num - 2)

def fibloop(num):
    f0 = 1
    f1 = 1
    for x in range(num):
        temp = f1
        f1 += f0
        f0 = temp

    return f0

def fibformula(num):
    return (((((1 + math.sqrt(5)) / 2) ** num) - (((1 - math.sqrt(5)) / 2) ** num)) / math.sqrt(5))

# start = datetime.datetime.now()
# fibmatrix(100000)
# end = datetime.datetime.now()
# matrixtime = end - start

# start = datetime.datetime.now()
# fibrecursive(30)
# end = datetime.datetime.now()
# recursivetime = end - start

# start = datetime.datetime.now()
# fibloop(100000)
# end = datetime.datetime.now()
# looptime = end - start

# start = datetime.datetime.now()
# fibformula(1000)
# end = datetime.datetime.now()
# formulatime = end - start

# print fibmatrix(100000) == fibloop(100000)

# print "Matrix took %r" % matrixtime.total_seconds()
# print "Recursive took %r" % recursivetime.total_seconds()
# print "Loop took %r" % looptime.total_seconds()
# print "Formula took %r" % formulatime.total_seconds()


# coding=utf-8
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import matplotlib.animation as animation
import matplotlib.pyplot as plt
import numpy as np
import seaborn

from src.fourier_series import piecewise_analytic

def animate_series(series, max_degree):
    series = series
    x = series.x
    x0 = series.x0
    gen = series()
    index = 0

    step_function = piecewise_analytic.step_function(x, x0, 1, 0)
    v_function = piecewise_analytic.v_function(x, x0, 1, -1, x0)

    seaborn.set()
    fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(2 * 6, 2 * 8))

    # Series
    axes[0, 0].set(xlim=(np.min(x), np.max(x)),
                   ylim=(np.min(step_function)-0.5, np.max(step_function)+0.5))
    axes[0, 1].set(xlim=(np.min(x), np.max(x)),
                   ylim=(np.min(v_function)-0.01, np.max(v_function)+0.01))
    # Error
    axes[1, 0].set(xlim=(np.min(x), np.max(x)), ylim=(10 ** -7, 1))
    axes[1, 1].set(xlim=(np.min(x), np.max(x)), ylim=(10 ** -7, 1))
    # Functions
    axes[0, 0].plot(x, step_function)
    axes[0, 1].plot(x, v_function)

    l1, = axes[0, 0].plot([], [], marker='o', markersize=2.0, linestyle='-',
                          color='black', alpha=0.5)
    l2, = axes[0, 1].plot([], [], marker='o', markersize=2.0, linestyle='-',
                          color='black', alpha=0.5)
    l3, = axes[1, 0].semilogy([], [], marker='o', markersize=2.0, linestyle='-',
                              color='black', alpha=0.5)
    l4, = axes[1, 1].semilogy([], [], marker='o', markersize=2.0, linestyle='-',
                              color='black', alpha=0.5)

    def update(_):
        # series expansion
        deg, functions = next(gen)

        step_function3 = functions[0]
        step_function_series = step_function3[index]
        step_function_error = np.abs(step_function - step_function_series)

        l1.set_xdata(x)
        l1.set_ydata(step_function_series)

        l3.set_xdata(x)
        l3.set_ydata(step_function_error)

        if len(functions) == 2:
            v_function3 = functions[1]

            v_function_series = v_function3[index]
            v_function_error = np.abs(v_function - v_function_series)

            l2.set_xdata(x)
            l2.set_ydata(v_function_series)

            l4.set_xdata(x)
            l4.set_ydata(v_function_error)

        print(deg)
        return l1, l2, l3, l4

    anim = animation.FuncAnimation(fig, update,
                                   frames=max_degree,
                                   repeat=False,
                                   interval=10,
                                   blit=True)

    plt.show()


from . import views
from django.urls import path

app_name = 'userscreen'

urlpatterns = [
    path('',views.mainpage_view, name="mainpage"),
    path('userview/', views.user_view, name="userview"),
    path('create/', views.userPage_create, name="create"),
    path('showpvmforms/', views.showPvmForms, name="showpvmforms"),
    path('createpvmform/', views.userPage_create, name="createpvmform"),
    path('createform/<id>/', views.createForm, name="createform"),
    path('addmembers/', views.addMembers, name="addmembers"),
    path('addvertybe/',views.addVertybe, name="addvertybe"),
    path('pdf/',views.get, name="get")
]


def image_extract(pmcid):

    url = f"https://europepmc.org/article/PMC/{pmcid}"

    import chromedriver_autoinstaller
    from selenium import webdriver
    from selenium.webdriver.common.by import By
    from selenium.webdriver.support import expected_conditions as ec
    from selenium.webdriver.common.keys import Keys
    from selenium.webdriver.support.ui import WebDriverWait
    from selenium.common.exceptions import NoSuchElementException, ElementNotVisibleException
    from selenium.webdriver.common.alert import Alert
    from selenium.webdriver.common.action_chains import ActionChains
    from selenium.common.exceptions import TimeoutException
    from selenium.webdriver.chrome.options import Options
    import urllib
    from selenium import webdriver

    chromedriver_autoinstaller.install()
    options = webdriver.ChromeOptions()
    webdriver = webdriver.Chrome(
        options=options
    )
    print("Started Searching")
    webdriver.get(url)

    images = []
    a = 0
    keep_seaching = True
    while(keep_seaching):
        try:

            results = WebDriverWait(webdriver, 10).until(ec.visibility_of_element_located(
                (By.XPATH, "//a[@class='figures--figure-box']")))
            results = webdriver.find_elements_by_xpath(
                "//a[@class='figures--figure-box']")
        except:
            break
        try:
            button_scroll = WebDriverWait(webdriver, 2).until(ec.visibility_of_element_located(
                (By.XPATH, "//i[@class='fas fa-angle-right fa-stack-1x']")))
        except:
            break

        for result in results:
            img_data = result.get_attribute("style")
            img_data = img_data.strip("background-image: ")
            img_data = img_data.strip('l("')
            img_data = img_data.lstrip('");')
            if img_data not in images:
                images.append(img_data)

        if button_scroll:
            webdriver.execute_script("arguments[0].click();", button_scroll)

        else:
            break

    for image in images:
        response = urllib.request.urlopen(str(image))
        with open(f'{pmcid}_{a}.jpg', 'wb') as f:
            f.write(response.file.read())
        a += 1

image_extract("PMC8120281")


from django.shortcuts import render
from django.http import HttpResponse, HttpResponseNotFound
from models import Table
from django.views.decorators.csrf import csrf_exempt

# Create your views here.

def login(request):
    if request.user.is_authenticated():
        return ("
You're: " + request.user.username +
                "
Logout")
    else:
        return ("
You aren't registred\nLogin")

def all(request):
    list = Table.objects.all()
    out = "
\n"
    for i in list:
        out += "
" + i.name + "
\n"
    out += "
\n"
    out += login(request)
    return HttpResponse(out)

@csrf_exempt
def number(request, recurso):
    if request.method == "GET":
        list = Table.objects.filter(name=recurso)
        if not list:
            return notfound(request, recurso)
        out = " "
        for i in list:
            out += i.name + ": " + str(i.number)
    if request.method == "PUT":
        if request.user.is_authenticated():
            new = Table(name=recurso, number=request.body)
            new.save()
            out = ("Saved Page, check it with GET")
        else:
            out = ("You must be registred")
    out += login(request)
    return HttpResponse(out)

def notfound(request, recurso):
    out = ("Not found: " + recurso)
    out += login(request)
    return HttpResponseNotFound(out)


#!/usr/bin/env python

# Edited by xbacov04

import sys
import signal
import rospy
from PyQt5 import QtWidgets, QtCore
from xbacov04.gui import SceneViewer

def sigint_handler(*args):
    """Handler for the SIGINT signal."""
    sys.stderr.write('\r')
    QtWidgets.QApplication.quit()

def main(args):

    rospy.init_node('rescuers_gui_projector', anonymous=True)
    rospy.sleep(1)

    signal.signal(signal.SIGINT, sigint_handler)

    app = QtWidgets.QApplication(sys.argv)

    ps = SceneViewer()

    timer = QtCore.QTimer()
    timer.start(500)
    timer.timeout.connect(lambda: None)  # Let the interpreter run each 500 ms.

    sys.exit(app.exec_())

if __name__ == '__main__':
    try:
        main(sys.argv)
    except KeyboardInterrupt:
        print("Shutting down")


# Lint as: python3
# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# pylint: disable=unused-variable

from absl import logging
from absl.testing import absltest
import numpy as np
from pyiree import compiler2 as compiler
from pyiree import rt

def create_add_scalar_module():
  binary = compiler.compile_str("""
    func @add_scalar(%arg0: i32, %arg1: i32) -> i32 attributes { iree.module.export } {
      %0 = addi %arg0, %arg1 : i32
      return %0 : i32
    }
    """,
                                target_backends=["vmla"])
  m = rt.VmModule.from_flatbuffer(binary)
  return m

def create_simple_static_mul_module():
  binary = compiler.compile_str("""
    func @simple_mul(%arg0: tensor<4xf32>, %arg1: tensor<4xf32>) -> tensor<4xf32>
          attributes { iree.module.export } {
        %0 = "mhlo.multiply"(%arg0, %arg1) {name = "mul.1"} : (tensor<4xf32>, tensor<4xf32>) -> tensor<4xf32>
        return %0 : tensor<4xf32>
    }
    """,
                                target_backends=["vmla"])
  m = rt.VmModule.from_flatbuffer(binary)
  return m

def create_simple_dynamic_abs_module():
  # TODO(laurenzo): Compile for more backends as dynamic shapes come online.
  target_backends = ["vmla"]
  binary = compiler.compile_str("""
    func @simple_mul(%arg0: tensor<?x?xf32>) -> tensor<?x?xf32>
          attributes { iree.module.export } {
        %0 = "mhlo.abs"(%arg0) : (tensor<?x?xf32>) -> tensor<?x?xf32>
        return %0 : tensor<?x?xf32>
    }
    """,
                                target_backends=target_backends)
  m = rt.VmModule.from_flatbuffer(binary)
  return m

class VmTest(absltest.TestCase):

  @classmethod
  def setUpClass(cls):
    super().setUpClass()
    driver_names = rt.HalDriver.query()
    logging.info("driver_names: %s", driver_names)
    cls.driver = rt.HalDriver.create("vmla")
    cls.device = cls.driver.create_default_device()
    cls.hal_module = rt.create_hal_module(cls.device)
    cls.htf = rt.HostTypeFactory.get_numpy()

  def test_variant_list(self):
    l = rt.VmVariantList(5)
    logging.info("variant_list: %s", l)
    self.assertEqual(l.size, 0)

  def test_context_id(self):
    instance = rt.VmInstance()
    context1 = rt.VmContext(instance)
    context2 = rt.VmContext(instance)
    self.assertGreater(context2.context_id, context1.context_id)

  def test_module_basics(self):
    m = create_simple_static_mul_module()
    f = m.lookup_function("simple_mul")
    self.assertGreater(f.ordinal, 0)
    notfound = m.lookup_function("notfound")
    self.assertIs(notfound, None)

  def test_dynamic_module_context(self):
    instance = rt.VmInstance()
    context = rt.VmContext(instance)
    m = create_simple_static_mul_module()
    context.register_modules([self.hal_module, m])

  def test_static_module_context(self):
    m = create_simple_static_mul_module()
    logging.info("module: %s", m)
    instance = rt.VmInstance()
    logging.info("instance: %s", instance)
    context = rt.VmContext(instance, modules=[self.hal_module, m])
    logging.info("context: %s", context)

  def test_dynamic_shape_compile(self):
    m = create_simple_dynamic_abs_module()
    logging.info("module: %s", m)
    instance = rt.VmInstance()
    logging.info("instance: %s", instance)
    context = rt.VmContext(instance, modules=[self.hal_module, m])
    logging.info("context: %s", context)

  def test_add_scalar(self):
    m = create_add_scalar_module()
    instance = rt.VmInstance()
    context = rt.VmContext(instance, modules=[self.hal_module, m])
    f = m.lookup_function("add_scalar")
    abi = context.create_function_abi(self.device, self.htf, f)
    logging.info("abi: %s", abi)

    inputs = abi.pack_inputs(5, 6)
    logging.info("serialize_inputs: %s", abi.serialize_vm_list(inputs))
    logging.info("inputs: %s", inputs)

    allocated_results = abi.allocate_results(inputs, static_alloc=False)
    logging.info("allocated_results: %s", allocated_results)
    logging.info("Invoking...")
    context.invoke(f, inputs, allocated_results)
    logging.info("...done")

    result = abi.unpack_results(allocated_results)
    logging.info("result: %s", result)
    self.assertEqual(result, 11)

  def test_synchronous_dynamic_shape_invoke_function(self):
    m = create_simple_dynamic_abs_module()
    instance = rt.VmInstance()
    context = rt.VmContext(instance, modules=[self.hal_module, m])
    f = m.lookup_function("simple_mul")
    abi = context.create_function_abi(self.device, self.htf, f)
    logging.info("abi: %s", abi)

    arg0 = np.array([[-1., 2.], [3., -4.]], dtype=np.float32)
    inputs = abi.pack_inputs(arg0)
    logging.info("Serialized inputs: %s", abi.serialize_vm_list(inputs))
    logging.info("inputs: %s", inputs)

    allocated_results = abi.allocate_results(inputs, static_alloc=False)
    logging.info("allocated_results: %s", allocated_results)
    logging.info("Invoking...")
    context.invoke(f, inputs, allocated_results)
    logging.info("...done")

    result = abi.unpack_results(allocated_results)
    logging.info("result: %s", result)
    np.testing.assert_allclose(result, [[1., 2.], [3., 4.]])

  def test_synchronous_invoke_function(self):
    m = create_simple_static_mul_module()
    instance = rt.VmInstance()
    context = rt.VmContext(instance, modules=[self.hal_module, m])
    f = m.lookup_function("simple_mul")
    abi = context.create_function_abi(self.device, self.htf, f)
    logging.info("abi: %s", abi)

    arg0 = np.array([1., 2., 3., 4.], dtype=np.float32)
    arg1 = np.array([4., 5., 6., 7.], dtype=np.float32)
    inputs = abi.pack_inputs(arg0, arg1)
    logging.info("Serialized inputs: %s", abi.serialize_vm_list(inputs))
    logging.info("inputs: %s", inputs)

    allocated_results = abi.allocate_results(inputs, static_alloc=False)
    logging.info("allocated_results: %s", allocated_results)
    logging.info("Invoking...")
    context.invoke(f, inputs, allocated_results)
    logging.info("...done")

    result = abi.unpack_results(allocated_results)
    logging.info("result: %s", result)
    np.testing.assert_allclose(result, [4., 10., 18., 28.])

if __name__ == "__main__":
  absltest.main()


from csv import DictReader
import json
from schema import userRecord
from address_parser import address_parser, nameSplit
import re
import uuid
import userGroup
def space_snip(string):
    # collapse consecutive spaces
    snipped = re.sub("  +", "", string)
    return snipped

def date_format(date):
    year = date[0:4]
    month = date[4:6]
    day = date[6:]
    return (f"{year}-{month}-{day}")

def userMaker(file):
    outputFile = []
    with open("amh_users.txt", 'r', encoding='utf8') as user_file:
        userparse =  DictReader(user_file, delimiter = "|")
        for row in userparse:
            x = uuid.uuid4()
            address = address_parser(row['ADDRESS1'])
            address = address.parse()
            name = nameSplit(row["Z303_NAME"])
            name = name.namesplit()
            name["email"] = row["EMAIL1"]
            name["phone"] = row["PHONE2"]
            if row['Z305_BOR_TYPE'] in ["EM", "RE", "SP"]:
                group = row["Z305_BOR_STATUS"] + "AC"
                #print (group)
            else:
                group =  row["Z305_BOR_STATUS"]+ row['Z305_BOR_TYPE']
            groupUUID = userGroup.groups("userGroups.csv", group)

            if address:

                if  len (space_snip(row["ZIP1"])) > 3:
                    address['postalCode'] = space_snip(row["ZIP1"])
                name["addresses"] = [address]

            newUser = userRecord(
                id = str(x),
                externalSystemId = space_snip(row["EXTERNAL_ID"]),
                barcode =  space_snip(row['BARCODE']),
                type = row['Z305_BOR_TYPE'],
                personal = name,
                expirationDate = date_format(row['Z305_EXPIRY_DATE']),
                patronGroup = groupUUID.getgroup(),
                customFields = {"bor_type": row['Z305_BOR_TYPE'],
                            "alephKey": space_snip(row["Z303_REC_KEY"]) })
            outputFile.append(newUser)

        return outputFile

if __name__ == "__main__":
    x = userMaker("mhc_users.txt")
    with open("outputFile.json", 'w') as output:
        json.dump(x, output, indent = 4)


import logging

from board.cfg_db import mod_debug
mod_debug()

#board
log = logging.getLogger("BOARD")
log.setLevel(logging.INFO)

from board.cfg_db import _name_board
import sys

class uCore:
    def __init__(self, mbus, umod, board):
        self.mbus = mbus
        self.umod = umod
        self.board = board

class Activate:

    def __init__(self, mbus, umod, board_id):

        self.mbus = mbus
        self.umod = umod

        log.info("BID: {}".format(board_id))

        s_mod = umod.mod_sel("board_cfg")

        if not len(s_mod):
            from mod_u.model import init_db
            init_db(umod)
            del sys.modules["mod_u.model"]

        from board.cfg_db import push_board
        push_board(umod)

        board = _name_board

        _modules = umod.call_cmd("_scan", "board_mod")

        for mod in _modules:
            if mod["active"]:
                name_mod = mod["name"]
                try:
                    init_db = __import__("{}_mod".format(name_mod)).init_db
                    init_db(umod)
                    log.info("MOD DB ACTIVATE: {}".format(name_mod))
                    del sys.modules["{}_mod".format(name_mod)]

                except Exception as e:
                    log.info("MOD: {}, init db err: {}".format(mod, e))

        from board.cfg_db import push_data
        push_data(umod)
        del sys.modules["board.cfg_db"]

        umod.call_cmd("_upd", "board_cfg", {"board": _name_board}, init=1, uid=board_id.decode())

        # s_mod = umod.rec_sel("cfg_board", name=_name_board)
        # if len(s_mod):
        #     board = s_mod[0]["board"]
        #     if s_mod[0]["init"] == 0:
        #         from board.cfg_db import push_data
        #         push_data(umod)
        #         umod.rec_upd("cfg_board", {"board": board}, init=1, uid=board_id.decode())
        # else:
        #     log.info("BOARD: NOT CONFIGURE")

        self.core = uCore(mbus, umod, board)
        for mod in _modules:
            if mod["active"]:
                name_mod = mod["name"]

                try:
                    init_act = __import__("{}_mod".format(name_mod)).init_act
                    self.core = init_act(self.core)
                    log.info("MOD ACT ACTIVATE: {}".format(name_mod))
                    del sys.modules["{}_mod".format(name_mod)]
                    # umod.call_cmd("_upd", "board_mod", {"name": name_mod}, status="loaded")
                except Exception as e:
                    log.info("MOD: {}, activate err: {}".format(mod, e))


import time

USER = "USER SAID: {}"
BOT = "BOT SAYS: {}"

lets_talk = True

def say_something():
    return input("What would you like to say to Echobot? ")

def respond(message):
    bot_says = "I hear you! This is what you said: {}"
    return bot_says.format(message)

def send_message(message):
    print(USER.format(message))
    time.sleep(0.5)
    print(BOT.format(respond(message)))

while lets_talk == True:
    message = say_something()
    if message.strip().lower() == "goodbye":
        lets_talk = False
    send_message(message)


# Question: https://leetcode.com/explore/challenge/card/may-leetcoding-challenge/537/week-4-may-22nd-may-28th/3339/

"""
Return the root node of a binary search tree that matches the given preorder traversal.
(Recall that a binary search tree is a binary tree where for every node, any descendant of
node.left has a value < node.val, and any descendant of node.right has a value > node.val.
Also recall that a preorder traversal displays the value of the node first, then traverses node.left, then traverses node.right.)

It's guaranteed that for the given test cases there is always possible to find a binary search tree with the given requirements.

Example 1:
    Input: [8,5,1,7,10,12]
    Output: [8,5,10,1,7,null,12]
                8
               / \
              5   10
             / \    \
            1   7    12

Constraints:
    (1) 1 <= preorder.length <= 100
    (2) 1 <= preorder[i] <= 10^8
    (3) The values of preorder are distinct.
"""

# Definition for a binary tree node.
class TreeNode:
    def __init__(self, val=0, left=None, right=None):
        self.val = val
        self.left = left
        self.right = right
class Solution:
    def bstFromPreorder(self, preorder):    # bstFromPreorder(self, preorder: List[int]) -> TreeNode
        bst_size = len(preorder)
        root = TreeNode(preorder[0])
        stack = [root]

        for i in range(1, bst_size):
            temp = None

            while stack and preorder[i] > stack[-1].val:
                temp = stack.pop()

            if temp:
                temp.right = TreeNode(preorder[i])
                stack.append(temp.right)
            else:
                temp = stack[-1]
                temp.left = TreeNode(preorder[i])
                stack.append(temp.left)

            # stack.append(TreeNode(preorder[i]))
            """
            Above statement just appends a node in stack.
            This causes the loss of linkage between the nodes.
            We just have individual nodes in the stack.
            """

        return root

2020 - May LeetCoding Challenge/1008-construct-binary-search-tree-from-preorder-traversal.py


from rapidsms.apps.base import AppBase
from rapidsms.models import Connection
from efatha.models import Message, Person

class MessageReceiver(AppBase):

    def handle(self, msg):
        if msg.text != '':
            connection, created = Connection.objects.get_or_create(
                backend=msg.connections[0].backend,
                identity=msg.connections[0].identity)

            if connection.contact is None:
                connection.contact = Person.objects.create()
                connection.save()
                Message(
                    person=connection.contact,
                    text=msg.text).save()
            else:
                Message(
                    person=connection.contact.person,
                    text=msg.text).save()
            return True
        return False


"""pokus

Revision ID: 9e7a47c85b58
Revises: 688fd85f3129
Create Date: 2019-12-08 18:59:30.176186

"""
from alembic import op
import sqlalchemy as sa
from datetime import datetime

# revision identifiers, used by Alembic.
revision = '9e7a47c85b58'
down_revision = '688fd85f3129'
branch_labels = None
depends_on = None

def upgrade():
    op.add_column("Kalkulacka", sa.Column("date", sa.DateTime, default=datetime.utcnow()))

def downgrade():
    op.drop_column("Kalkulacka", "date")


def merge_sort(lst):
    if len(lst) > 1:
        mid = len(lst) // 2
        last = lst[:mid]
        first = lst[mid:]
        merge_sort(last)
        merge_sort(first)
        i = j = k = 0

        while i < len(last) and j < len(first):
            if last[i] < first[j]:
                lst[k] = last[i]
                i += 1
            else:
                lst[k] = first[j]
                j += 1
            k += 1
        while i < len(last):
            lst[k] = last[i]
            i += 1
            k += 1
        while j < len(first):
            lst[k] = first[j]
            j += 1
            k += 1
    return lst

number_list = [int(x) for x in input("Enter list of number seperated by space ").split()]
print(f'Number list: {number_list}')
print(f'Sorted list: {merge_sort(number_list)}')


from textblob import TextBlob
from textblob.sentiments import NaiveBayesAnalyzer
import math
import storm
import json
hateWords = set()
hateFile = open("resources/hatewords.txt", "r")
for word in hateFile:
    hateWords.add(word.rstrip().lower())
hateFile.close()

#For this to work, it is necessary to use the following command:
#pip install textblob
#The test.txt uses hate words to say positive stuff about racism, etc
#The score will be greater than 0 because the statement is positive
#even though the content is not.

class tweetFilteredSentiment(storm.BasicBolt):
    global hateWords

    def process(self,tup):
    	tweet = tup.values[0]
    	tweet['tweettext'] = tweet['text']
    	del tweet['text']
    	tweet['sentimentscore'] = str(self.checkForHateWords(tweet['tweettext']))
    	storm.emit([tweet])

    def checkForHateWords(self,tweet):
        for word in tweet.split():
            if word.lower() in hateWords:
                score = TextBlob(tweet)
                return abs(float(score.sentiment.polarity))
        return 0.0

tweetFilteredSentiment().run()


#打猴子补丁  打补丁： 在不修改程序源码的情况下，为程序添加原来不支持的功能
from gevent import monkey
monkey.patch_all() #破解所有

import time,gevent
#创建生成器1
def work1():
    while True:
        print("正在执行work1",gevent.getcurrent()) #没有产生新的线程 在原线程自动划分不同编号
        time.sleep(0.5) #默认情况 不被gevent 识别为耗时操作 需要打补丁
        # gevent.sleep(0.5)
        #打补丁： 在不修改程序源码的情况下，为程序添加原来不支持的功能

#创建生成器1
def work2():
    while True:
        print("正在执行work2",gevent.getcurrent())
        # time.sleep(0.5)
        gevent.sleep(0.5)

if __name__ == '__main__':
    #创建greenlet的对象
    g1=gevent.spawn(work1)
    g2=gevent.spawn(work2)
    g1.join() #让主线程等待协程执行完毕后再退出
    # gevent.joinall() 批量让携程join


'''
Smith waterman algorithm for local allignment
overall global score will be >= 0
'''

def smith_waterman(seq1, seq2, score):
    '''Calculated score matrix for given sequence and scoring schema.
    '''
    row, col = len(seq1), len(seq1)
    max_score = 0
    matrix = [[0 for _ in range(col + 1)] for _ in range(row + 1)]

    for i in range(1, row + 1):
        for j in range(1, col + 1):
            similarity = score["match"] if str1[i - 1] == str2[j - 1] else score["mismatch"]

            score_matrix = max(
                0,
                matrix[i - 1][j] + score["gap"],
                matrix[i][j - 1] + score["gap"],
                matrix[i - 1][j - 1] + similarity
            )
            matrix[i][j] = score_matrix
            max_score = max(score_matrix, max_score)

    return matrix, max_score

def print_matrix(martix):
    '''Print calculated scoring matrix.
    '''
    print("Smith Waterman matrix :")
    for r in matrix:
        print(r, sep=", ")

def main():
    score = {"gap": -2, "mismatch": -1, "match": 2}
    str2 = "ccg"
    str1 = "cgg"
    matrix, max_score = smith_waterman(str1, str2, score)
    print_matrix(matrix)
    print("Local alignment score =", max_score)

if __name__ == "__main__":
    main()


#!/usr/bin/env python
# coding: utf-8

# # Targeting Direct Marketing
#
# ---
#
# ## Background
# Direct marketing, either through mail, email, phone, etc., is a common tactic to acquire customers. Because resources and a customer's attention are limited, the goal is to only target the subset of prospects who are likely to engage with a specific offer. Predicting those potential customers based on readily available information like demographics, past interactions, and environmental factors is a common machine learning problem.
#
# This notebook presents an example problem to predict if a customer will enroll for a term deposit at a bank after one or more phone calls. The steps include:
#
# * Preparing your Amazon SageMaker notebook
# * Downloading data from the internet into Amazon SageMaker
# * Investigating and transforming the data so that it can be fed to Amazon SageMaker algorithms
# * Estimating a model
# * Evaluating the effectiveness of the model
# * Setting the model up to make ongoing predictions
#
# ---
#
# ## Preparation
#
# _This notebook was created and tested on an ml.m4.xlarge notebook instance._
#
# Start by specifying:
#
# - The Amazon Simple Storage Service (Amazon S3) bucket and prefix that you want to use for training and model data. This should be within the same Region as the Notebook Instance, training, and hosting.
# - The AWS Identity and Access Management (IAM) role Amazon Resource Name (ARN) used to give training and hosting access to your data. See the documentation for how to create these.
#
# **Note:** If more than one role is required for notebook instances, training, and/or hosting, please replace the boto regexp with the appropriate full IAM role ARN string(s).
#
# Replace **`<LabBucketName>`** with the resource name that was provided with your lab account.

# In[1]:

#bucket = 'ml-pipeline-bucket'
bucket = 'ml-pipeline-bucket'
prefix = 'sagemaker/DEMO-xgboost-dm'

# Define IAM role
import boto3
import re
from sagemaker import get_execution_role

role = get_execution_role()

# Bring in the Python libraries that you'll use throughout the analysis.

# In[2]:

import numpy as np                                # For matrix operations and numerical processing
import pandas as pd                               # For munging tabular data
import matplotlib.pyplot as plt                   # For charts and visualizations
from IPython.display import Image                 # For displaying images in the notebook
from IPython.display import display               # For displaying outputs in the notebook
from time import gmtime, strftime                 # For labeling Amazon SageMaker models, endpoints, etc.
import sys                                        # For writing outputs to the notebook
import math                                       # For ceiling function
import json                                       # For parsing hosting outputs
import os                                         # For manipulating filepath names
import sagemaker                                  # Amazon SageMaker's Python SDK provides many helper functions
from sagemaker.predictor import CSVSerializer    # Converts strings for HTTP POST requests on inference

# ---
#
# ## Data
# Start by downloading the [Bank Marketing Data Set](https://archive.ics.uci.edu/ml/datasets/bank+marketing) from the University of California, Irvine (UCI) Machine Learning ML Repository.
#
# **Source information:**
#
# S. Moro, P. Cortez and P. Rita. A Data-Driven Approach to Predict the Success of Bank Telemarketing. Decision Support Systems, Elsevier, 62:22-31, June 2014

# In[3]:

get_ipython().system('aws s3 cp s3://aws-tc-largeobjects/ILT-TF-200-MLDWTS/lab1/bank-additional.zip .')
#!wget https://archive.ics.uci.edu/ml/machine-learning-databases/00222/bank-additional.zip
get_ipython().system('unzip -o bank-additional.zip')

# Now, read this into a Pandas dataframe and take a look.

# In[4]:

data = pd.read_csv('./bank-additional/bank-additional-full.csv', sep=';')
pd.set_option('display.max_columns', 500)     # Makes sure you can see all of the columns
pd.set_option('display.max_rows', 20)         # Keeps the output on one page

# In[5]:

data.head()

# Let's talk about the data.  At a high level, you can see:
#
# * There are just over 40K customer records and 20 features for each customer.
# * The features are mixed--some numeric, some categorical.
# * The data appears to be sorted, at least by `time` and `contact`.
#

# ### Exploration
# Let's explore the data. First, let's understand how the features are distributed.

# In[6]:

# Frequency tables for each categorical feature
for column in data.select_dtypes(include=['object']).columns:
    display(pd.crosstab(index=data[column], columns='% observations', normalize='columns')*100)

# In[7]:

# Histograms for each numeric feature
display(data.describe())
display(data.describe(include=np.object))
#%matplotlib inline
hist = data.hist(bins=30, sharey=True, figsize=(10, 10))

# Notice that:
#
# - Almost 90% of the values for our target variable y are "no", so most customers did not subscribe to a term deposit.
# - Many of the predictive features take on values of "unknown". Some are more common than others. We should think carefully as to what causes a value of "unknown" (are these customers non-representative in some way?) and how to handle that.
# - Even if "unknown" is included as its own distinct category, what does it mean, given that those observations likely fall within one of the other categories of that feature?
# - Many of the predictive features have categories with few observations in them. If we find a small category to be highly predictive of our target outcome, do we have enough evidence to make a generalization about that?
# - Contact timing is particularly skewed--almost a third in May and less than 1% in December. What does this mean for predicting our target variable next December?
# - There are no missing values in our numeric features, or missing values have already been imputed.
#   - `pdays` takes a value near 1,000 for almost all customers. This is likely a placeholder value signifying no previous contact.
# - Several numeric features have a long tail. Do we need to handle these few observations with extremely large values differently?
# - Several numeric features (particularly the macroeconomic ones) occur in distinct buckets. Should these be treated as categorical?
#
# Next, let's look at how our features relate to the target that we are attempting to predict.

# In[8]:

for column in data.select_dtypes(include=['object']).columns:
    if column != 'y':
        display(pd.crosstab(index=data[column], columns=data['y'], normalize='columns'))

for column in data.select_dtypes(exclude=['object']).columns:
    print(column)
    hist = data[[column, 'y']].hist(by='y', bins=30)
    plt.show()

# Notice that:
# - Customers who are "blue-collar", "married", "unknown" default status, contacted by "telephone", and/or in "May" are a substantially lower portion of "yes" than "no" for subscribing.
# - Distributions for numeric variables are different across "yes" and "no" subscribing groups, but the relationships may not be straightforward or obvious.
#
# Now let's look at how our features relate to one another.

# In[9]:

display(data.corr())
pd.plotting.scatter_matrix(data, figsize=(12, 12))
plt.show()

# Notice that:
#
# - Features vary widely in their relationships with one another. Some have highly negative correlation; others have highly positive correlation.
# - Relationships between features are non-linear and discrete in many cases.

# ### Transformation
#
# Cleaning up data is part of nearly every machine learning project.  It arguably presents the biggest risk if done incorrectly and is one of the more subjective aspects in the process.

# In[10]:

data['no_previous_contact'] = np.where(data['pdays'] == 999, 1, 0)                                 # Indicator variable to capture when pdays takes a value of 999
data['not_working'] = np.where(np.in1d(data['job'], ['student', 'retired', 'unemployed']), 1, 0)   # Indicator for individuals not actively employed
model_data = pd.get_dummies(data)                                                                  # Convert categorical variables to sets of indicators

# In[11]:

model_data = model_data.drop(['duration', 'emp.var.rate', 'cons.price.idx', 'cons.conf.idx', 'euribor3m', 'nr.employed'], axis=1)

# ### Train-test split

# In[12]:

train_data, validation_data, test_data = np.split(model_data.sample(frac=1, random_state=1729), [int(0.7 * len(model_data)), int(0.9 * len(model_data))])   # Randomly sort the data; then split out first 70%, second 20%, and last 10%

# Amazon SageMaker's XGBoost container expects data in the libSVM or CSV data format. For this example, you'll stick to CSV. Note that the first column must be the target variable, and the CSV should not include headers. Also, notice that although repetitive, it's easiest to do this after the train|validation|test split rather than before. This avoids any misalignment issues due to random reordering.

# In[13]:

pd.concat([train_data['y_yes'], train_data.drop(['y_no', 'y_yes'], axis=1)], axis=1).to_csv('train.csv', index=False, header=False)
pd.concat([validation_data['y_yes'], validation_data.drop(['y_no', 'y_yes'], axis=1)], axis=1).to_csv('validation.csv', index=False, header=False)

# Now, copy the file to Amazon S3 for Amazon SageMaker's managed training to pick up.

# In[14]:

boto3.Session().resource('s3').Bucket(bucket).Object(os.path.join(prefix, 'train/train.csv')).upload_file('train.csv')
boto3.Session().resource('s3').Bucket(bucket).Object(os.path.join(prefix, 'validation/validation.csv')).upload_file('validation.csv')

# ---
#
# ## Training
#
# First, you need to specify the Amazon Elastic Container Registry (Amazon ECR) container location for Amazon SageMaker's implementation of XGBoost.

# In[15]:

container = sagemaker.image_uris.retrieve('xgboost',boto3.Session().region_name, '1.0-1')

# Then, because you're training with the CSV file format, create `s3_input`s for the training function to use as a pointer to the files in Amazon S3. This also specifies that the content type is CSV.

# In[16]:

s3_input_train = sagemaker.inputs.TrainingInput(s3_data='s3://{}/{}/train'.format(bucket, prefix), content_type='csv')
s3_input_validation = sagemaker.inputs.TrainingInput(s3_data='s3://{}/{}/validation/'.format(bucket, prefix), content_type='csv')

# Now, you need to specify training parameters to the estimator. This includes:
# - The `xgboost` algorithm container
# - The IAM role to use
# - Training instance type and count
# - Amazon S3 location for output data
# - Algorithm hyperparameters
#
# And then a `.fit()` function, which specifies the Amazon S3 location for output data. In this case, both a training and validation set are passed in.

# In[17]:

sess = sagemaker.Session()

xgb = sagemaker.estimator.Estimator(container,
                                    role,
                                    instance_count=1,
                                    instance_type='ml.m4.xlarge',
                                    output_path='s3://{}/{}/output'.format(bucket, prefix),
                                    sagemaker_session=sess)
xgb.set_hyperparameters(max_depth=5,
                        eta=0.2,
                        gamma=4,
                        min_child_weight=6,
                        subsample=0.8,
                        silent=0,
                        objective='binary:logistic',
                        num_round=100)

xgb.fit({'train': s3_input_train, 'validation': s3_input_validation})

# ---
#
# ## Hosting
# Now that you've trained the `xgboost` algorithm on the data, deploy a model that's hosted behind a real-time endpoint.
#
# First, you need to determine how to pass data into and receive data from the endpoint. The data is currently stored as NumPy arrays in the memory of your notebook instance. To send it in an HTTP POST request, you'll serialize it as a CSV string and then decode the resulting CSV.
#
# **Note:** For inference with the CSV format, Amazon SageMaker XGBoost requires that the data does NOT include the target variable.

# In[18]:

xgb_predictor = xgb.deploy(initial_instance_count=1,
                           serializer=CSVSerializer(),
                           instance_type='ml.m4.xlarge')

# ---
#
# ## Evaluation
# There are many ways to compare the performance of a machine learning model, but start by simply comparing actual to predicted values. In this case, you're simply predicting whether the customer subscribed to a term deposit (`1`) or not (`0`), which produces a simple confusion matrix.
#

# Now, use a simple function to:
#
# 1. Loop over the test dataset
# 1. Split it into mini-batches of rows
# 1. Convert those mini-batches to CSV string payloads (notice that the target variable is dropped from the dataset first)
# 1. Retrieve mini-batch predictions by invoking the XGBoost endpoint
# 1. Collect predictions and convert from the CSV output that the model provides into a NumPy array

# In[19]:

def predict(data, rows=500):
    split_array = np.array_split(data, int(data.shape[0] / float(rows) + 1))
    predictions = ''
    for array in split_array:
        predictions = ','.join([predictions, xgb_predictor.predict(array).decode('utf-8')])

    return np.fromstring(predictions[1:], sep=',')

predictions = predict(test_data.drop(['y_no', 'y_yes'], axis=1).values)

# Now, check the confusion matrix to see how well the model predicted vs. actuals.

# In[20]:

pd.crosstab(index=test_data['y_yes'], columns=np.round(predictions), rownames=['actuals'], colnames=['predictions'])

# Of about 4,000 potential customers, the model predicted that 136 would subscribe, and 94 actually did. There were also 389 customers who subscribed that were not predicted to subscribe. This is less than desirable, but the model can (and should) be tuned to improve this. Most importantly, note that with minimal effort, the model produced accuracies similar to those published [here](http://media.salford-systems.com/video/tutorial/2015/targeted_marketing.pdf).
#
# **Note:** Because there is some element of randomness in the algorithm's subsample, your results may differ slightly from the text written above.

# ### Cleanup
#
# If you are finished with this notebook, run the following cell. This removes the hosted endpoint you created and avoids any charges from a stray instance being left on.

# In[21]:

sagemaker.Session().delete_endpoint(xgb_predictor.endpoint_name)

# In[ ]:


import requests
from bs4 import BeautifulSoup
import csv

attri=[ 'Name', 'URL', 'Author', 'Price' , 'Number of Ratings' , 'Average Rating']

with open('in_book.csv','a') as fil:
          wriat= csv.writer(fil)
          wriat.writerow(attri)
          attri.clear()

for pg in range(1,5):
     infor=requests.get("https://www.amazon.in/gp/bestsellers/books/ref=zg_bs_pg_1?ie=UTF8&pg="+str(pg))
     data=infor.text
     soup=BeautifulSoup(data,'lxml')
     mov=soup.find_all('div',class_='zg_itemImmersion')

     for bst in range(20):
             try:
                   name=mov[bst].find_all('a',class_='a-link-normal')
                   attri.append(name[0].text)
             except:
                   attri.append("NOT AVAILABLE")
             try:
                   url=mov[bst].find_all('a',class_='a-section a-spacing-none p13n-asin')
                   attri.append(url[0].a['href'])
             except:
                  attri.append("NOT AVAILABLE")

             try:
                   auth=mov[bst].find_all('a',class_='a-row a-size-small')
                   attri.append(auth[0].text)
             except:
                  attri.append("NOT AVAILABLE")

             try:
                   ratnum=mov[bst].find_all('a',class_='a-link-normal a-text-normal')
                   attri.append("Rs"+str(name[0].text))
             except:
                  attri.append("NOT AVAILABLE")

             try:
                   avgrat=mov[bst].find_all('a',class_='a-size-small a-link-normal')
                   attri.append(avgrat[0].text)
             except:
                  attri.append("NOT AVAILABLE")

             try:
                   name=mov[bst].find_all('a',class_='a-link-normal')
                   attri.append(name[0].a['title'])
             except:
                  attri.append("NOT AVAILABLE")

             with open('in_book.csv','a') as fil:
                       wriat= csv.writer(fil)
                       wriat.writerow(attri)
                       attri.clear()


# -*- coding: utf-8 -*-
"""
Created on Mon Feb  4 17:49:56 2019

@author: tiwarisudhir
"""

import gym
import torch
import numpy as np
from collections import deque
from unityagents import UnityEnvironment
import qnetwork

# set device placement

SEED            = 0
MEM_BUFFER      = int(1e5)
MEM_BATCH       = 64
GAMMA           = 0.99
TAU             = 1e-3
LEARNING_RATE   = 5e-4
UPDATE_FREQ     = 4
CHECKPOINT_NAME = 'checkpoint.pth'
MIN_THRESHOLD   = 14.
device          = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

def get_environment():
    '''
    Get gym environment and brain for the navigator
    '''

    env = UnityEnvironment(file_name="/data/Banana_Linux_NoVis/Banana.x86_64")

    return env

def get_agent(env):
    '''
    Get DQN agent and environment brain
    '''

    brain_name = env.brain_names[0]
    brain = env.brains[brain_name]
    env_info = env.reset(train_mode=True)[brain_name]

    states = len(env_info.vector_observations[0])
    actions = brain.vector_action_space_size

    agent = qnetwork.DQNAgent(states, actions, seed = SEED, device = device,
                              mem_buffer = MEM_BUFFER, mem_batch = MEM_BATCH,
                              gamma = GAMMA, tau = TAU,
                              learning_rate = LEARNING_RATE, update_freq = UPDATE_FREQ)

    return agent

def train_navigator(env, agent, episodes = 2000, max_steps = None, epsilon_max = 1., epsilon_min = 0.01, epsilon_decay = 0.997):
    '''
    Train a DQN for navigator to collect Bananas
    param episodes: number of episodes to train for
    param max_steps: max number of time steps per episode
    param epsilon_max: starting epsilon value for greedy policy selection
    param epsilon_end: min value of the epsilon for greedy policy selection
    param epsilon_decay: decay rate of epsilon per episode
    '''

    # monitor learning by keeping track of rewards
    epsilon             = epsilon_max
    best_avg_rewards    = 0.
    episode_rewards     = []
    window_rewards      = deque(maxlen = 100)

    for episode in range(episodes):

        step       = 0
        rewards    = 0.
        brain_name = env.brain_names[0]
        env_info   = env.reset(train_mode = True)[brain_name]
        state      = env_info.vector_observations[0]

        while (max_steps == None) or (step < max_steps):

            # get agent's action
            action = agent.select_action(state, epsilon)

            # take action that agent has selected
            env_info = env.step(action)[brain_name]

            # get the transitioned state and reward
            new_state = env_info.vector_observations[0]
            reward   = env_info.rewards[0]
            done     = env_info.local_done[0]

            # pass the reward to agent
            agent.experience(state, action, reward, new_state, done)

            # transition to new state
            state = new_state
            rewards += reward

            if done:
                break

            step += 1

        # decay epsilon
        epsilon = max(epsilon_min, epsilon * epsilon_decay)

        # record rewards
        episode_rewards.append(rewards)
        window_rewards.append(rewards)

        print('\r Episode-{}, Eps: {:.4f}, Avg Reward: {:.2f}'.format((episode + 1), epsilon, np.mean(window_rewards)), end = "")

        if (episode + 1) % 100 == 0:
            print('\r Episode-{}, Avg Reward: {:.2f}'.format((episode + 1), np.mean(window_rewards)))

        if (np.mean(window_rewards) >= MIN_THRESHOLD) and (np.mean(window_rewards) > best_avg_rewards):

            best_avg_rewards = np.mean(window_rewards)

            print('\n Agent has learned to solve the task. Episode-{}, Avg Reward: {:.2f}'.format((episode + 1), best_avg_rewards))

            agent.save_memory(checkpoint_name = CHECKPOINT_NAME)

    return episode_rewards

def restore_memory(agent):
    '''
    Restore agent's past learning
    '''

    agent.restore_memory(checkpoint_name = CHECKPOINT_NAME)


#!/usr/bin/env python
import numpy as np
import util
import matplotlib.pyplot as plt
from sklearn import preprocessing, decomposition, manifold
from sklearn.model_selection import GridSearchCV, cross_val_score
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

# A one-word-descriptor for the experiment
SUBMISSION_FILE_SUFFIX = ''

# Data hyper-parameters
TRAIN_COUNT = 278
TEST_COUNT = 138

# Alredy preprocessed data. Data files must reside under DATA_PATH (see util.py)
# If it is False, the raw data is loaded and preprocessed from scratch.
USE_PREPROCESSED_DATA = ('train_full_reduced.npy', 'test_full_reduced.npy')

# If USE_PREPROCESSED_DATA = False, then raw data of the provided z-planes
# will be loaded and preprocessed.
PLANES_Z = [20, 35, 56, 70, 87, 97, 107, 117, 127, 137]
#PLANES_Z = list(range(20,170)) # Load all planes.

# Regression hyper-parameters
REDUCED_DIM = 50
MIX_TRAIN_TEST_FOR_REDUCTION = True
ESTIMATOR_POOL = {
    'svm_rbf': True,
    'huber': False,
    'lasso': False,
    'decision_tree': False,
    'random_forest': False,
    'adaboost': True,
    'gradientBR': True, # GradientBoostingRegressor
}
USE_FEATURE_POOL = True
FEATURE_POOL = { # Specifies corresponding dimensionalites. Active if it is larger than 0.
    'tsne': 10,
    'lda': 10,
    'pca': 30,
    'mds': 0
}
PRINT_ESTIMATOR_RESULTS = False # If True, prints results for all possible configurations.
N_JOBS = 6 # Num of CPU cores for parallel processing.

def load_10_planes():
    training_planes = []
    for i in range(len(PLANES_Z)):
        z = PLANES_Z[i]
        print('Loading training plane %d/%d (z = %d)' % (i + 1, len(PLANES_Z), z))
        training_planes.append(util.nonzero_rv(util.load_all_z_planes(z)))

    test_planes = []
    for i in range(len(PLANES_Z)):
        z = PLANES_Z[i]
        print('Loading test plane %d/%d (z = %d)' % (i + 1, len(PLANES_Z), z))
        test_planes.append(util.nonzero_rv(util.load_all_z_planes(z, False)))

    return (np.concatenate(training_planes, axis=0), np.concatenate(test_planes, axis=0))

def main():
    #########
    # Prepare feature matrices.
    #########
    normalizer = preprocessing.StandardScaler()
    training_targets = util.load_refs() # Load targets (i.e. ages)
    # Load train & test data.
    if not USE_PREPROCESSED_DATA:
        print('Loading and preprocessing raw data.')
        training_planes, test_planes = load_10_planes()
        training_planes = training_planes.T
        test_planes = test_planes.T
        all_planes = np.vstack((training_planes, test_planes))
        if MIX_TRAIN_TEST_FOR_REDUCTION == True:
            print('Using both training and test data for reduction.')
            all_planes = normalizer.fit_transform(all_planes)
            # Apply data reduction on both train and test data.
            # Note that this is not the best practice.
            #all_planes = decomposition.PCA(n_components=REDUCED_DIM, whiten=False).fit_transform(all_planes)
            all_planes = decomposition.KernelPCA(n_components=REDUCED_DIM, kernel='rbf').fit_transform(all_planes)
            training_planes = all_planes[:TRAIN_COUNT,:]
            test_planes = all_planes[TRAIN_COUNT:,:]
        else:
            print('Using only training data for reduction.')
            # Normalize data.
            training_planes = normalizer.fit_transform(training_planes)
            test_planes = normalizer.transform(test_planes)
            svd_decomp = decomposition.PCA(n_components=REDUCED_DIM, whiten=False)
            training_planes = svd_decomp.fit_transform(training_planes)
            test_planes = svd_decomp.transform(test_planes)
            all_planes = np.vstack((training_planes, test_planes))
    else:
        print('Using preprocessed-data.')
        training_planes, test_planes = util.load_preprocessed_data(USE_PREPROCESSED_DATA)
        training_planes = training_planes.T
        test_planes = test_planes.T
        # Normalize data.
        if MIX_TRAIN_TEST_FOR_REDUCTION == True:
            all_planes = np.vstack((training_planes, test_planes))
            all_planes = normalizer.fit_transform(all_planes)
            training_planes = all_planes[:TRAIN_COUNT,:]
            test_planes = all_planes[TRAIN_COUNT:,:]
        else:
            training_planes = normalizer.fit_transform(training_planes.T)
            test_planes = normalizer.transform(test_planes.T)
            all_planes = np.vstack((training_planes, test_planes))
    print('Data dimensionality ' + str(all_planes.shape))

    # Extract Features
    if USE_FEATURE_POOL == True:
        tsne_features = np.zeros((TRAIN_COUNT+TEST_COUNT, 0))
        lda_features = np.zeros((TRAIN_COUNT+TEST_COUNT, 0))
        pca_features = np.zeros((TRAIN_COUNT+TEST_COUNT, 0))
        mds_features = np.zeros((TRAIN_COUNT+TEST_COUNT, 0))
        if FEATURE_POOL['tsne'] > 0:
            print('Running t-SNE [%d]' % FEATURE_POOL['tsne'])
            tsne_features = manifold.TSNE(n_components=FEATURE_POOL['tsne'], init='pca', random_state=0).fit_transform(all_planes)
        if FEATURE_POOL['lda'] > 0:
            print('Running LDA [%d]' % FEATURE_POOL['lda'])
            lda = LinearDiscriminantAnalysis(n_components=FEATURE_POOL['lda'])
            lda_features_training = lda.fit_transform(training_planes, training_targets) #Supervised
            lda_features_test = lda.transform(test_planes)
            lda_features = np.vstack((lda_features_training, lda_features_test))
        if FEATURE_POOL['pca'] > 0:
            if REDUCED_DIM > FEATURE_POOL['pca']:
                print('Running PCA [%d]' % FEATURE_POOL['pca'])
                pca_features = decomposition.PCA(n_components=FEATURE_POOL['pca'], whiten=False).fit_transform(all_planes)
            else:
                print('Using old PCA [%d]' % FEATURE_POOL['pca'])
                pca_features = all_planes # Use already reduced data.
        if FEATURE_POOL['mds'] > 0:
            print('Running MDS [%d]' % FEATURE_POOL['mds'])
            mds_features = manifold.MDS(n_components=FEATURE_POOL['mds'], n_init=1, max_iter=1000).fit_transform(all_planes)

        # Construct feature matrix by concatenation
        feature_matrix = np.hstack((tsne_features, lda_features, pca_features, mds_features))
        training_feature_matrix = all_planes[:TRAIN_COUNT,:]
        test_feature_matrix = all_planes[TRAIN_COUNT:,:]
    else:
        feature_matrix = all_planes
        training_feature_matrix = training_planes
        test_feature_matrix = test_planes

    print('Feature matrix dimensionality ' + str(feature_matrix.shape))
    #########
    # Grid Search and Regression
    #########
    # SVM (RBF Kernel)
    if ESTIMATOR_POOL['svm_rbf'] == True:
        print('Doing SVM (RBF) Regression...')
        from sklearn.svm import SVR
        # Parameter grid
        Cs = np.logspace(-4, 6, 20)
        gammas = np.logspace(-8, 1, 20)
        svr_rbf = SVR(kernel='rbf')
        clf = GridSearchCV(estimator=svr_rbf, param_grid=dict(C=Cs, gamma=gammas), n_jobs=N_JOBS, cv=15, scoring='neg_mean_squared_error')
        clf.fit(training_feature_matrix, training_targets)
        if PRINT_ESTIMATOR_RESULTS == True:
            for params, mean_score, scores in clf.grid_scores_:
                print("%0.3f (+/-%0.03f) for %r" % (mean_score, scores.std() * 2, params))
        print("SVM(RBF) [%0.2f] - The best parameters are %s" % (-clf.best_score_, clf.best_params_))
        predicted_labels = clf.predict(test_feature_matrix)
        util.create_submission_file(predicted_labels, 'submission_rbf_svm_%s.csv' % (SUBMISSION_FILE_SUFFIX))
    # Huber Regression
    if ESTIMATOR_POOL['huber'] == True:
        print('Doing Huber Regression...')
        from sklearn.linear_model import HuberRegressor
        alphas = np.linspace(0.1, 5, 10)
        epsilons = np.linspace(1, 10, 10)
        huber = HuberRegressor(fit_intercept=True, max_iter=500)
        clf = GridSearchCV(estimator=huber, param_grid=dict(alpha=alphas, epsilon=epsilons), n_jobs=N_JOBS, cv=10, scoring='neg_mean_squared_error')
        clf.fit(training_feature_matrix, training_targets)
        if PRINT_ESTIMATOR_RESULTS == True:
            for params, mean_score, scores in clf.grid_scores_:
                print("%0.3f (+/-%0.03f) for %r" % (mean_score, scores.std() * 2, params))
        print("Huber Regression [%0.2f] - The best parameters are %s" % (-clf.best_score_, clf.best_params_))
        predicted_labels = clf.predict(test_feature_matrix)
        util.create_submission_file(predicted_labels, 'submission_huber_%s.csv' % (SUBMISSION_FILE_SUFFIX))
    # LASSO
    if ESTIMATOR_POOL['lasso'] == True:
        print('Doing LASSO Regression...')
        from sklearn.linear_model import Lasso
        alphas = np.linspace(0.1, 6, 20)
        lasso = Lasso(alpha=0.1, copy_X=True, fit_intercept=False, max_iter=1000)
        clf = GridSearchCV(estimator=lasso, param_grid=dict(alpha=alphas), n_jobs=N_JOBS, cv=10, scoring='neg_mean_squared_error')
        clf.fit(training_feature_matrix, training_targets)
        if PRINT_ESTIMATOR_RESULTS == True:
            for params, mean_score, scores in clf.grid_scores_:
                print("%0.3f (+/-%0.03f) for %r" % (mean_score, scores.std() * 2, params))
        print("LASSO Regression [%0.2f] - The best parameters are %s" % (-clf.best_score_, clf.best_params_))
        predicted_labels = clf.predict(test_feature_matrix)
        util.create_submission_file(predicted_labels, 'submission_lasso_%s.csv' % (SUBMISSION_FILE_SUFFIX))
    # Decision Tree
    if ESTIMATOR_POOL['decision_tree'] == True:
        print('Doing Decision Tree Regression...')
        # Use random search instead of grid search.
        from scipy.stats import randint as sp_randint
        from sklearn.tree import DecisionTreeRegressor
        from sklearn.grid_search import RandomizedSearchCV
        decision_tree = DecisionTreeRegressor(random_state=0)
        param_dist = {"max_depth": [3, 5, None],
              "max_features": sp_randint(1, 20),
              "min_samples_split": sp_randint(1, 20),
              "min_samples_leaf": sp_randint(1, 20)
              }
        n_iter_search = 500
        clf = RandomizedSearchCV(decision_tree, param_distributions=param_dist, n_iter=n_iter_search, n_jobs=N_JOBS, cv=10, scoring='neg_mean_squared_error')
        #clf = GridSearchCV(estimator=decision_tree, param_grid=dict(), n_jobs=-1, cv=5, scoring='neg_mean_squared_error')
        clf.fit(training_feature_matrix, training_targets)
        if PRINT_ESTIMATOR_RESULTS == True:
            for params, mean_score, scores in clf.grid_scores_:
                print("%0.3f (+/-%0.03f) for %r" % (mean_score, scores.std() * 2, params))
        print("Decision Tree Regression [%0.2f] - The best parameters are %s" % (-clf.best_score_, clf.best_params_))
        predicted_labels = clf.predict(test_feature_matrix)
        util.create_submission_file(predicted_labels, 'submission_decision_tree_%s.csv' % (SUBMISSION_FILE_SUFFIX))
    # Random Forest
    if ESTIMATOR_POOL['random_forest'] == True:
        print('Doing Random Forest Regression...')
        # Use random search instead of grid search.
        from scipy.stats import randint as sp_randint
        from sklearn.ensemble import RandomForestRegressor
        from sklearn.grid_search import RandomizedSearchCV
        forest = RandomForestRegressor(random_state=0)
        param_dist = {"max_depth": [3, 5, None],
              "max_features": sp_randint(1, 20),
              "min_samples_split": sp_randint(1, 20),
              "min_samples_leaf": sp_randint(1, 20),
              'n_estimators': [10, 25, 50, 100],
              'bootstrap': [True, False],
              }
        n_iter_search = 200
        clf = RandomizedSearchCV(forest, param_distributions=param_dist, n_iter=n_iter_search, n_jobs=N_JOBS, cv=10, scoring='neg_mean_squared_error')
        #clf = GridSearchCV(estimator=decision_tree, param_grid=dict(), n_jobs=-1, cv=5, scoring='neg_mean_squared_error')
        clf.fit(training_feature_matrix, training_targets)
        if PRINT_ESTIMATOR_RESULTS == True:
            for params, mean_score, scores in clf.grid_scores_:
                print("%0.3f (+/-%0.03f) for %r" % (mean_score, scores.std() * 2, params))
        print("Random Forest Regression [%0.2f] - The best parameters are %s" % (-clf.best_score_, clf.best_params_))
        predicted_labels = clf.predict(test_feature_matrix)
        util.create_submission_file(predicted_labels, 'submission_random_forest_%s.csv' % (SUBMISSION_FILE_SUFFIX))
    # AdaBoost
    if ESTIMATOR_POOL['adaboost'] == True:
        print('Doing AdaBoost Regression...')
        from sklearn.ensemble import AdaBoostRegressor
        adaboost = AdaBoostRegressor(loss='square', random_state=0)
        param_dist = {
              "learning_rate": [1e-4, 1e-3, 1e-2, 1e-1, 1, 2],
              'n_estimators': [10, 25, 50, 100, 200],
              }
        clf = GridSearchCV(estimator=adaboost, param_grid=param_dist, n_jobs=N_JOBS, cv=10, scoring='neg_mean_squared_error')
        clf.fit(training_feature_matrix, training_targets)
        if PRINT_ESTIMATOR_RESULTS == True:
            for params, mean_score, scores in clf.grid_scores_:
                print("%0.3f (+/-%0.03f) for %r" % (mean_score, scores.std() * 2, params))
        print("AdaBoost Regression [%0.2f] - The best parameters are %s" % (-clf.best_score_, clf.best_params_))
        predicted_labels = clf.predict(test_feature_matrix)
        util.create_submission_file(predicted_labels, 'submission_adaboost_%s.csv' % (SUBMISSION_FILE_SUFFIX))
    # GradientBoostingRegressor
    if ESTIMATOR_POOL['gradientBR'] == True:
        from sklearn.ensemble import GradientBoostingRegressor
        param_dist = {
            'loss' : ['ls', 'lad', 'huber'],
            'n_estimators': [20, 100, 200],
            'learning_rate': [1e-3, 1e-1, 1],
            'max_depth': [3, 6],
            'min_samples_split': [2, 5],
            }
        gbr = GradientBoostingRegressor(random_state=0)
        clf = GridSearchCV(estimator=gbr, param_grid=param_dist, n_jobs=N_JOBS, cv=6, scoring='neg_mean_squared_error')
        clf.fit(training_feature_matrix, training_targets)
        if PRINT_ESTIMATOR_RESULTS == True:
            for params, mean_score, scores in clf.grid_scores_:
                print("%0.3f (+/-%0.03f) for %r" % (mean_score, scores.std() * 2, params))
        print("GradientBoostingRegressor Regression [%0.2f] - The best parameters are %s" % (-clf.best_score_, clf.best_params_))
        predicted_labels = clf.predict(test_feature_matrix)
        util.create_submission_file(predicted_labels, 'submission_gbr_%s.csv' % (SUBMISSION_FILE_SUFFIX))

if __name__ == '__main__':
    main()


from django.urls import path
from . import views

urlpatterns = [
    path('',views.ArtilceListView.as_view(),name='home'),
    path(
        'articles/<int:pk>',
        views.ArticleDetailView.as_view(),
        name='article_detail'
    ),
    path(
        'articles/profile',
        views.ArtilceProfileListView.as_view(),
        name='profile_articles'
    ),
    path(
        'articles/create',
        views.ArticleCreateView.as_view(),
        name='article_create'
        ),
    path(
        'articles/<int:pk>/delete',
        views.ArticleDeleteView.as_view(),
        name='article_delete'
    ),
    path(
        'articles/<int:pk>/update',
        views.ArticleUpdateView.as_view(),
        name='article_update'
    ),
    path(
        'comment/add',
        views.CommentCreateView.as_view(),
        name='comment_create'
    ),
]


# This file can be automatically generaeted using the following command
# m4 ../python.m4 ../xam/vector_double_xam.m4 > vector_double_xam.py
# -----------------------------------------------------------------------------
#         cppad_swig: A C++ Object Library and SWIG Interface to CppAD
#          Copyright (C) 2017-17 Bradley M. Bell (bradbell@seanet.com)
#              This program is distributed under the terms of the
#          GNU Affero General Public License version 3.0 or later see
#                     http://www.gnu.org/licenses/agpl.txt
# -----------------------------------------------------------------------------
# std::vector<double>
# -----------------------------------------------------------------------------
def vector_double_xam() :
	#
	# load the cppad swig library
	import py_cppad
	#
	# initialize return variable
	ok = True
	n = 4
	vec = py_cppad.vector_double(n)
	#
	# check size
	ok = ok and vec.size() == n
	#
	# setting elements
	for i in range( n  ) :
		vec[i] = 2.0 * i
	#
	# getting elements
	for i in range( n  ) :
		element = vec[i]
		ok = ok and element == 2.0 * i
	#
	return( ok )
#