#NUMPY BASICS
#You've contacted FIFA for some data and they handed you two lists.
#The lists are the following:
#positions = ['GK', 'M', 'A', 'D', ...]
#heights = [191, 184, 185, 180, ...]
#Each element in the lists corresponds to a player.
#The first list, positions, contains strings representing each player's position.
#The possible positions are: 'GK' (goalkeeper), 'M' (midfield), 'A' (attack) and 'D' (defense).
#The second list, heights, contains integers representing the height of the player in cm.
#The first player in the lists is a goalkeeper and is pretty tall (191 cm).

# heights and positions are available as lists

# Import numpy
import numpy as np

# Convert positions and heights to numpy arrays: np_positions, np_heights
np_heights=np.array(heights)
np_positions=np.array(positions)

# Heights of the goalkeepers: gk_heights
gk_heights=np_heights[np_positions=='GK']

# Heights of the other players: other_heights
other_heights=np_heights[np_positions!='GK']

# Print out the median height of goalkeepers. Replace 'None'
med1=np.median(gk_heights)
print("Median height of goalkeepers: " + str(med1))

# Print out the median height of other players. Replace 'None'
med2=np.median(other_heights)
print("Median height of other players: " + str(med2))


targetOP = op('record_and_playback')
parent().copy(targetOP);
path = ui.chooseFile(load=False,title="save the tox")
if path:
	listOfExternalOperators= targetOP.findChildren(parName='externaltox',onlyNonDefaults=True)
	print(path+".tox")
	#externals= {} #dict to remember our external paths
	for c in listOfExternalOperators:
		#externals[c.id] = c.par.externaltox
		c.par.externaltox = ''
	targetOP.par.externaltox =''

	listOfExternalTextFiles= targetOP.findChildren(parName='file', type=DAT)
	for txtop in listOfExternalTextFiles:
		print(txtop)
		txtop.par.file = ''
		#txtop.par.loadonstart = 0

	f = targetOP.save(path+".tox")


'''
Test that each of the expected code paths functions without error. This will get
more complicated as time goes on. For tests about response time/latency
performance see test/test_app_sla.
'''
import json
import os
import urllib

try:
    from urllib.parse import quote
except ImportError:
    from urllib import quote

from app import server as flask_app
# from Insight.service import server as flask_service
from nose.tools import ok_, assert_equal
from nose.tools import timed

app_client = None
service_client = None

def setup_module():
    global flask_app
    flask_app.config['TESTING'] = True
    if 'STATUS' not in os.environ:
        os.environ['STATUS'] = 'TESTING'
    global app_client
    app_client = flask_app.test_client()
    app_client.set_cookie('127.0.0.1', 'user_id', str(42))

def teardown_module():
    pass

def test_click_fail():
    # trying to get the /click endpoint should fail (endpoint not found)
    res = app_client.get('/click')
    if (not res.status_code == 404):
        print(str(res.data))
        print(str(res.status_code))
        print(str(res.headers.items()))
    assert_equal(404, res.status_code)

def test_load_success():
    data = {}
    data['page'] = '/'
    data['screen_w'] = 120
    data['screen_h'] = 120
    data['window_x'] = 0
    data['window_y'] = 0
    data['window_w'] = 120
    data['window_h'] = 120
    json_data = json.dumps(data)
    res = app_client.get('/click/l?d='+quote(str(json_data)))
    if (not res.status_code == 200):
        print(str(res.data))
        print(str(res.status_code))
        print(str(list(res.headers.items())))
    assert_equal(200, res.status_code)

def test_load_fail():
    data = {}
    data['key'] = 'value'
    json_data = json.dumps(data)
    res = app_client.get('/click/l?d='+quote(str(json_data)))
    if (not res.status_code == 400):
        print(str(res.data))
        print(str(res.status_code))
        print(str(list(res.headers.items())))
    assert_equal(400, res.status_code)

def test_mouse_success():
    data = {}
    data['page'] = '/'
    data['scroll_x'] = 0
    data['scroll_y'] = 120
    data['mouse_x'] = 0
    data['mouse_y'] = 120
    data['window_x'] = 0
    data['window_y'] = 0
    data['window_w'] = 120
    data['window_h'] = 120
    data['time'] = 9001
    json_data = json.dumps(data)
    res = app_client.get('/click/m?d='+quote(str(json_data)))
    if (not res.status_code == 200):
        print(str(res.data))
        print(str(res.status_code))
        print(str(list(res.headers.items())))
    assert_equal(200, res.status_code)

def test_mouse_fail():
    data = {}
    data['key'] = 'value'
    json_data = json.dumps(data)
    res = app_client.get('/click/m?d='+quote(str(json_data)))
    if (not res.status_code == 400):
        print(str(res.data))
        print(str(res.status_code))
        print(str(list(res.headers.items())))
    assert_equal(400, res.status_code)


#!/usr/bin/python3
from djitellopy import TelloSwarm

def call_swarm():
	print("assembling swarm")
	#absolute path m9
	dronelist='/opt/dronenetlist'
	fo=open(dronelist, "r")
	if(fo.read(10)==None):
		print("drone list not found or null")
		fo.close()
		exit()
	else:
		fo.close()
		assembly = TelloSwarm.fromFile(dronelist)
		return assembly

def connect_and_run(swarm):
	swarm.connect()
	print("performing health check routine")
	for drone in swarm:
		print("IP: " + str(drone) + " Battery: " + str(drone.get_battery()))
	swarm.end()

if __name__ == "__main__":
	connect_and_run(call_swarm())


from System.Drawing import Color

class ForexBollingerBandBot(QCAlgorithm):

    def Initialize(self):
        self.SetStartDate(2015, 1, 1)
        self.SetEndDate(2021, 1, 1)
        self.SetCash(100000)
        self.pair = self.AddForex("EURUSD", Resolution.Daily, Market.Oanda).Symbol
        self.bb = self.BB(self.pair, 20, 2)

        stockPlot = Chart('Trade Plot')
        stockPlot.AddSeries(Series('Buy', SeriesType.Scatter, '$',
                            Color.Green, ScatterMarkerSymbol.Triangle))
        stockPlot.AddSeries(Series('Sell', SeriesType.Scatter, '$',
                            Color.Red, ScatterMarkerSymbol.TriangleDown))
        stockPlot.AddSeries(Series('Liquidate', SeriesType.Scatter, '$',
                            Color.Blue, ScatterMarkerSymbol.Diamond))
        self.AddChart(stockPlot)

    def OnData(self, data):
        if not self.bb.IsReady:
            return

        price = data[self.pair].Price

        self.Plot("Trade Plot", "Price", price)
        self.Plot("Trade Plot", "MiddleBand", self.bb.MiddleBand.Current.Value)
        self.Plot("Trade Plot", "UpperBand", self.bb.UpperBand.Current.Value)
        self.Plot("Trade Plot", "LowerBand", self.bb.LowerBand.Current.Value)

        if not self.Portfolio.Invested:
            if self.bb.LowerBand.Current.Value > price:
                self.SetHoldings(self.pair, 1)
                self.Plot("Trade Plot", "Buy", price)
            elif self.bb.UpperBand.Current.Value < price:
                self.SetHoldings(self.pair, -1)
                self.Plot("Trade Plot", "Sell", price)
        else:
            if self.Portfolio[self.pair].IsLong:
                if self.bb.MiddleBand.Current.Value < price:
                    self.Liquidate()
                    self.Plot("Trade Plot", "Liquidate", price)
            elif self.bb.MiddleBand.Current.Value > price:
                self.Liquidate()
                self.Plot("Trade Plot", "Liquidate", price)


from bnnbench.models import BaseModel, MCDropout, MCBatchNorm, DeepEnsemble, DNGO
from bnnbench.utils.data_utils import HPOBenchData
import logging
from enum import IntEnum

from bnnbench.emukit_interfaces.models import PyBNNModel
import numpy as np
from emukit.core.loop.model_updaters import NoopModelUpdater
from emukit.core.loop.candidate_point_calculators import RandomSampling
from emukit.core.loop import OuterLoop
from emukit.bayesian_optimization.loops.bayesian_optimization_loop import BayesianOptimizationLoop
from emukit.core import ParameterSpace
from emukit.core.loop.loop_state import LoopState, create_loop_state, UserFunctionResult
from emukit.examples.gp_bayesian_optimization.single_objective_bayesian_optimization import GPBayesianOptimization
from emukit.examples.gp_bayesian_optimization.enums import AcquisitionType
from typing import Sequence, Union, Tuple, Callable, Dict

_log = logging.getLogger(__name__)

class ModelType(IntEnum):
    MCDROPOUT = 0
    MCBATCHNORM = 1
    DNGO = 2
    ENSEMBLE = 3

model_classes = [MCDropout, MCBatchNorm, DNGO, DeepEnsemble]

def create_random_search_loop(space: ParameterSpace, initial_state: LoopState, **kwargs) -> OuterLoop:
    _log.debug("Generating new random search loop.")

    return OuterLoop(
        candidate_point_calculator=RandomSampling(parameter_space=space),
        model_updaters=NoopModelUpdater(),
        loop_state=initial_state
    )

def create_pybnn_bo_loop(model_type: ModelType, model_params: dict, space: ParameterSpace, initial_state: LoopState,
                         rng: Union[None, int, np.random.RandomState] = None, optimize_hypers_only_once: bool = False,
                         **kwargs) -> BayesianOptimizationLoop:

    _log.debug("Creating Bayesian Optimization Loop for PyBNN model of type %s, parameter space %s, and an initial "
               "loop state containing %d points." % (model_type, space, initial_state.X.shape[0]))
    model_type = model_classes[model_type]
    model_params = model_type.modelParamsContainer()._replace(**model_params, rng=rng)
    pybnn_model = PyBNNModel(model=model_type, model_params=model_params)
    pybnn_model.set_data(initial_state.X, initial_state.Y)
    if optimize_hypers_only_once:
        # Disable internal hyper-parameter optimization for all future calls to "fit()" for this model and store the
        # optimized model parameters for re-use in all upcoming calls to "fit()" on this object's model.
        pybnn_model.model_params = pybnn_model.model.model_params._replace(optimize_hypers=False)

    boloop = BayesianOptimizationLoop(space=space, model=pybnn_model)
    boloop.loop_state = LoopState(initial_results=initial_state.results[:])
    _log.info("BOLoop for PyBNN model of type %s initialized." % str(model_type).split('.')[-1])
    return boloop

def create_gp_bo_loop(space: ParameterSpace, initial_state: LoopState, acquisition_type: AcquisitionType,
                      **kwargs) -> BayesianOptimizationLoop:
    """ Creates a Bayesian Optimization Loop using a GP model. The keyword arguments are passed as is to
    emukit.examples.gp_bayesian_optimization.single_objective_bayesian_optimization.GPBayesianOptimization. """
    _log.debug("Generating new GP BO Loop with %s acquisition function." % str(acquisition_type))
    kwargs.pop("rng") # This isn't compatible
    loop = GPBayesianOptimization(variables_list=space.parameters, X=initial_state.X, Y=initial_state.Y,
                                  acquisition_type=acquisition_type, **kwargs)
    loop.loop_state = LoopState(initial_results=initial_state.results[:])
    return loop

class LoopGenerator:
    """ Generates a new Loop instance for use with Benchmarker by iterating over an internal data generator. Useful to
    substitute for a missing initialization hook in Benchmarker. """

    def __init__(self, loops: Sequence[Tuple[str, Callable, Dict]], data: HPOBenchData, seed: int):
        """
        :param loops: Array of 3-tuples (name, func, kwargs)
            Each tuple corresponds to the requirements for initializing a different Loop object. 'name' is a string,
            'func' is a function that returns the relevant loop object and should accept a keyword argument
            'initial_state' for a LoopState object used to initialize the Loop, and kwargs is a dictionary of fixed
            keyword arguments passed to func.
        :param data: HPOBenchData
            A data holder object that will be used to coordinate the current training/test splits.
        :param seed: int
            An integer value used to re-seed the global numpy RNG before generating every new Loop.
        """

        self._loops = loops
        self.data = data
        self.seed = seed

        def loop_cycle():
            nonlocal self
            while True:
                _log.debug("Starting new iteration of loop initializers.")
                self.data.update()
                for loop in self._loops:
                    yield loop

        self._loop_cycle = loop_cycle() # Actually responsible for properly iterating over the loops
        self.loop_list = [(l[0], self.generate_next_loop) for l in self._loops] # Sole Purpose: Expose to Benchmarker
        _log.info("Initialized LoopGenerator object for %d loops: %s" %
                  (len(self._loops), ", ".join(l[0] for l in self._loops)))

    def generate_next_loop(self, benchmarker_state: LoopState):
        """ Expected to be passed to the initializer of Benchmarker, will iteratively generate up-to-date Loop
        objects. """

        np.random.seed(self.seed)
        loop_name, loop_init_func, loop_kwargs = next(self._loop_cycle)
        _log.debug("Generating loop %s" % loop_name)
        init_state = self._create_initial_loop_state(benchmarker_loop_state=benchmarker_state)
        loop = loop_init_func(initial_state=init_state, **loop_kwargs, rng=np.random.randint(0, 1_000_000_000))
        return loop

    def _create_initial_loop_state(self, benchmarker_loop_state: LoopState):
        # Assume that the data object will handle generation of appropriate splits
        train_X, train_Y, train_meta = self.data.train_X, self.data.train_Y, self.data.train_meta
        meta_headers = self.data.meta_headers
        benchmarker_metadata = benchmarker_loop_state.results[-1].extra_outputs
        return create_loop_state(
            x_init=train_X,
            y_init=train_Y,
            # We only support 1d metadata
            **{key: train_meta[:, idx].reshape(-1, 1) for idx, key in enumerate(meta_headers)},
            # Use the most recent UserFunctionResult entry in the LoopState generated by Benchmarker to populate any
            # missing meta data headers. This ensures that all metadata headers are enabled in the new LoopState.
            **{key: np.full((train_meta.shape[0], 1), benchmarker_metadata[key])
               for key in benchmarker_metadata.keys() if key not in meta_headers}
        )


import os
import json
import music21 as mc
import numpy as np
from collections import Counter

import seaborn as sns
sns.set()

import random
random.seed(20201308)

from gensim.models import Word2Vec

def get_duration_to_id(X_raw, ONEHOT_DURATION_COUNT):
    x_durations = []
    for x in X_raw:
        for xx in x:
            x_durations.append(xx.split('_')[-1])
    print("Found", len(set(x_durations)), "unique durations")
    most_common_d = sorted([pair[0] for pair in Counter(x_durations).most_common(ONEHOT_DURATION_COUNT)])
    d_to_id = {d : idx for idx, d in enumerate(most_common_d)}
    return d_to_id

def _encode_duration(d, duration_to_id):
    onehot = np.zeros(len(duration_to_id)+1)
    d_str = str(d)
    try:
        d_idx = duration_to_id[d_str]
    except KeyError: #outside
        d_idx = len(duration_to_id)
    onehot[d_idx] = 1.0
    return onehot

def get_parsed_data(data_path):
    with open(os.path.join(data_path, "parse_4.json"), "r") as handle:
        return json.load(handle)

def _fix_with_most_common(chords_mapping_dict):
    chords_mapping_dict['F major seventh chord'] = 'C+E+F+A'
    chords_mapping_dict['C major second major tetrachord'] = 'C+D+E+G'
    chords_mapping_dict['enharmonic equivalent to major triad above C#'] = 'C#+F+G#'
    chords_mapping_dict['C major seventh chord'] = 'C+E+G+B'
    chords_mapping_dict['Bb major seventh chord'] = 'D+F+A+B-'
    chords_mapping_dict['F# half diminished seventh chord'] = 'C+E+F#+A'
    chords_mapping_dict['F phrygian tetrachord'] = 'E+F+G+A'
    chords_mapping_dict['C# quartal tetramirror'] = 'C#+E+F#+B'
    chords_mapping_dict['B tritone fourth'] = 'C+F+B'
    chords_mapping_dict['C minor seventh chord'] = 'C+E-+F+G#'
    chords_mapping_dict['Eb quartal trichord'] = 'E-+F+B-'
    chords_mapping_dict['enharmonic equivalent to minor triad above Eb'] = 'E-+G#+B'
    chords_mapping_dict['Eb major seventh chord'] = 'D+E-+G+B-'
    chords_mapping_dict['C# minor seventh chord'] = 'C#+E+G#+B'
    chords_mapping_dict['Eb phrygian tetrachord'] = 'D+E-+F+G'
    chords_mapping_dict['C# major second major tetrachord'] = 'C#+E-+F+G#'
    chords_mapping_dict['E tritone fourth'] = 'E+F+B-'
    chords_mapping_dict['enharmonic to dominant seventh chord above C#'] = 'C#+E-+G+B-'
    chords_mapping_dict['A tritone fourth'] = 'E-+E+A'
    chords_mapping_dict['C# half diminished seventh chord'] = 'C#+E+G+B'
    chords_mapping_dict['E all interval tetrachord'] = 'E+F+G+B'
    chords_mapping_dict['C phrygian tetrachord'] = 'C+D+E+B'
    chords_mapping_dict['E whole tone tetramirror'] = 'D+E+F#+G#'
    chords_mapping_dict['A all interval tetrachord'] = 'C#+E-+E+A'
    chords_mapping_dict['G# minor second diminished tetrachord'] = 'D+G#+A+B'
    chords_mapping_dict['C perfect fourth major tetrachord'] = 'C+E+F+G'
    chords_mapping_dict['G# whole tone tetramirror'] = 'C+D+G#+B-'
    chords_mapping_dict['enharmonic equivalent to diminished triad above C#'] = 'C#+E+B-'
    chords_mapping_dict['C all interval tetrachord'] = 'C+D+E-+G#'
    chords_mapping_dict['C# minor tetramirror'] = 'C#+G#+B-+B'
    chords_mapping_dict['A double fourth tetramirror'] = 'C#+D+G#+A'
    chords_mapping_dict['D all interval tetrachord'] = 'D+E-+F+A'
    chords_mapping_dict['F augmented major tetrachord'] = 'C#+F+G#+A'
    chords_mapping_dict['C minor second quartal tetrachord'] = 'C+D+G+G#'
    chords_mapping_dict['F# minor second diminished tetrachord'] = 'C+F#+G+A'
    chords_mapping_dict['F# tritone fourth'] = 'C+F#+G'
    chords_mapping_dict['C augmented seventh chord'] = 'C+D+E+G#'
    chords_mapping_dict['Eb minor second quartal tetrachord'] = 'D+E-+G+A'
    chords_mapping_dict['Eb perfect fourth major tetrachord'] = 'E-+E+F#+B'
    chords_mapping_dict['Eb tritone fourth'] = 'D+E-+G#'
    chords_mapping_dict['A tritone quartal tetrachord'] = 'C#+D+G+A'
    chords_mapping_dict['F# minor tetramirror'] = 'E+F#+G+A'
    chords_mapping_dict['Eb double fourth tetramirror'] = 'D+E-+G+G#'
    chords_mapping_dict['C# tritone fourth'] = 'C#+D+G'
    chords_mapping_dict['C# perfect fourth minor tetrachord'] = 'C#+E-+F+B-'
    chords_mapping_dict['F augmented seventh chord'] = 'C#+E-+F+A'
    chords_mapping_dict['Bb all interval tetrachord'] = 'D+E+F+B-'
    chords_mapping_dict['Bb augmented major tetrachord'] = 'D+F#+A+B-'
    chords_mapping_dict['C# minor second diminished tetrachord'] = 'C#+E+B-+B'
    return chords_mapping_dict

def get_chords_mapping(data_path):
    with open(os.path.join(data_path, "chord_names.json"), "r") as handle:
        content = json.load(handle)
    new_dict = {content[key] : key for key in content}
    new_dict = _fix_with_most_common(new_dict)
    return new_dict

def name_to_ps(name):
    return mc.pitch.Pitch(name).ps if name != "REST" else -1.0

def unify_note(note):
    name, duration = note.split('_')
    pitch = name_to_ps(name)
    return f"{pitch}_{duration}"

chord_names_dict = {}

def fix_equivalent_name(name):
    part2, part1 = name.split('enharmonic equivalent to ')[-1].split(' above ')
    return ' '.join([part1, part2])

def unify_chord(chord, truncate_chord=4):
    names, duration = chord.split('_')
    new_names = []

    for name in names.split('.'):
        if name not in new_names:
            new_names.append(name)
    new_names = new_names[:truncate_chord]

    new_name = '+'.join(new_names)
    global chord_names_dict
    if new_name not in chord_names_dict:
        pitched_common_name = mc.chord.Chord(new_names).pitchedCommonName.replace('-', ' ')
#         if 'equivalent' in pitched_common_name:
#             pitched_common_name = fix_equivalent_name(pitched_common_name)
        chord_names_dict[new_name] = pitched_common_name

    return f"{chord_names_dict[new_name]}_{duration}"

def select_test(parsed_data, data_path, chorus_verse_only=True):
    if os.path.isfile(os.path.join(data_path, "test_idx.json")):
        with open(os.path.join(data_path, "test_idx.json"), "r") as handle:
            TEST_SELECT = json.load(handle)
            print("Loaded TEST indices")
    else:
        TEST_SIZE = int(len(parsed_data['parse_success']) * 0.1)
        TEST_SELECT = random.sample(range(0, len(parsed_data['parse_success'])), TEST_SIZE)

        with open(os.path.join(PATH, "test_idx.json"), "w") as handle:
            json.dump(TEST_SELECT, handle)

    result = []
    for idx in TEST_SELECT:
        path = parsed_data["parse_success"][idx]
        if (
            chorus_verse_only and ("chorus" in path.lower() or "verse" in path.lower())
        ) or not chorus_verse_only:
            result.append(idx)

    return result

def select_train(parsed_data, artist="*", test_select=None, chorus_verse_only=True):
    TRAIN_SELECT = []
    for idx, path in enumerate(parsed_data["parse_success"]):
        if (
            (isinstance(artist, str) and (artist == "*" or f"/{artist}/" in path))
            or
            (isinstance(artist, list) and any(f"/{ar}/" in path for ar in artist))
        ) and (idx not in test_select or not test_select):
            if (
                chorus_verse_only and ("chorus" in path.lower() or "verse" in path.lower())
            ) or not chorus_verse_only:
                TRAIN_SELECT.append(idx)
    return TRAIN_SELECT

def select_songs(parsed_data, SELECT, REPEAT=1):
    SONGS = []
    notes_translated, chords_translated = [], []
    for idx, (notes, chords) in enumerate(zip(parsed_data['notes'], parsed_data['chords'])):
        if idx in SELECT:
            notes_translated.append([unify_note(n) for n in notes] * REPEAT)
            chords_translated.append([unify_chord(n) for n in chords] * REPEAT)
            SONGS.append(parsed_data["parse_success"][idx])
    return SONGS, notes_translated, chords_translated

def get_train_songs(parsed_data, TRAIN_SELECT, REPEAT=1):
    TRAIN_SONGS = []
    notes_translated, chords_translated = [], []
    for idx, (notes, chords) in enumerate(zip(parsed_data['notes'], parsed_data['chords'])):
        if idx in TRAIN_SELECT:
            notes_translated.append([unify_note(n) for n in notes] * REPEAT)
            chords_translated.append([unify_chord(n) for n in chords] * REPEAT)
            TRAIN_SONGS.append(parsed_data["parse_success"][idx])
    return TRAIN_SONGS, notes_translated, chords_translated

def get_test_songs(parsed_data, TEST_SELECT, REPEAT=1):
    TEST_SONGS = []
    notes_translated_test, chords_translated_test = [], []
    for idx, (notes, chords) in enumerate(zip(parsed_data['notes'], parsed_data['chords'])):
        if idx in TEST_SELECT:
            notes_translated_test.append([unify_note(n) for n in notes])
            chords_translated_test.append([unify_chord(n) for n in chords])
            TEST_SONGS.append(parsed_data["parse_success"][idx])
    return TEST_SONGS, notes_translated_test, chords_translated_test

def load_embeddings(data_path, embeddings_dir="embeddings", emb_size=100, emb_window=8):
    models = {}
    notes_emb_filename = "embeddings_NOTES_size75_window8_CBOW_H-SFMAX.bin"
    chords_emb_filename = "embeddings_CHORDS-SHORT_size100_window2_CBOW_NEG-S.bin"
    print("> Load notes embeddings:", notes_emb_filename)
    print("> Load chords embeddings:", chords_emb_filename)
    models = {
        "NOTES": Word2Vec.load(os.path.join(data_path, embeddings_dir, "NOTES", notes_emb_filename)),
        "CHORDS": Word2Vec.load(os.path.join(data_path, embeddings_dir, "CHORDS", chords_emb_filename))
    }
    return models

def load_embeddings_OLD(data_path, embeddings_dir="embeddings", emb_size=100, emb_window=8):
    models = {}
    assert emb_size == 100
    assert emb_window == 8

    for emb_type in ["NOTES",]: #, "CHORDS"]:
        emb_filename = f"embeddings_{emb_type}_size{emb_size}_window{emb_window}.bin"
        model = Word2Vec.load(os.path.join(data_path, embeddings_dir, emb_type, emb_filename))
        models[emb_type] = model

    custom_embeddings = "embeddings_CHORDS-SHORT_size100_window4_CBOW_H-SFMAX.bin"
    models["CHORDS"] = Word2Vec.load(os.path.join(data_path, embeddings_dir, "CHORDS", custom_embeddings))

    return models

def decode_note(note):
    pitch, duration = note.split('_')
    return pitch, float(duration)

def align_notes_to_chords(notes, chords):
    note_to_chord_align = []

    chord_idx = 0
    _, current_chord_duration = decode_note(chords[chord_idx])

    offset = 0.0
    for idx, note in enumerate(notes):
        current_align = []
        _, duration = decode_note(note)
        current_align.append(chord_idx)
        offset += duration

        while current_chord_duration < offset:
            if chord_idx == len(chords)-1:
                break
            chord_idx += 1
            _, nd = decode_note(chords[chord_idx])
            current_chord_duration += nd
            current_align.append(chord_idx)

        note_to_chord_align.append(current_align)

    return note_to_chord_align

def extract_notes_pitches_durations(notes_slice):
    input_pitches = []
    input_durations = []
    for n in notes_slice:
        pitch, duration = n.split('_')
        input_pitches.append(float(pitch))
        input_durations.append(float(duration))
    return input_pitches, input_durations

def filter_out_chords(chords, window_size):
    if window_size == 1:
        return [chords[-1]] # only last chord - list for compatibility?
    elif window_size == 2:
        return [chords[0], chords[-1]] # first and last only
    else:
        result = []
        durations = np.array([decode_note(chord)[1] for chord in chords[1:-1]])
        top_durations_arg = np.argpartition(durations, -(window_size-2))[-(window_size-2):]
        top_durations = durations[top_durations_arg]
        for idx, d in enumerate(durations):
            if d in top_durations and len(result) < window_size-2:
                result.append(chords[idx])
            if len(result) >= window_size:
                break

        # always include the first and the last chord
        result = [chords[0], *result, chords[-1]]
        assert len(result) == window_size
        return result

def find_aligned_chords(aligns, target_idx, chords_window=4):
    # this operates on INDICES of chords! this is what 'aligns' store
    target_aligned_chords = aligns[target_idx]
    j = 1
    while len(target_aligned_chords) < chords_window and target_idx-j > 0:
        more_chords = aligns[target_idx-j]
        for ch in more_chords[::-1]:
            if ch not in target_aligned_chords:
                target_aligned_chords.insert(0, ch)
            if len(target_aligned_chords) == chords_window:
                break
        j += 1

    j = 1
    next_chord = None
    while not next_chord and target_idx+j < len(aligns):
        next_note_aligned_chords = aligns[target_idx+j]
        for ch in next_note_aligned_chords:
            if ch != target_aligned_chords[-1]:
                next_chord = ch
                break
        j += 1

    return target_aligned_chords, next_chord

def chord_pitches_to_fixed(pitches, pitches_per_chord=4):
    pitches = pitches.split('+')
    result = [0] * pitches_per_chord
    for i in range(min(pitches_per_chord, len(pitches))):
        result[i] = float(pitches[i])
    return result

def _normalize_octave(note, min_octave=3, max_octave=6):
    if note.octave < min_octave:
        note.octave = min_octave
    if note.octave > max_octave:
        note.octave = max_octave
    return note

def _note_pitched_to_named(note):
    pitched, duration = note.split('_')
    if pitched.replace('#', '').replace('-', '').isalnum():
        # is 'C3'
        name = pitched
    else:
        # probably can convert to float
        pitched = float(pitched)
        if pitched == -1.0:
            name = "REST"
        else:
            mc_note = mc.note.Note(pitched)
            mc_note = _normalize_octave(mc_note)
            name = mc_note.nameWithOctave
    return '_'.join([name, duration])

def _split_notes(notes):
    splitted = [nt.split('_') for nt in notes]
    pitches = [s[0] for s in splitted]
    durations = [s[1] for s in splitted]
    return pitches, durations

def prepare_raw_X_y(input_notes, input_chords, notes_window=8, chords_window=4, use_next_chord=True, rep=1):
    X_notes_raw, X_chords_raw, y_raw = [], [], []
    is_start, row_to_song_idx_map = [], []

    translate_notes = _should_translate_notes(input_notes[-1][-1])

    for s_dx, (ns, chs) in enumerate(zip(input_notes, input_chords)):
        notes = (ns.copy()) * rep
        chords = (chs.copy()) * rep

        note_off = 0.0
        chord_off = 0.0
        chidx = 0
        for i in range(notes_window, len(notes)):
            # ____ NOTES
            notes_slice = notes[i-notes_window:i]
            notes_slice = [_note_pitched_to_named(nt) for nt in notes_slice]
            # ____ TARGET
            target = _note_pitched_to_named(notes[i])

            ds = [float(note.split('_')[-1]) for note in notes_slice]
            if note_off == 0.0:
                note_off += sum(ds)

            while chord_off <= note_off and chidx < len(chords):
                chord_off += float(chords[chidx].split('_')[-1])
                chidx += 1
            # ____ CURRENT CHORD
            current_chord = chords[chidx-1]
            current_chord_rem = chord_off-note_off

            # ____ NEXT_CHORD
            if chidx < len(chords):
                next_chord = chords[chidx]
            else:
                next_chord = 0

            # ____ PREVIOUS CHORD
            prev_chords = chords[max(0, chidx-1-(chords_window-1)):max(0, chidx-1)]
            while len(prev_chords) < chords_window-1:
                prev_chords.insert(0, 0)

            X_notes_raw.append(notes_slice)
            X_chords_raw.append([*prev_chords, (current_chord, current_chord_rem), next_chord])
            y_raw.append(target)

            target_d = float(target.split('_')[-1])
            note_off += target_d

            is_start.append(1 if i == 0 else 0)
            row_to_song_idx_map.append(s_dx)

    assert len(X_notes_raw) == len(X_chords_raw) == len(y_raw) == len(row_to_song_idx_map)
    return X_notes_raw, X_chords_raw, y_raw, is_start, row_to_song_idx_map

def trim_rare_labels(X_notes_raw, X_chords_raw, y_raw, is_start, row_to_song_idx_map, min_freq=3, unk="UNK"):
    y_counts = Counter(y_raw)
    X_notes_new, X_chords_new, y_new, is_start_new, row_to_song_idx_new = [], [], [], [], []

    for xn, xch, y, is_st, s_idx in zip(X_notes_raw, X_chords_raw, y_raw, is_start, row_to_song_idx_map):
        X_notes_new.append(xn)
        X_chords_new.append(xch)

        if y_counts[y] >= min_freq:
            y_new.append(y)
        else:
            y_new.append(unk)

        is_start_new.append(is_st)
        row_to_song_idx_new.append(s_idx)

    return X_notes_new, X_chords_new, y_new, is_start_new, row_to_song_idx_new

def trim_rare_labels_separately(y_pitches_raw, y_durations_raw, min_freq=3, unk="UNK", keep=["START", "END", "0.0"]):
    y_p_counts = Counter(y_pitches_raw)
    y_d_counts = Counter(y_durations_raw)
    y_pitches_new, y_durations_new = [], []

    for y_p, y_d in zip(y_pitches_raw, y_durations_raw):
        if y_p_counts[y_p] >= min_freq or y_p in keep:
            y_pitches_new.append(y_p)
        else:
            y_pitches_new.append(unk)

        if y_d_counts[y_d] >= min_freq or y_d in keep:
            y_durations_new.append(y_d)
        else:
            y_durations_new.append(unk)

    return y_pitches_new, y_durations_new

_note_name_to_pitch_cache = {}
def _note_name_to_pitch(note_name):
    global _note_name_to_pitch_cache
    if note_name == 'REST':
        return -1.0
    else:
        if note_name not in _note_name_to_pitch_cache:
            _note_name_to_pitch_cache[note_name] = mc.note.Note(note_name).pitch.ps
        return _note_name_to_pitch_cache[note_name]

def prepare_input_X(X_notes_raw, X_chords_raw, is_start, row_to_song_idx, mode="", notes_window=8, chords_window=4, chord_size=4, indices_cache={}, embeddings=None, vectorizer=None, chords_mapping={},
duration_to_id_notes=None, duration_to_id_chords=None,):
    if mode.lower() == "embeddings":
        return _prepare_input_X_embeddings(
            X_notes_raw,
            X_chords_raw,
            is_start,
            row_to_song_idx,
            embeddings,
            notes_window=notes_window,
            chords_window=chords_window,
            indices_cache=indices_cache
        )
    elif mode.lower() == "pitches":
        return _prepare_input_X_pitches(
            X_notes_raw,
            X_chords_raw,
            is_start,
            row_to_song_idx,
            notes_window=notes_window,
            chords_window=chords_window,
            indices_cache=indices_cache,
            chord_size=chord_size,
            chords_mapping=chords_mapping
        )
    elif mode.lower() == "custom":
        return _prepare_input_X_custom(
            X_notes_raw,
            X_chords_raw,
            row_to_song_idx,
            embeddings,
            vectorizer,
            duration_to_id_notes,
            duration_to_id_chords,
            notes_window=notes_window,
            chords_window=chords_window,
            indices_cache=indices_cache,
        )

def _neutral_embedding(embeddings):
    return np.mean(embeddings.wv.vectors, axis=0) / np.std(embeddings.wv.vectors, axis=0)

def _chord_emb(chords, ch_embeddings):
    ch_emb = []
    for ch in chords:
        if ch != 0:
            if isinstance(ch, str):
                ch_name = ch.split('_')[0]
            elif isinstance(ch, tuple):
                ch_name = ch[0].split('_')[0]
            try:
                ch_emb.append(ch_embeddings[ch_name])
            except KeyError:
                ch_emb.append(_neutral_embedding(ch_embeddings))
        elif ch == 0:
            ch_emb.append([0] * ch_embeddings.vector_size)
        else:
            raise Exception(f"CHORD is {ch}!")
    return ch_emb

def _encode_embeddings(notes, chords, embeddings, is_start=False, translate_notes=False, notes_window=8, chords_window=3):
    notes_emb_size = embeddings["NOTES"].vector_size
    chords_emb_size = embeddings["CHORDS"].vector_size

    vector = np.zeros(
        notes_window*notes_emb_size + notes_window + (chords_window+1)*chords_emb_size + (chords_window+1) + 1 + 1,
        dtype=np.float32
    )
    of = 0

    # NOTE PITCHES
    n_pitches = [note.split('_')[0] for note in notes]
    if translate_notes:
        n_pitches = [str(_note_name_to_pitch(n)) for n in n_pitches]

    n_emb = []
    for n in n_pitches:
        try:
            n_emb.append(embeddings["NOTES"][n])
        except KeyError:
            n_emb.append(_neutral_embedding(embeddings["NOTES"]))

    vector[of:of+(notes_window*notes_emb_size)] = np.array(n_emb).reshape(-1)
    of += (notes_window*notes_emb_size)

    # NOTE DURATIONS
    vector[of:of+notes_window] = [float(note.split('_')[1]) for note in notes]
    of += notes_window

    # CHORDS PITCHES
    assert len(chords) == chords_window+1
    ch_emb = _chord_emb(chords, embeddings["CHORDS"])
    vector[of:of + (chords_window+1)*chords_emb_size] = np.array(ch_emb).reshape(-1)
    of += (chords_window+1)*chords_emb_size

    # CHORDS DURATIONS
    ch_durations = []
    for ch in chords:
        if ch != 0:
            if isinstance(ch, str):
                d = [float(ch.split('_')[1])]
            elif isinstance(ch, tuple):
                d1 = float(ch[0].split('_')[1]) # full duration
                d2 = ch[1] # currnt duration
                d = [d1, d2]
            ch_durations.extend(d)
        elif ch == 0:
            ch_durations.append(0)
        else:
            raise Exception(f"CHORD is {ch}!")

    vector[of:of + (chords_window+1+1)] = np.array(ch_durations)
    of += (chords_window+1+1)

    # IS START
    vector[-1] = 1 if is_start else 0

    return vector

def _encode_custom(notes, chords, embeddings, vectorizer, duration_to_id_notes, duration_to_id_chords, translate_notes=False, notes_window=8, chords_window=3):
    notes_emb_size = embeddings["NOTES"].vector_size
    vectorizer_size = len(vectorizer.vocabulary_)

    vector = np.zeros(
        notes_window*notes_emb_size + notes_window*(len(duration_to_id_notes)+2) + (chords_window+1)*vectorizer_size + (chords_window+1+1)*(len(duration_to_id_chords)+2),
        dtype=np.float32
    )
    of = 0

    # NOTE PITCHES
    n_pitches = [note.split('_')[0] for note in notes]
    if translate_notes:
        n_pitches = [str(_note_name_to_pitch(n)) for n in n_pitches]

    n_emb = []
    for n in n_pitches:
        try:
            n_emb.append(embeddings["NOTES"][n])
        except KeyError:
            n_emb.append(_neutral_embedding(embeddings["NOTES"]))

    vector[of:of+(notes_window*notes_emb_size)] = np.array(n_emb).reshape(-1)
    of += (notes_window*notes_emb_size)

    # NOTE DURATIONS
    for d in [note.split('_')[1] for note in notes]:
        onehot = _encode_duration(d, duration_to_id_notes)
        vector[of:of+len(onehot)] = onehot
        of += len(onehot)
        vector[of] = float(d)
        of += 1

    # CHORDS PITCHES
    assert len(chords) == chords_window+1
    for ch in chords:
        if isinstance(ch, str):
            emb = vectorizer.transform([ch]).toarray().reshape(-1)
        else:
            emb = [0] * vectorizer_size
        vector[of:of+vectorizer_size] = emb
        of += vectorizer_size

    # CHORDS DURATIONS
    ch_durations = []
    for ch in chords:
        if ch != 0:
            if isinstance(ch, str):
                d = ch.split('_')[1]
                d_vec = [*_encode_duration(d, duration_to_id_chords), float(d)]
            elif isinstance(ch, tuple):
                d1 = ch[0].split('_')[1] # full duration
                d2 = ch[1] # currnt duration
                d_vec = [*_encode_duration(d1, duration_to_id_chords), float(d1), *_encode_duration(d2, duration_to_id_chords), float(d2)]
            ch_durations.extend(d_vec)
        elif ch == 0:
            ch_durations.extend([0] * (len(duration_to_id_chords) + 1 + 1))
        else:
            raise Exception(f"CHORD is {ch}!")

    assert len(ch_durations) == (chords_window+1+1) * (len(duration_to_id_chords)+1+1)
    vector[of:of + len(ch_durations)] = np.array(ch_durations)
    of += len(ch_durations)

    assert of == len(vector)
    return vector

def _prepare_input_X_embeddings(X_notes_raw, X_chords_raw, is_start, row_to_song_idx, embeddings, notes_window, chords_window, indices_cache):
    notes_emb_size = embeddings["NOTES"].vector_size
    chords_emb_size = embeddings["CHORDS"].vector_size

    chords_neutral_emb = _neutral_embedding(embeddings["CHORDS"])
    chord_miss_count = 0

    X = np.zeros(
        (
                len(X_notes_raw),
                                                        #embed each chord   #prev, current & next duration: current x2!
                notes_window*notes_emb_size + notes_window + (chords_window+1)*chords_emb_size + (chords_window+1) + 1 + 1 # + is_start
        ),
        dtype=np.float32
    )

    # NOTE PITCH, NOTE DURATION, CHORD PITCHES, CHORD DURATION, IS_START

    iterator = list(enumerate(zip(X_notes_raw, X_chords_raw, is_start, row_to_song_idx)))

    sample_note = X_notes_raw[-1][-1].split('_')[0]
    translate_notes = _should_translate_notes(X_notes_raw[-1][-1])

    for idx, (notes, chords, st, song_idx) in iterator:
        X[idx, :] = _encode_embeddings(
            notes,
            chords,
            embeddings,
            is_start=False,
            translate_notes=translate_notes,
            notes_window=notes_window,
            chords_window=chords_window,
        )

        # CACHE HELPER
        try:
            indices_cache[song_idx].append(idx)
        except KeyError:
            indices_cache[song_idx] = [idx]

    # shrink cache helper
    for idx in indices_cache:
        indices_cache[idx] = [indices_cache[idx][0], indices_cache[idx][-1]]

    print(f"No. UNK chords: {chord_miss_count}")
    return X, indices_cache

def _should_translate_notes(sample):
    sample_note = sample.split('_')[0].replace('-', '').replace('#', '')
    if sample_note.isalnum() or sample_note == "REST":
        return True
    return False

def _prepare_input_X_custom(
    X_notes_raw,
    X_chords_raw,
    row_to_song_idx,
    embeddings,
    vectorizer,
    duration_to_id_notes,
    duration_to_id_chords,
    notes_window,
    chords_window,
    indices_cache
):
    notes_emb_size = embeddings["NOTES"].vector_size
    chords_neutral_emb = _neutral_embedding(embeddings["CHORDS"])
    vectorizer_size = len(vectorizer.vocabulary_)

    X = np.zeros(
        (
                len(X_notes_raw),
                notes_window*notes_emb_size + notes_window*(len(duration_to_id_notes)+2) + (chords_window+1)*vectorizer_size + (chords_window+1+1)*(len(duration_to_id_chords)+2)
        ),
        dtype=np.float32
    )

    # NOTE PITCH, NOTE DURATION, CHORD PITCHES, CHORD DURATION, IS_START

    iterator = list(enumerate(zip(X_notes_raw, X_chords_raw, row_to_song_idx)))
    translate_notes = _should_translate_notes(X_notes_raw[-1][-1])

    for idx, (notes, chords, song_idx) in iterator:
        X[idx, :] = _encode_custom(
            notes,
            chords,
            embeddings,
            vectorizer,
            duration_to_id_notes,
            duration_to_id_chords,
            translate_notes=translate_notes,
            notes_window=notes_window,
            chords_window=chords_window,
        )

        # CACHE HELPER
        try:
            indices_cache[song_idx].append(idx)
        except KeyError:
            indices_cache[song_idx] = [idx]

    # shrink cache helper
    for idx in indices_cache:
        indices_cache[idx] = [indices_cache[idx][0], indices_cache[idx][-1]]

    return X, indices_cache

def _encode_pitches(notes, chords, is_start=False, translate_notes=False, notes_window=8, chords_window=3, chord_size=4, chords_mapping={}):
    vector = np.zeros(
        (notes_window*1 + notes_window + (chords_window+1)*chord_size + (chords_window+1) + 1 + 1),
        dtype=np.float32
    )
    of = 0

    # NOTE PITCHES
    n_pitches = [note.split('_')[0] for note in notes]
    if translate_notes:
        n_pitches = [_note_name_to_pitch(n) for n in n_pitches]
    else:
        n_pitches = [float(n) for n in n_pitches]

    vector[of:of+(notes_window*1)] = np.array(n_pitches).reshape(-1)
    of += (notes_window*1)

    # NOTE DURATIONS
    n_durations = [float(note.split('_')[1]) for note in notes]
    vector[of:of+notes_window] = n_durations
    of += notes_window

    # CHORDS PITCHES
    assert len(chords) == chords_window+1
    ch_p = []
    for ch in chords:
        if ch != 0:
            if isinstance(ch, str):
                ch_name = ch.split('_')[0]
            elif isinstance(ch, tuple):
                ch_name = ch[0].split('_')[0]
            ch_notes = chords_mapping[ch_name]
            ch_pitches = [_note_name_to_pitch(n) for n in ch_notes.split('+')][:chord_size]
            ch_pitches += [0] * max(0, chord_size-len(ch_pitches))
            assert len(ch_pitches) == chord_size
            ch_p.extend(ch_pitches)
        elif ch == 0:
            ch_p.extend([0] * chord_size)
        else:
            raise Exception(f"CHORD is {ch}!")

    vector[of:of + (chords_window+1)*chord_size] = np.array(ch_p).reshape(-1)
    of += (chords_window+1)*chord_size

    # CHORDS DURATIONS
    ch_durations = []
    for ch in chords:
        if ch != 0:
            if isinstance(ch, str):
                d = [float(ch.split('_')[1])]
            elif isinstance(ch, tuple):
                d1 = float(ch[0].split('_')[1]) # full duration
                d2 = ch[1] # currnt duration
                d = [d1, d2]
            ch_durations.extend(d)
        elif ch == 0:
            ch_durations.append(0)
        else:
            raise Exception(f"CHORD is {ch}!")

    vector[of:of + (chords_window+1+1)] = np.array(ch_durations)
    of += (chords_window+1+1)

    # IS START
    vector[-1] = 1 if is_start else 0
    return vector

def _prepare_input_X_pitches(X_notes_raw, X_chords_raw, is_start, row_to_song_idx, notes_window, chords_window, chord_size, indices_cache, chords_mapping={}):
    X = np.zeros(
        (
                len(X_notes_raw),
                                                        #embed each chord   #prev, current & next duration: current x2!
                notes_window*1 + notes_window + (chords_window+1)*chord_size + (chords_window+1) + 1 + 1 # + is_start
        ),
        dtype=np.float32
    )

    # NOTE PITCH, NOTE DURATION, CHORD PITCHES, CHORD DURATION, IS_START

    iterator = list(enumerate(zip(X_notes_raw, X_chords_raw, is_start, row_to_song_idx)))
    translate_notes = _should_translate_notes(X_notes_raw[-1][-1])

    for idx, (notes, chords, st, song_idx) in iterator:
        vector = _encode_pitches(
            notes,
            chords,
            is_start=st,
            translate_notes=translate_notes,
            notes_window=notes_window,
            chords_window=chords_window,
            chord_size=chord_size,
            chords_mapping=chords_mapping
        )
        X[idx, :] = vector

        # CACHE HELPER
        try:
            indices_cache[song_idx].append(idx)
        except KeyError:
            indices_cache[song_idx] = [idx]

    # shrink cache helper
    for idx in indices_cache:
        indices_cache[idx] = [indices_cache[idx][0], indices_cache[idx][-1]]

    return X, indices_cache

from keras.utils import np_utils

def prepare_input_y(y_raw, limit=None, noneclass=True):
    all_tokens = list(sorted(set(y_raw)))
    if noneclass:
        all_tokens.append('NOTHING')
    token_to_id = {token : idx for idx, token in enumerate(all_tokens)}

    if not limit:
        y = np_utils.to_categorical([token_to_id[token] for token in y_raw], num_classes=len(all_tokens))
    else:
        y = np_utils.to_categorical([token_to_id[token] for token in y_raw[:limit]], num_classes=len(all_tokens))
    return y, token_to_id

def get_duration(piece):
    _, duration = piece.split("_")
    return float(duration)

def chord_to_pitches_duration_fixed(chord):
    pitches, duration = chord.split("_")
    pitches = chord_pitches_to_fixed(pitches)
    return pitches, float(duration)

def extract_X_by_song_idx(X, sample_song_idx, row_to_song_idx_map):
    indices = np.where(np.array(row_to_song_idx_map) == sample_song_idx)[0]
    first = indices[0]
    last = indices[-1]
    return X[first:last+1, :, :]

def extract_Xy_by_song_idx(X, y, sample_song_idx, row_to_song_idx_map, indices_cache={}):
    try:
        indices = indices_cache[sample_song_idx]
    except KeyError:
        indices = np.where(np.array(row_to_song_idx_map) == sample_song_idx)[0]
        indices_cache[sample_song_idx] = indices
    first = indices[0]
    last = indices[-1]
    return X[first:last+1, :, :], y[first:last+1], indices_cache

def extract_first_last_by_song_idx(sample_song_idx, row_to_song_idx_map, indices_cache={}):
    try:
        indices = indices_cache[sample_song_idx]
    except KeyError:
        indices = np.where(np.array(row_to_song_idx_map) == sample_song_idx)[0]
        indices_cache[sample_song_idx] = indices
    first = indices[0]
    last = indices[-1]
    return first, last, indices_cache

def get_bigram_transitions_chord_wise(notes_translated, chords_translated, translate_pitch_to_name=True, flat=False):
    transitions = {}

    for notes, chords in zip(notes_translated, chords_translated):

        note_t, chord_t = 0.0, 0.0
        ch_idx = 0
        prev_note = None
        for note in notes:
            note_tr = _note_pitched_to_named(note) if translate_pitch_to_name else note
            current_chord = chords[ch_idx]
            label = current_chord if not flat else current_chord.split('_')[0]
            if not prev_note:
                transitions.setdefault(label, {}).setdefault("START_0.0", []).append(note_tr)
            else:
                transitions.setdefault(label, {}).setdefault(prev_note, []).append(note_tr)
            note_t += float(note_tr.split('_')[-1])
            while chord_t < note_t:
                chord_t += float(chords[ch_idx].split('_')[-1])
                ch_idx += 1
                if ch_idx == len(chords):
                    break
                else:
                    current_chord = chords[ch_idx]
                    # transitions.setdefault(chords[ch_idx], {}).setdefault(prev_note, []).append(note) # (TO CONSIDER...)

            if ch_idx == len(chords):
                break

        transitions.setdefault(label, {}).setdefault(note_tr, []).append("END_0.0")

    for ch in transitions:
        for prev_note in transitions[ch]:
            data = transitions[ch][prev_note]
            sum_events = len(data)
            counts = Counter(data).most_common()
            new_data = {}
            for pair in counts:
                new_data[pair[0]] = pair[1] / sum_events
            transitions[ch][prev_note] = new_data

    return transitions


import numpy as np
from ple import PLE
from ple.games.waterworld import WaterWorld
import random
import operator

def round(state):
    new_state = {}

    for i in state:
        if i != 'creep_pos' and i != 'player_x' and i != 'player_y':
            if i == 'creep_dist':
                new_state['creep_dist_BAD'] = tuple(
                    x // 200 for x in state['creep_dist']['BAD'])
                new_state['creep_dist_GOOD'] = tuple(
                    x // 200 for x in state['creep_dist']['GOOD'])
                pass
            else:
                new_state[i] = int(state[i] / 100)
    return new_state

# Making an agent from game env
agent = WaterWorld(width=256, height=256, num_creeps=8)

# Creating a game environment (use lower fps so we can see whats happening a little easier)
env = PLE(agent, fps=15, force_fps=False, display_screen=True)

env.init()
actions = env.getActionSet()
q_table = {}
alpha = 0.1
gamma = 0.5

while (True):
    oldGameState = round(agent.getGameState())
    if env.game_over():
        env.reset_game()

    right = q_table.get(tuple(oldGameState.values()) + (100,), 0)
    left = q_table.get(tuple(oldGameState.values()) + (97,), 0)
    up = q_table.get(tuple(oldGameState.values()) + (119,), 0)
    down = q_table.get(tuple(oldGameState.values()) + (115,), 0)
    none = q_table.get(tuple(oldGameState.values()) + (None,), 0)
    stats = {119: up, 115: down, right: 100, 97: left, None: none}
    if (up == down == none == right == left):
        action = actions[random.randint(0, 4)]  # random actions
    else:
        action = max(stats.items(), key=operator.itemgetter(1))[0]
    reward = env.act(action)
    newGameState = round(agent.getGameState())
    oldQState = tuple(oldGameState.values()) + (action,)
    if oldQState not in q_table.keys():
        q_table[oldQState] = 0

    right_next = q_table.get(tuple(newGameState.values()) + (100,), 0)
    left_next = q_table.get(tuple(newGameState.values()) + (97,), 0)
    up_next = q_table.get(tuple(newGameState.values()) + (119,), 0)
    down_next = q_table.get(tuple(newGameState.values()) + (115,), 0)
    none_next = q_table.get(tuple(newGameState.values()) + (None,), 0)
    next_max = max(up_next, down_next, none_next, left_next, right_next)
    sample = reward + gamma * next_max
    q_table[oldQState] = (1 - alpha) * q_table[oldQState] + alpha * sample


from django import forms
from django.contrib.auth import get_user_model
from ilan.models import Ilan, Comment
from django.conf import settings
from django.utils.translation import gettext_lazy as _

from ckeditor_uploader.widgets import CKEditorUploadingWidget

class IlanForm(forms.ModelForm):
    metin = forms.CharField(widget=CKEditorUploadingWidget())
    class Meta:
        model = Ilan
        fields = ['baslik', 'metin', 'ilan_foto', 'ilan_fon_muzik', 'arkaplan_rengi', 'arkaplan_resmi']
        labels = {
            'baslik': _('Başlık'),
            'metin': _('İlan Metni'),
            'ilan_foto': _('İlan Görseli'),
            'ilan_fon_muzik': _('Fon Müziği'),
            'arkaplan_rengi': _('Arkaplan Rengi'),
            'arkaplan_resmi': _('Arkaplan Resmi'),
        }

class CommentForm(forms.ModelForm):
    class Meta:
        model = Comment
        fields = ['yorum_metni']


"""Bridges Midi from Voicelive to OSC RASPI, and to Local midi PC sampler. bridges notes from Raspi/LH to sampler"""

import logging
import sys
import time

sys.path.append("..")

import rtmidi
from rtmidi.midiutil import open_midiinput

from pythonosc import udp_client
from pythonosc import osc_server
from pythonosc import dispatcher

import os
import socket

from threading import Timer

from SmoothDefines import *

#==========================================================
# Constant definitions
hostname = socket.gethostname()
LOCALHOST_IP   = socket.gethostbyname(hostname)

LIVESERVER_PORT     = 8000
LH_PORT = 8010

INPUT_PORT  = 'AudioBox USB 96 MIDI In 1'
OUTPUT_PORT = 'Springbeats vMIDI4'

#==========================================================

class MidiInputHandler(object):
    def __init__(self, port):
        self.port = port
        self._wallclock = time.time()
        self.scheduled_timer = None

    def debounced_func(self, wait, func, *args, **kwargs):

        if self.scheduled_timer and not self.scheduled_timer.finished.is_set():
            print("cancelling Timer")
            self.scheduled_timer.cancel()

        self.scheduled_timer = Timer(wait, func, args=args, kwargs=kwargs)
        self.scheduled_timer.start()

    def debounce(self, wait, func, *args, **kwargs):
        """Returns a debounced version of a function.

        The debounced function delays invoking `func` until after `wait`
        seconds have elapsed since the last time the debounced function
        was invoked.
        """
        print("Debouncing")
        return self.debounced_func(wait, func, *args)

    def forwardProgramChange(self, value):
        print("Forwarding Program change : " + str(value))
        client.send_message("/midi/voicelive", [0xC0, value, 0])

        if(value == 1): #We don't receive the preset 1(index 0...), so let's use the number 2(index 1).
            #shutdown
            print("Shuting down immediately")
            client.send_message("/midi/shutdown", 1)
            cmd = "Shutdown -s -f -t 0"
            os.system(cmd)

    def __call__(self, event, data=None):
        message, deltatime = event
        self._wallclock += deltatime
        print("[%s] @%0.6f %r" % (self.port, self._wallclock, message))
        #Forward program changes received from Voicelive
        if(message[0] == 0xC0):
           #send program change thru OSC
           programVal = message[1]
           self.debounce(DEBOUNCE_PRESET_DELAY, self.forwardProgramChange, programVal)

        else:
           #Forward Control change received from Voicelive
           #print("Forwarding Control change" + str(message[1]))
           client.send_message("/midi/voicelive", [message[0], message[1], message[2]])

#Receives note on and off coming from LaserHarp
def print_laserharp_handler(osc_address, args, velocity):
  print("received OSC message from laser harp")
  print("address" + osc_address)
  print(velocity)

  temp = osc_address.split("/")
  note = int(temp[4])
  print (note);
  if(velocity != 0):
    note = [0x90, note, velocity]    # Note ON
  else:
    note = [0x80, note, 0]           #note OFF
  midiout.send_message(note)

#==========================================================
#Main code

#Open output port
midiout = rtmidi.MidiOut()
ports = midiout.get_ports()
for port, name in enumerate(ports):
    print("[%i] #%s#" % (port, name))
    if(OUTPUT_PORT in name):
       print("Opening output port [%i] #%s#" % (port, name))
       midiout.open_port(port)

log = logging.getLogger('midiin_callback')
logging.basicConfig(level=logging.DEBUG)

#Open input port
midiin = rtmidi.MidiIn()
ports = midiin.get_ports()
for port, name in enumerate(ports):
    print("[%i] #%s#" % (port, name))
    if(INPUT_PORT in name):
       print("Opening input port [%i] #%s#" % (port, name))
       midiin.open_port(port)
       port_name = name

client = udp_client.SimpleUDPClient(LOCALHOST_IP, LIVESERVER_PORT)

dispatcher = dispatcher.Dispatcher()
dispatcher.map("/vkb_midi/0/*", print_laserharp_handler, "midi_laserharp")

server = osc_server.ThreadingOSCUDPServer(
    (LOCALHOST_IP, LH_PORT), dispatcher)

print("Attaching MIDI input callback handler.")

midiin.set_callback(MidiInputHandler(port_name))

try:
    print("Entering main loop. Press Control-C to exit.")
    server.serve_forever()
except KeyboardInterrupt:

    exit()


'''
Eric Pfleiderer
PHY3075-Numerical modelling: Modelisation numerique en phyique
Projet 5: Mutation adaptive
'''

import pandas as pd
import numpy as np
from scipy.optimize import bisect

'''
BUG:
    -All data becomes NaN if e>1 or e<-1, must impose bounderies?
    -No error caused by omega, but same idea? impose bounderies?
'''

'''
TO DO:
    -Constrain e (DONE)
    -Constrain w (DONE
    -Modify mutation_vector on the fly
    -Implement try/except on bisect optimizer in event f(a)*f(b)>0 (DONE)
    -Implement partial randomized initialization of parameters using input vector (continue from last opti)
'''

#Genetic optimizer
class GenOpti:

    #Constructor
    #N must be even!
    def __init__(self, N=10, vector_size=6, mutation_size=0.25, path='data/eta_Bootis.csv'):
        self.N = N
        self.vector_size = vector_size
        self.data = pd.read_csv(path)
        self.population = self.initialize_random() #P, tau, omega, e, K, V0
        self.mutation_vector = self.initialize_mutation(mutation_size=mutation_size)

    #Initialize population of N solution vectors according to specific intervals obtained from data analysis
    def initialize_random(self):

        population = np.zeros((self.N, self.vector_size))

        for n in range(self.N):
            population[n][0] = np.random.uniform(200, 800) #P
            population[n][1] = np.random.uniform(self.data.iloc[0, 0], self.data.iloc[0, 0] + population[n][0]) #tau
            population[n][2] = np.random.uniform(0, 2*np.pi) #omega
            population[n][3] = np.random.uniform(0, 1) #e
            population[n][4] = np.random.uniform(0, self.data['radial_velocity'].max() - self.data['radial_velocity'].min()) #K
            population[n][5] = np.random.uniform(self.data['radial_velocity'].min(), self.data['radial_velocity'].max()) #V0

        return population

    #Simulate the life of N generations with specified genetic pressures
    def simulate_generations(self, generations=20, selective_pressure=1/5, p_c=0.8, p_m=0.1):

        for generation in range(generations):

            if generation % 50 is 0:
                print('Generation:', generation)

            #Evaluate current solutions
            current_solution = self.evaluate_objective(self.population)

            #Rank solutions from worse to best by index
            rank = np.argsort(current_solution)

            #Select N new solutions, repetitions allowed (parents)
            parents_idx = np.empty(0)

            for n in range(self.N//2):
                best_parent, second_parent = self.generate_parents(rank, selective_pressure)
                parents_idx = np.append(parents_idx, best_parent)
                parents_idx = np.append(parents_idx, second_parent)
                parents_idx = np.int32(parents_idx)

            children = self.generate_kids(parents_idx, p_c)

            #Mutate children if need be
            for n in range(self.N):
                for component in range(self.vector_size):
                    if np.random.uniform(0, 1) <= p_m:
                        children[n][component] += np.random.normal(0, abs(self.mutation_vector[component]))

            #Constrain e to [0, 1] by definition
            mask_neg = np.where(children[:, 3] < 0)
            mask_pos = np.where(children[:, 3] > 1)
            children[mask_neg, 3] = 0.05
            children[mask_pos, 3] = 0.95

            #Constrain omega to positive values by definition
            mask_neg = np.where(children[:, 2] < 0)
            children[mask_neg, 2] = 0.05

            #Preserve best solution (elitism) and update current population
            top_sol = self.population[rank[self.N-1], :]
            self.population = children
            self.population[0, :] = top_sol

    #Generate a new population from the parents and genetic pressures
    def generate_kids(self, parents_idx, p_c=0.8):

        r = np.empty(self.vector_size)

        #Generate a ratio for every vector component
        for n in range(r.size):
            r[n] = np.random.uniform(0, 1)

        children = np.empty((self.N, self.vector_size))

        #For every couple of parents
        for n in range(self.N//2):

            #Find solution vectors for current pair of reproducing parents
            parent1_sol = self.population[parents_idx[2*n], :]
            parent2_sol = self.population[parents_idx[2*n+1], :]

            #Generate 2 new solutions vectors from parent vectors
            for component in range(parent1_sol.size):
                if np.random.uniform(0, 1) <= p_c:
                    children[2*n][component] = r[component] * parent1_sol[component] + (1-r[component]) * parent2_sol[component]
                    children[2*n+1][component] = (1-r[component]) * parent1_sol[component] + r[component] * parent2_sol[component]
                else:
                    children[2*n][component] = parent1_sol[component]
                    children[2*n+1][component] = parent2_sol[component]

        return children

    #Returns index of chosen parents from rank
    def generate_parents(self, rank, selective_pressure=1/5):
        # Choose best parent randomly according to selective pressure
        best_index = rank[np.random.randint(int((1 - selective_pressure) * self.N), self.N)]

        # Choose second parent
        second_index = best_index
        while second_index == best_index:
            second_index = np.random.randint(0, self.N - 1)

        return best_index, second_index

    #Evaluate objective function for each individual (solution matrix must be n by 6 2darray)
    def evaluate_objective(self, solution_matrix):
        chi = 0
        for idx, row in self.data.iterrows():
            chi += ((row[1] - self.evaluate_solution(solution_matrix, row[0]))/row[2])**2
        chi *= 1/(self.data.shape[0])

        return 1/chi

    #Evaluate solution at specific time t given a certain solution matrix
    def evaluate_solution(self, solution_matrix, t):

        E = np.empty(0)

        #Parameter syntaxic shortcuts
        P = solution_matrix[:, 0]
        tau = solution_matrix[:, 1]
        omega = solution_matrix[:, 2]
        e = solution_matrix[:, 3]
        K = solution_matrix[:, 4]
        V0 = solution_matrix[:, 5]

        #Solve keplers equation for the root
        for n in range(P.shape[0]):

            #Try/catch in case the interval does not bound the zero
            bounding_interval = False
            a, b = -1000, 1000
            while not bounding_interval:
                try:
                    E = np.append(E, bisect(self.evaluate_kepler, a, b, args=(t, P[n], tau[n], e[n])))
                    bounding_interval = True
                except:
                    a, b = 2*a, 2*b

        #Solve for projected velocity v
        orbital_velocity = 2*np.arctan(np.sqrt((1+e)/(1-e)) * np.tan(E/2)) #CANT IMPLEMENT ATAN2, ONLY 1 PARAM?

        #Solve for radial velocity
        radial_velocity = V0 + K * (np.cos(omega + orbital_velocity) + e*np.cos(omega))

        return radial_velocity

    #Keplers equation
    def evaluate_kepler(self, E, t, P, tau, e):
        return E - e * np.sin(E) - (2*np.pi/P) * (t-tau)

    #To check if needed for different order of parameters
    def initialize_mutation(self, mutation_size=0.25):

        mutation_vector = np.zeros(self.vector_size)

        for vector_idx in range(self.vector_size):
            mutation_vector[vector_idx] = mutation_size * (1/self.vector_size)*np.sum(self.population[:, vector_idx])

        return mutation_vector


#!/usr/bin/env python2.6
# $Id$

import copy
import re
import unittest
import mox
import difflib
import checkpkg
import checkpkg_lib
import database
import models as m
import tag
import package_stats
import sqlite3
import sqlobject
import test_base
import common_constants

from testdata.tree_stats import pkgstats as tree_stats
from testdata.neon_stats import pkgstats as neon_stats

"""A set of unit tests for the library checking code.

A bunch of lines to test in the interactive Python shell.

import sys
sys.path.append("lib/python")
import checkpkg

sqlite3 ~/.checkpkg/var-sadm-install-contents-cache-build8x
SELECT * FROM systempkgmap WHERE basename = 'libncursesw.so.5';
"""

class ExtractorsUnitTest(unittest.TestCase):

  def testExtractDescriptionFromGoodData(self):
    data = {"NAME": "nspr_devel - Netscape Portable Runtime header files"}
    result = "Netscape Portable Runtime header files"
    self.assertEqual(result, checkpkg.ExtractDescription(data))

  def testExtractDescriptionWithBadCatalogname(self):
    data = {"NAME": "foo-bar - Bad catalogname shouldn't break this function"}
    result = "Bad catalogname shouldn't break this function"
    self.assertEqual(result, checkpkg.ExtractDescription(data))

  def testExtractMaintainerName(self):
    data = {"VENDOR": "https://ftp.mozilla.org/pub/mozilla.org/"
                      "nspr/releases/v4.8/src/ packaged for CSW by "
                      "Maciej Blizinski"}
    result = "Maciej Blizinski"
    self.assertEqual(result, checkpkg.ExtractMaintainerName(data))

  def testPstampRegex(self):
    pstamp = "hson@solaris9s-csw-20100313144445"
    expected = {
        'username': 'hson',
        'timestamp': '20100313144445',
        'hostname': 'solaris9s-csw'
    }
    self.assertEqual(expected, re.match(common_constants.PSTAMP_RE, pstamp).groupdict())

class SliceListUnitTest(unittest.TestCase):

  def testOne(self):
    l = [1, 2, 3, 4, 5]
    s = 1
    expected = [[1], [2], [3], [4], [5]]
    self.assertTrue(expected, checkpkg_lib.SliceList(l, s))

  def testTwo(self):
    l = [1, 2, 3, 4, 5]
    s = 2
    expected = [[1, 2], [3, 4], [5]]
    self.assertTrue(expected, checkpkg_lib.SliceList(l, s))

class SqliteUnitTest(unittest.TestCase):

  "Makes sure that we can lose state between tests."

  def setUp(self):
    self.conn = sqlite3.connect(":memory:")
    self.c = self.conn.cursor()

  def tearDown(self):
    self.conn = None

  def testCannotCreateTwoTables(self):
    self.c.execute("CREATE TABLE foo (INT bar);")
    self.assertRaises(
        sqlite3.OperationalError,
        self.c.execute, "CREATE TABLE foo (INT bar);")

  def testOne(self):
    self.c.execute("CREATE TABLE foo (INT bar);")

  def testTwo(self):
    self.c.execute("CREATE TABLE foo (INT bar);")

class SqlobjectUnitTest(test_base.SqlObjectTestMixin, unittest.TestCase):

  "Makes sure that we can lose state between methods."

  class TestModel(sqlobject.SQLObject):
    name = sqlobject.UnicodeCol(length=255, unique=True, notNone=True)

  # This does not work. Why?
  # def testCannotCreateTwoTables(self):
  #   self.TestModel.createTable()
  #   self.assertRaises(
  #       sqlite3.OperationalError,
  #       self.TestModel.createTable)

  def testOne(self):
    self.TestModel.createTable()

  def testTwo(self):
    self.TestModel.createTable()

if __name__ == '__main__':
  # import logging
  # logging.basicConfig(level=logging.DEBUG)
  unittest.main()


from django.http import QueryDict, HttpResponseRedirect
from django.views.generic import TemplateView
from django.contrib.sites.models import Site

from search.models import SearchPage
from aids.views import SearchView, AdvancedSearchView, AidDetailView
from alerts.views import AlertCreate

class MinisiteMixin:
    """Common code for all minisite urls.

    For every minisite view, we need to fetch the actual SearchPage object and
    inject it into the template.

    Note: there are several ways access the search page, hence there are
    several ways to access the SearchPage slug.

     1/ https://aides-territoires.beta.gouv.fr/recherche/<partner>/
     2/ https://<partner>.aides-territoires.beta.gouv.fr/
     3/ https://aides.<partner>.com (completely custom domain)

    In the fist case, we get the SearchPage's slug from the url parameter;
    Second case, from the subdomain;
    Third case, it is passed by a custom http header set in Nginx's settings.
    """

    def get(self, request, *args, **kwargs):
        self.search_page = self.get_search_page()

        if self.search_page is None:
            # Here, we have a bit of a problem.
            # If the SearchPage object is not found, we can't display a 404
            # error, because the 404 template needs a SearchPage to be
            # displayed.
            # Hence, the best way is to redirect to a sensible url.
            # Those errors are mainly caused by very old links with invalid
            # urls existing in the wild,
            # e.g https://www.aides-territoires.beta.gouv.fr
            return HttpResponseRedirect(self.get_redirection_url())

        return super().get(request, *args, **kwargs)

    def get_redirection_url(self):
        """What url to redirect to in case of missing SearchPage object.

        For now, just redirect to the main site's homepage.
        """
        site = Site.objects.get_current()
        url = 'https://{domain}'.format(domain=site.domain)
        return url

    def get_search_page(self):
        """Get the custom page from url."""

        HEADER = 'X-Minisite-Name'

        if 'search_slug' in self.kwargs:
            page_slug = self.kwargs.get('search_slug')
        elif HEADER in self.request.headers:
            page_slug = self.request.headers[HEADER]
        else:
            host = self.request.get_host()
            page_slug = host.split('.')[0]

        qs = SearchPage.objects.filter(slug=page_slug)
        try:
            obj = qs.get()
        except qs.model.DoesNotExist:
            obj = None
        return obj

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['search_page'] = self.search_page
        context['site_url'] = self.request.build_absolute_uri('').rstrip('/')
        return context

class SiteHome(MinisiteMixin, SearchView):
    """A static search page with admin-customizable content."""

    template_name = 'minisites/search_page.html'

    def get_form_kwargs(self):
        """Set the data passed to the form.

        If no data was provided by the user, then we use the initial
        querystring provided by admins.

        If the form was submitted, the GET values are set, we use those
        instead.
        """
        initial_data = QueryDict(
            self.search_page.search_querystring, mutable=True)
        user_data = self.request.GET.copy()
        user_data.pop('page', None)
        user_data.pop('integration', None)
        data = user_data or initial_data
        kwargs = super().get_form_kwargs()
        kwargs['data'] = data
        return kwargs

    def get_form(self, form_class=None):
        form = super().get_form(form_class)

        if self.search_page.available_categories:
            categories_qs = self.search_page \
                .available_categories \
                .select_related('theme')
            form.fields['categories'].queryset = categories_qs

        return form

class SiteSearch(MinisiteMixin, AdvancedSearchView):
    """The full search form."""

    template_name = 'minisites/advanced_search.html'

class SiteAid(MinisiteMixin, AidDetailView):
    """The detail page of a single aid."""

    template_name = 'minisites/aid_detail.html'

class SiteAlert(MinisiteMixin, AlertCreate):
    pass

class SiteLegalMentions(MinisiteMixin, TemplateView):
    template_name = 'minisites/legal_mentions.html'

class Error(MinisiteMixin, TemplateView):
    template_name = 'minisites/404.html'
    status_code = 404

    def render_to_response(self, context, **response_kwargs):
        response_kwargs['status'] = self.status_code
        return super().render_to_response(context, **response_kwargs)


import unittest
import sys
import os

current_path = os.path.dirname(os.path.abspath(__file__))
sys.path.append(current_path)

from classes.githubrepos import GithubRepos

class GithubReposTest(unittest.TestCase):
    def setUp(self):
        self.github_repos = GithubRepos()
        self.response_dict = self.github_repos.display_popular_repos()

    def test_response_dict_is_a_dictionary(self):
        self.assertTrue(type(self.response_dict) is dict,
        "response dict should be a dictionary")

if __name__ == '__main__':
    unittest.main()


import socket

sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)  # AF_INET - IP протокол версии 4, SOCK_STREAM протокол TCP
sock.bind(('127.0.0.1', 8888))  # резервируем порт
sock.listen(5)  # размер очереди, сколько соединений мы хотим слушать

while True:
    try:
        client, addr = sock.accept()
    except KeyboardInterrupt:
        sock.close()
        break
    else:
        result = client.recv(1024)
        client.close()
        print('Message', result.decode('utf-8'))


'''
 Sort the jobs by start time, (NOT end time).

    Iterate through the jobs, use a heap to maintain all the job schedules,
    which are represented by (end, totalProfit) entries.
    At each iteration, pop out all the job schedules that are compatible with the next
    job, and keep the 'ans' updated to the max totalProfit we can achieve from all schedules.
    At the end of each iteration, we add the job and its profit as our latest schedule.

    A key is that, as we process the next job, 'ans' will keep track of the max profit from
    the past schedules that are compatible with the next job, because the next job has a
    greater-or-equal start time, as we sorted the jobs that way.

    We end the iteration with a list of schedules that are mutually incompatible, and we find
    the best profit out of them
'''
class Solution:
    def jobScheduling(self, startTime, endTime, profit):
        # Sort job by start time, this ensures
        # job at i <= i +1
        jobs = sorted(zip(startTime,endTime,profit) , key=lambda x : x[0])

        hp = []

        total = 0

        for s,e,p in jobs:
            # If heap exists and the value
            # of the heap end time is <= start
            # ie,jobs dont clash
            # Since jobs is sorted and as we go ahead
            # s will increase that means if old job ended before current start.
            while hp and hp[0][0] <= s:
                popped = heappop(hp)
                # Pick the higher profit from non clashing job
                total = max(total,popped[1])

            # Clashing job end time and profit stored here
            heappush(hp,(e,p+total))

        # While clashing jobs exist
        # Pick max of clashing job and current total
        while hp:
            popped = heappop(hp)
            total = max(total,popped[1])

        return total


import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output

import pandas as pd

df = pd.read_csv('https://raw.githubusercontent.com/a-camarillo/NFL-running-backs-2019/master/Data%20Cleaning/rushing_stats_2019')

app = dash.Dash(__name__)

app.layout = html.Div([
    html.Div([
        dcc.Graph(
            id='graph-with-slider',
            clickData={'points':[{'text': 'Dalvin Cook'}]}),
        dcc.Slider(
         id='week-slider',
         min=df['week'].min(),
         max=df['week'].max(),
         value=df['week'].min(),
         marks={str(week): 'Week {}'.format(week) for week in df['week'].unique()},
            step=None
        )
    ]),
    html.Div([
        dcc.Graph(id='line-graph')
    ])
])

@app.callback(
    Output('graph-with-slider', 'figure'),
    [Input('week-slider', 'value')])
def update_figure(selected_week):
    filtered_df = df[df.week == selected_week]
    traces = []
    for i in filtered_df.player.unique():
        df_by_player = filtered_df[filtered_df['player'] == i]
        traces.append(dict(
            x=df_by_player['tot_yds'],
            y=df_by_player['tot_att'],
            text=df_by_player['player'],
            mode='markers',
            opacity=0.7,
            marker={
                'size': 15,
                'line' : {'width': 0.5, 'color': 'white'}
            },
            name=i
        ))
    return {
        'data': traces,
        'layout':dict(
            xaxis={'type':'scatter','title':'Yards'},
            yaxis={'type':'scatter','title':'Attempts'},
            hovermode='closest',
            transition = {'duration': 500}
        )
    }

@app.callback(
    Output('line-graph', 'figure'),
    [Input('graph-with-slider', 'clickData')])
def update_linegraph(clickData):
    player_name = clickData['points'][0]['text']
    player_df = df[df.player == player_name]
    return {
        'data':[dict(
            x=player_df['week'],
            y=player_df['yds']
        )],
        'layout': dict(
            xaxis={'title':'Week'},
            yaxis={'title':'Yards'},
            title=player_name
    )
}

if __name__ == '__main__':
    app.run_server(debug=True)


# Cho số tự nhiên n. Hãy tính số chữ số 0 tận cùng của n! (n! (hay còn gọi là n giai thừa) là tích các số tự nhiên liên tiếp từ 1 tới n)
# Đầu vào: integer n
# 1 <= n <= 10^12

def numberZeroDigits(n):
    count5 = 0
    i = 1
    while n!=0:
        count5 += n//5
        n //= 5

    return count5


from flask import current_app as app
from flask import jsonify, request, url_for
from . import api
from ..models import Shop, Offer

@api.route('/shops/<int:id>')
def get_shop(id):
	shop = Shop.query.get_or_404(id)
	return jsonify(shop.to_json())

@api.route('/shops/<int:id>/offers/')
def get_shop_offers(id):
	shop = Shop.query.get_or_404(id)
	page = request.args.get('page', 1, type=int)
	pagination = shop.offers.order_by(Offer.timestamp.desc()).paginate(page, per_page=app.config['PROSFORES4U_POSTS_PER_PAGE'], error_out=False)
	offers = pagination.items
	prev = None
	if pagination.has_prev:
		prev = url_for('api.get_shop_offers', id=id, page=page-1)
	next = None
	if pagination.has_next:
		next = url_for('api.get_shop_offers', id=id, page=page+1)
	return jsonify({
		'offers': [offer.to_json() for offer in offers],
		'prev': prev,
		'next': next,
		'count': pagination.total
	})


import random
import numpy as np

from utils.constants import *
from utils.strings import *

class Environment:
    '''Exchange Simulator for Bitcoin based upon per minute historical prices'''

    def __init__(self, logger, config, price_blocks):
        self.logger = logger
        self.action_size = 3
        self.history_length = config[HISTORY_LENGTH]
        self.horizon = config[HORIZON]

        self.price_blocks = price_blocks

        self.action_dict = {
            0: NEUTRAL,
            1: LONG,
            2: SHORT
        }

    def new_random_episode(self, history):
        self.liquid, self.borrow, self.long, self.short = 0., 0., 0, 0
        self.timesteps = 0

        block_index = random.randint(0, len(self.price_blocks) - 1)
        self.historical_prices = self.price_blocks[block_index]

        self.current = random.randint(self.history_length,
                                        len(self.historical_prices) - self.horizon)
        history.set_history(self.historical_prices[self.current - self.history_length:self.current])

    def act(self, action):
        price = self.historical_prices[self.current][0]

        if self.action_dict[action] is LONG:
            self.long = self.long + 1
            self.borrow = self.borrow + price

        elif self.action_dict[action] is SHORT:
            self.short = self.short + 1
            self.liquid = self.liquid + price

        self.timesteps = self.timesteps + 1
        if self.timesteps is not self.horizon:
            return self.historical_prices[self.current], 0, False
        else:
            reward = self.liquid - self.borrow + \
                        (self.long - self.short) * price
            return self.historical_prices[self.current], reward, True


# -*- coding:utf-8 -*-

from freetime.util import log as ftlog
from hall.servers.common.base_http_checker import BaseHttpMsgChecker
from poker.protocol import router, runhttp
from poker.protocol.decorator import markHttpHandler, markHttpMethod
from freetime.entity.msg import MsgPack
from majiang2.entity.quick_start import MajiangCreateTable
from queshou.entity.configure.conf import GAMEID

@markHttpHandler
class MJAdmin(BaseHttpMsgChecker):

    def __init__(self):
        super(MJAdmin, self).__init__()

    def _check_param_roomId(self, key, params):
        roomId = runhttp.getParamInt(key, -1)
        if isinstance(roomId, int) and roomId >= 0:
            return None, roomId
        return None, 0

    def _check_param_tableId(self, key, params):
        tableId = runhttp.getParamInt(key, -1)
        if isinstance(tableId, int) and tableId >= 0:
            return None, tableId
        return None, 0

    @markHttpMethod(httppath='/queshou/clear_table')
    def clearTable(self, roomId, tableId):
        ftlog.debug('MJAdmin.clearTable roomId:', roomId, ' tableId:', tableId)

        mo = MsgPack()
        mo.setCmd('table_manage')
        mo.setAction('clear_table')
        mo.setParam('roomId', roomId)
        mo.setParam('tableId', tableId)
        router.sendTableServer(mo, roomId)
        return {'info': 'ok', 'code': 0}

    @markHttpMethod(httppath='/queshou/kick_user')
    def kickUser(self, userId, roomId, tableId):
        ftlog.debug('MJAdmin.kickUser roomId:', roomId, ' tableId:', tableId, ' userId:', userId)

        mo = MsgPack()
        mo.setCmd('table_manage')
        mo.setAction('leave')
        mo.setParam('roomId', roomId)
        mo.setParam('tableId', tableId)
        mo.setParam('userId', userId)
        router.sendTableServer(mo, roomId)
        return {'info': 'ok', 'code': 0}

    @markHttpMethod(httppath='/queshou/power_test')
    def powerTest(self, userId):
        ftlog.debug('MJAdmin.powerTest userId:', userId)
        roomId, checkResult = MajiangCreateTable._chooseCreateRoom(userId, GAMEID, 'queshou', 3)
        ftlog.debug('MajiangCreateTable._chooseCreateRoom roomId:', roomId, ' checkResult:', checkResult)
        msg = MsgPack()
        msg.setCmdAction("room", "create_table")
        msg.setParam("roomId", roomId)
        msg.setParam("gameId", GAMEID)
        msg.setParam("userId", userId)
        msg.setParam("itemParams", {"sanQiBian":1,"playerType":3,"cardCount":1,"chunJia":0,"guaDaFeng":0,"hongZhongBao":0})
        msg.setParam('needFangka', 0)
        ftlog.debug('MajiangCreateTable._chooseCreateRoom send message to room:', msg)

        router.sendRoomServer(msg, roomId)
        return {'info': 'ok', 'code': 0}

    @markHttpMethod(httppath='/queshou/check_table_tiles')
    def checkTableTiles(self, roomId, tableId):
        ftlog.debug('MJAdmin.checkTableTiles roomId:', roomId, ' tableId:', tableId)

        mo = MsgPack()
        mo.setCmd('table_manage')
        mo.setAction('tableTiles')
        mo.setParam('roomId', roomId)
        mo.setParam('tableId', tableId)
        router.sendTableServer(mo, roomId)
        return {'info': 'ok', 'code': 0}


import json
import requests

class GCMSender:
    URL_SEND = 'https://gcm-http.googleapis.com/gbm/send'

    def __init__(self, api_key):
        global headers
        headers = {
            'Content-Type': 'application/json',
            'Authorization': 'key=' + api_key
        }

    def send(self, message, timestamp, reg_token):
        global headers, URL_SEND
        payloads = {
            "data": {
                "message": message,
                "timestamp": timestamp
            },
            "to": reg_token
        }
        try:
            r = requests.post(url=URL_SEND, data=json.dumps(payloads), header=headers)
        except Exception as e:
            # Cannot complete POST requests
            pass


import sys
import os
import subprocess
import uuid
import logging.config
import collections
import subprocess
import tldextract
from subprocess import call, run, PIPE, check_output, STDOUT
import platform
import wslpy as wp

log = logging.getLogger('postgres')

def measure_dns(website, har, dns_type, resolver):
    operation_system = platform.system()
    domains = get_unique_domains(har)
    domains_filename = "domains.txt"
    write_domains(domains, domains_filename)

    try:
        if dns_type == 'dns':
            dns_type = 'do53'

        if operation_system == "Linux":
            cmd = ["dns-timing/dns-timing", dns_type, resolver, domains_filename]
            output = check_output(cmd, stderr=STDOUT)

        elif operation_system == "Windows":
            cmd = "dns-timing/dns-timing {0} {1} {2}".format(dns_type, resolver, domains_filename)
            project_path = os.getcwd()
            project_path = project_path.split("\\")
            project_path[0] = project_path[0][:-1].lower()
            project_path = "/".join(project_path)
            project_path = "cd ../../mnt/" + project_path
            run_input = project_path + " && " + cmd
            output = run("ubuntu", shell=True, stdout=PIPE, input=run_input, encoding='ascii')

        output = output.decode('utf-8')
        all_dns_info = parse_output(output, website, domains)
        os.remove(domains_filename)
        return all_dns_info
    except Exception as e:
        err = 'Error getting DNS timings: website {0}, dns_type {1}, ' \
              'resolver {2}'
        os.remove(domains_filename)
        log.error(err.format(website, dns_type, resolver))
    return None

def parse_output(output, website, domains):
    # Initialize the dict with all domains in the HAR
    all_dns_info = {}
    for domain in domains:
        all_dns_info[domain] = {'response_time': 0.,
                                'response_size': 0,
                                'error': 0}

    # If there's no output from the DNS tool, return immediately
    if not output:
        return all_dns_info

    # For each domain in the HAR, record DNS response time and size
    try:
        lines = output.splitlines()
        for line in lines:
            status, domain, response_time, size_or_error = line.split(',', 4)
            if status == "ok":
                response_size = int(size_or_error)
                error = None
            else:
                response_size = None
                error = int(size_or_error)

            all_dns_info[domain] = {'response_time': float(response_time),
                                    'response_size': response_size,
                                    'error': error}
    except Exception as e:
        err = 'Error parsing DNS output for website {0}: {1}'
        log.error(err.format(website, e))
    return all_dns_info

def get_unique_domains(har):
    if not har:
        return []

    log = har["log"]
    if "entries" not in log:
        return []
    entries = log["entries"]

    if len(entries) == 1:
        return []

    domains = []
    for entry in entries:
        # If a DNS request was made, record the timings
        if "request" not in entry:
            continue
        request = entry["request"]

        if "url" not in request:
            continue
        url = request["url"]

        ext = tldextract.extract(url)
        if ext.subdomain:
            fqdn = ext.subdomain + "." + ext.domain + "." + ext.suffix
        else:
            fqdn = ext.domain + "." + ext.suffix
        domains.append(fqdn)
    return list(set(domains))

def write_domains(domains, domains_filename):
    with open(domains_filename, 'w') as f:
        for d in domains:
            f.write("{0}\n".format(d))
        f.close()


# Copyright (c) 2011 Mick Thomure
# All rights reserved.
#
# Please see the file COPYING in this distribution for usage terms.

#
# Functions for dealing with N-dimensional arrays.
#

import Image
import math
import numpy

ACTIVATION_DTYPE = numpy.float32

def ArgMax(array):
  """Short-hand to find array indices containing the maximum value."""
  return numpy.transpose(numpy.nonzero(array == array.max()))

def ArgMin(array):
  """Short-hand to find array indices containing the minimum value."""
  return numpy.transpose(numpy.nonzero(array == array.min()))

def ScaleUnitNorm(x):
  """Scale elements of vector (in place), such that result has unit norm."""
  norm = numpy.linalg.norm(x)
#  norm = math.sqrt((x**2).sum())
  if norm == 0:
    x[:] = 1.0 / math.sqrt(x.size)
  else:
    x /= norm
  return x

def ArrayToGreyscaleImage(array, normalize = True):
  """Create a greyscale image from a 2D numpy array. Assumes range of input
     values contains 0."""
  if array.dtype != numpy.float32:
    array = array.astype(numpy.float32)
  if len(array.shape) > 2:
    # Stack bands vertically.
    array = array.reshape((-1, array.shape[-1]))
  if normalize:
    # Normalize array values to lie in [0, 255].
    max_val = max(abs(array.min()), array.max())
    # Map values to lie in [-.5, .5]
    array = array / (2 * max_val)       # makes copy of input
    # Map to [0, 1]
    array += 0.5
    # Map to [0, 255]
    array *= 255
  # Convert to unsigned chars
  array = numpy.asarray(array, dtype = numpy.uint8)
  return Image.fromarray(array, 'L')

def ArrayToRGBImage(array):
  return Image.fromarray(array, 'RGB')

def ArrayListToVector(arrays):
  """Convert list of numpy arrays to a single numpy vector."""
  assert len(arrays) > 0
  out_size = sum(a.size for a in arrays)
  out = numpy.empty((out_size,), arrays[0].dtype)
  offset = 0
  for a in arrays:
    out[offset : offset + a.size] = a.flat
    offset += a.size
  return out

def PadArray(data, out_shape, cval):
  """Pad the border of an array with a constant value."""
  out_shape = numpy.array(out_shape)
  in_shape = numpy.array(data.shape)
  result = numpy.empty(out_shape)
  result[:] = cval
  begin = ((out_shape - in_shape) / 2.0).astype(int)
  result[ [ slice(b, e) for b, e in zip(begin, begin + in_shape) ] ] = data
  return result

def CropArray(data, out_shape):
  """Remove the border of an array.
  data -- input array
  out_shape -- shape of central region to return
  RETURNS view of the central region of input array.
  """
  assert numpy.all(numpy.array(data.shape) >= numpy.array(out_shape))
  out_shape = numpy.array(out_shape)
  in_shape = numpy.array(data.shape)
  begin = ((in_shape - out_shape) / 2.0).astype(int)
  return data[ [ slice(b, e) for b, e in zip(begin, begin + out_shape) ] ]


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Initializes a new model for a corpus LSTM with NTM memory.
"""

from __future__ import print_function

import cPickle
import numpy as np

from os import path

from nnet import init_params

def main(in_vocab_file, out_model_file, hidden_state_size, num_mem_rows, num_mem_cols,
         pickle_protocol, rand_seed):
  """Main method called with command-line args."""

  np.random.seed(rand_seed)

  with open(in_vocab_file, "rb") as f_in:
    token_ids = cPickle.load(f_in)
    vocab_size = len(token_ids)

  model_params = init_params(vocab_size, hidden_state_size, num_mem_rows, num_mem_cols)

  with open(out_model_file, "wb") as f_out:
    cPickle.dump(model_params, f_out, pickle_protocol)

if __name__ == "__main__":
  import argparse

  parser = argparse.ArgumentParser(description=__doc__)

  parser.add_argument("in_vocab_file", help="Binary file storing pickled vocab")
  parser.add_argument("out_model_file", help="Binary file storing pickled model weights")

  parser.add_argument("--hidden_state_size", default=1000, type=int, metavar="n",
                      help="Number of LSTM hidden state nodes (default: 1000)")

  parser.add_argument("--num_mem_rows", default=500, type=int, metavar="n",
                      help="Number of NTM memory locations (default: 500)")
  parser.add_argument("--num_mem_cols", default=300, type=int, metavar="n",
                      help="Size of NTM memory vectors (default: 300)")

  parser.add_argument("--pickle_protocol", default=-1, type=int, metavar="p",
                      help="Pickle protocol to use for saving model data (default: -1)")

  parser.add_argument("--rand_seed", default=123, type=int, metavar="s",
                      help="Random seed to use for initialization (default: 123)")

  args = parser.parse_args()

  main(path.abspath(args.in_vocab_file), path.abspath(args.out_model_file),
       args.hidden_state_size, args.num_mem_rows, args.num_mem_cols,
       args.pickle_protocol, args.rand_seed)


import argparse

if __name__=='__main__':
    ###### argument parse
    parser = argparse.ArgumentParser(
        prog='FormatConverter',
        )

    subparsers = parser.add_subparsers(title = 'subcommands',dest = "subcommand_name")

    # argparse for FastqToFasta
    parser_a = subparsers.add_parser('outfmt5to6',
                                     help = 'convert blast results from outfmt 5 to 6',
                                     description = 'convert blast results from outfmt 5 to 6')

    parser_a.add_argument('input_file', type=str, help='input file with outfmt 5')
    parser_a.add_argument('output_file', type=str, help='output file path')

    # argparse for outfmt5To6
    parser_a = subparsers.add_parser('outfmt5To6',
                                     help = 'convert blast results from outfmt 5 to 6',
                                     description = 'convert blast results from outfmt 5 to 6')

    parser_a.add_argument('input_file', type=str, help='input file with outfmt 5')
    parser_a.add_argument('output_file', type=str, help='output file path')

    args = parser.parse_args()
    args_dict = vars(args)

    #---------------------------------------------------------
    #### ccommand detail

    #outfmt5to6
    if args_dict["subcommand_name"] == "outfmt5To6":
        input_file = args.input_file
        output_file = args.output_file
        import lib.file_parser.outfmt5_parser as outfmt5
        output_dict = outfmt5.outfmt5_read(input_file)
        with open(output_file, 'w') as f:
            for i in outfmt5.outfmt6_write(output_dict):
                f.write(i+"\n")

    if args_dict["subcommand_name"] == "b":
        print(args.baz)