# -*- coding: UTF-8 -*-
from __future__ import print_function

import atexit
import os
import os.path
import math
import sys

from colorama import init, Fore, Style
import copy
init(autoreset=True)

from settings import logger,SELF_ENSEMBLE
from games import keypress
from games.board import Board
from .transition import Transition
import games.boardUtils as bU

class Game(object):
    """
    A 2048 game
    """

    __dirs = {
        keypress.UP:      Board.UP,
        keypress.DOWN:    Board.DOWN,
        keypress.LEFT:    Board.LEFT,
        keypress.RIGHT:   Board.RIGHT,
        keypress.SPACE:   Board.PAUSE,
    }

    __is_windows = os.name == 'nt'

    COLORS = {
        2:    Fore.GREEN,
        4:    Fore.BLUE + Style.BRIGHT,
        8:    Fore.CYAN,
        16:   Fore.RED,
        # Don't use MAGENTA directly; it doesn't display well on Windows.
        # see https://github.com/bfontaine/games/issues/24
        32:  Fore.MAGENTA + Style.BRIGHT,
        64:   Fore.CYAN,
        128:  Fore.BLUE + Style.BRIGHT,
        256:  Fore.MAGENTA + Style.BRIGHT,
        512:  Fore.GREEN,
        1024: Fore.RED,
        2048: Fore.YELLOW,
        # just in case people set an higher goal they still have colors
        4096: Fore.RED,
        8192: Fore.CYAN,
    }

    # see Game#adjustColors
    # these are color replacements for various modes
    __color_modes = {
        'dark': {
            Fore.BLUE: Fore.WHITE,
            Fore.BLUE + Style.BRIGHT: Fore.WHITE,
        },
        'light': {
            Fore.YELLOW: Fore.BLACK,
        },
    }

    SCORES_FILE = '%s/.games.scores' % os.path.expanduser('~')
    STORE_FILE = '%s/.games.store' % os.path.expanduser('~')

    def __init__(self, scores_file=SCORES_FILE, colors=COLORS,
                 store_file=STORE_FILE, clear_screen=True,
                 mode=None, azmode=False, **kws):
        """
        Create a new game.
            scores_file: file to use for the best score (default
                         is ~/.games.scores)
            colors: dictionnary with colors to use for each tile
            store_file: file that stores game session's snapshot
            mode: color mode. This adjust a few colors and can be 'dark' or
                  'light'. See the adjustColors functions for more info.
            other options are passed to the underlying Board object.
        """
        self.board = Board(**kws)
        self.score = 0
        self.scores_file = scores_file
        self.store_file = store_file
        self.clear_screen = clear_screen

        self.__colors = colors
        self.__azmode = azmode

        self.loadBestScore()
        self.adjustColors(mode)

    def init_state(self):
        self.board=Board()
        self.score = 0

    def adjustColors(self, mode='dark'):
        """
        Change a few colors depending on the mode to use. The default mode
        doesn't assume anything and avoid using white & black colors. The dark
        mode use white and avoid dark blue while the light mode use black and
        avoid yellow, to give a few examples.
        """
        rp = Game.__color_modes.get(mode, {})
        for k, color in self.__colors.items():
            self.__colors[k] = rp.get(color, color)

    def loadBestScore(self):
        """
        load local best score from the default file
        """
        try:
            with open(self.scores_file, 'r') as f:
                self.best_score = int(f.readline(), 10)
        except:
            self.best_score = 0
            return False
        return True

    def saveBestScore(self):
        """
        save current best score in the default file
        """
        if self.score > self.best_score:
            self.best_score = self.score
        try:
            with open(self.scores_file, 'w') as f:
                f.write(str(self.best_score))
        except:
            return False
        return True

    def incScore(self, pts):
        """
        update the current score by adding it the specified number of points
        """
        self.score += pts
        if self.score > self.best_score:
            self.best_score = self.score

    def readMove(self):
        """
        read and return a move to pass to a board
        """
        k = keypress.getKey()
        return Game.__dirs.get(k)

    def store(self):
        """
        save the current game session's score and data for further use
        """
        size = self.board.SIZE
        cells = []

        for i in range(size):
            for j in range(size):
                cells.append(str(self.board.getCell(j, i)))

        score_str = "%s\n%d" % (' '.join(cells), self.score)

        try:
            with open(self.store_file, 'w') as f:
                f.write(score_str)
        except:
            return False
        return True

    def restore(self):
        """
        restore the saved game score and data
        """

        size = self.board.SIZE

        try:
            with open(self.store_file, 'r') as f:
                lines = f.readlines()
                score_str = lines[0]
                self.score = int(lines[1])
        except:
            return False

        score_str_list = score_str.split(' ')
        count = 0

        for i in range(size):
            for j in range(size):
                value = score_str_list[count]
                self.board.setCell(j, i, int(value))
                count += 1

        return True

    def clearScreen(self):
        """Clear the console"""
        if self.clear_screen:
            os.system('cls' if self.__is_windows else 'clear')
        else:
            print('\n')

    def hideCursor(self):
        """
        Hide the cursor. Don't forget to call ``showCursor`` to restore
        the normal shell behavior. This is a no-op if ``clear_screen`` is
        falsy.
        """
        if not self.clear_screen:
            return
        if not self.__is_windows:
            sys.stdout.write('\033[?25l')

    def showCursor(self):
        """Show the cursor."""
        if not self.__is_windows:
            sys.stdout.write('\033[?25h')

    def compute_transition(self,state,action):
        """Return transition with after state and reward"""
        after=copy.deepcopy(state)
        rw=after.move(action,add_tile=False)
        tran=Transition(state,action,after,rw)
        return tran

    def get_actions(self,state):
        actions=[]
        for i in range(1,5):
            new_state = copy.deepcopy(state)
            _ = new_state.move(i,add_tile=False)
            if state.cells != new_state.cells :
                actions.append(i)
        return actions

    def get_best_after_state(self,state,vfunction):
        """Return transition leading best after state"""
        bestv= float("-inf")
        besttran=None
        actions=self.get_actions(state)
        for act in actions:
            tran=self.compute_transition(state,act)
            v=tran.reward+vfunction.get_value(tran.after_state.get_features())
            if v>bestv:
                bestv=v
                besttran=tran
        return besttran

    def get_best_value(self,state,vfunction):
        """Return actual reward leading best next after state. To update after state value"""
        bestv= float("-inf")
        actions=self.get_actions(state)
        for act in actions:
            tran=self.compute_transition(state,act)
            v=tran.reward+vfunction.get_value(tran.after_state.get_features())
            if v>bestv:
                bestv=v
        return bestv

    def TD_after_state_learn(self,vfunction,learning_rate):
        """move based on after state value"""

        margins = {'left': 4, 'top': 4, 'bottom': 4}
        # atexit.register(self.showCursor)
        self.init_state()
        # logger.debug('----updated %s'%str(vfunction.tuples()[0].get_weights()[:5]))
        # try:
        #     self.hideCursor()
        while not(self.board.won() or not self.board.canMove()):
            # self.clearScreen()
            # print(self.__str__(margins=margins))
            tran=self.get_best_after_state(self.board,vfunction)
            nexts=copy.deepcopy(tran.after_state)
            nexts.addTile()
            correctv=0
            if not(nexts.won() or not nexts.canMove()):
                correctv+=self.get_best_value(nexts,vfunction)
            vfunction.update(tran.after_state.get_features(),correctv,learning_rate)
            self.board=nexts
            self.incScore(tran.reward)

        # except KeyboardInterrupt:
        #     self.saveBestScore()
        #     exit(-1)

        self.saveBestScore()
        if self.board.won():
            logger.info('You won!')
        # print('You won!' if self.board.won() else 'Game Over')

    def TD_after_state_learn_ensemble(self,vfunction,learning_rate):
        """move based on after state value"""
        margins = {'left': 4, 'top': 4, 'bottom': 4}
        self.init_state()
        while not(self.board.won() or not self.board.canMove()):
            # self.clearScreen()
            # print(self.__str__(margins=margins))
            tran=self.get_best_after_state_ensemble(self.board,vfunction)
            nexts=copy.deepcopy(tran.after_state)
            nexts.addTile()
            correctv=0
            if not(nexts.won() or not nexts.canMove()):
                correctv+=self.get_best_value_ensemble(nexts,vfunction)
            vfunction.update(tran.after_state.get_features(),correctv,learning_rate)
            self.board=nexts
            self.incScore(tran.reward)

        self.saveBestScore()
        if self.board.won():
            logger.info('You won!')

    def get_best_after_state_ensemble(self,state,vfunction):
        """Return transition leading best after state"""
        bestv= float("-inf")
        besttran=None
        actions=self.get_actions(state)
        for act in actions:
            tran=self.compute_transition(state,act)
            v=0
            for i in range(SELF_ENSEMBLE):
                rota_cells=bU.nth_rotate(tran.after_state.cells,i)
                v+=vfunction.get_value(bU.features(rota_cells))
            v/=SELF_ENSEMBLE
            v+=tran.reward
            if v>bestv:
                bestv=v
                besttran=tran
        return besttran

    def get_best_value_ensemble(self,state,vfunction):
        """Return actual reward leading best next after state. To update after state value"""
        bestv= float("-inf")
        actions=self.get_actions(state)
        for act in actions:
            tran=self.compute_transition(state,act)
            v=0
            for i in range(SELF_ENSEMBLE):
                rota_cells=bU.nth_rotate(tran.after_state.cells,i)
                v+=vfunction.get_value(bU.features(rota_cells))
            v/=SELF_ENSEMBLE
            v+=tran.reward
            if v>bestv:
                bestv=v
        return bestv

    def play_after_learn_ensemble(self,vfunction):
        margins = {'left': 4, 'top': 4, 'bottom': 4}
        self.init_state()
        max_tile=0
        while not(self.board.won() or not self.board.canMove()):
            tran=self.get_best_after_state_ensemble(self.board,vfunction)
            nexts=copy.deepcopy(tran.after_state)
            max_tile=max(max_tile,self.get_max_tile(max_tile,nexts))
            nexts.addTile()
            self.board=nexts
            self.incScore(tran.reward)

        if self.board.won():
            logger.info('You won!')
        return self.score,max_tile

    def play_after_learn(self,vfunction):
        margins = {'left': 4, 'top': 4, 'bottom': 4}
        # atexit.register(self.showCursor)
        # logger.debug('----evaluate func %s'%str(vfunction.tuples()[0].get_weights()[:5]))
        self.init_state()
        max_tile=0
        # try:
        #     self.hideCursor()
        while not(self.board.won() or not self.board.canMove()):
            # self.clearScreen()
            # print(self.__str__(margins=margins))
            tran=self.get_best_after_state(self.board,vfunction)
            nexts=copy.deepcopy(tran.after_state)
            max_tile=max(max_tile,self.get_max_tile(max_tile,nexts))
            nexts.addTile()
            self.board=nexts
            self.incScore(tran.reward)
        # except KeyboardInterrupt:
        #     self.saveBestScore()
        #     exit(-1)

        if self.board.won():
            logger.info('You won!')
        return self.score,max_tile

    def get_max_tile(self,max_tile,state):
        maxt=max_tile
        for r in range(state.SIZE):
            for c in range(state.SIZE):
                maxt=max(maxt,state.cells[r][c])
        return maxt

    def loop(self):
        """
        main game loop. returns the final score.
        """
        pause_key = self.board.PAUSE
        margins = {'left': 4, 'top': 4, 'bottom': 4}

        atexit.register(self.showCursor)

        try:
            self.hideCursor()
            while True:
                self.clearScreen()
                print(self.__str__(margins=margins))
                if self.board.won() or not self.board.canMove():
                    break
                m = self.readMove()

                if (m == pause_key):
                    self.saveBestScore()
                    if self.store():
                        print("Game successfully saved. "
                              "Resume it with `games --resume`.")
                        return self.score
                    print("An error ocurred while saving your game.")
                    return

                self.incScore(self.board.move(m))

        except KeyboardInterrupt:
            self.saveBestScore()
            return

        self.saveBestScore()
        print('You won!' if self.board.won() else 'Game Over')
        return self.score

    def getCellStr(self, x, y):  # TODO: refactor regarding issue #11
        """
        return a string representation of the cell located at x,y.
        """
        c = self.board.getCell(x, y)

        if c == 0:
            return '.' if self.__azmode else '  .'

        elif self.__azmode:
            az = {}
            for i in range(1, int(math.log(self.board.goal(), 2))):
                az[2 ** i] = chr(i + 96)

            if c not in az:
                return '?'
            s = az[c]
        elif c == 1024:
            s = ' 1k'
        elif c == 2048:
            s = ' 2k'
        else:
            s = '%3d' % c

        return self.__colors.get(c, Fore.RESET) + s + Style.RESET_ALL

    def boardToString(self, margins=None):
        """
        return a string representation of the current board.
        """
        if margins is None:
            margins={}
        b = self.board
        rg = range(b.size())
        left = ' '*margins.get('left', 0)
        s = '\n'.join(
            [left + ' '.join([self.getCellStr(x, y) for x in rg]) for y in rg])
        return s

    def __str__(self, margins=None):
        if margins is None:
            margins={}
        b = self.boardToString(margins=margins)
        top = '\n'*margins.get('top', 0)
        bottom = '\n'*margins.get('bottom', 0)
        scores = ' \tScore: %5d  Best: %5d\n' % (self.score, self.best_score)
        return top + b.replace('\n', scores, 1) + bottom


"""
Unittests for: pyin.core
"""

import sys

import pytest

import pyin.core
import tests._test_module

def test_single_expr():
    result = list(pyin.core.pmap("20 <= line <= 80", range(100)))
    assert len(result) == len(list(range(20, 81)))
    for item in result:
        assert 20 <= item <= 80

def test_with_map():
    result = list(pyin.core.pmap(
        "list(map(int, line.split('-')))", ['2015-01-01']))
    assert result == [[2015, 1, 1]]

@pytest.mark.parametrize("obj", [
    'map', 'reduce', 'op', 'it'
])
def test_scope(obj):
    """Make sure specific objects aren't removed from the scope."""
    for res in pyin.core.pmap(obj, 'word'):
        pass

def test_floating_point_division():
    result = next(pyin.core.pmap('5 / 3', ['']))
    assert isinstance(result, float)
    assert 1 < result < 2

@pytest.mark.parametrize("expression", [
    'return',  # Evaluates as a statement, not string.
    ''
])
def test_invalid_expression(expression):
    with pytest.raises(SyntaxError):
        next(pyin.core.pmap(expression, []))


import json

from .....execution import Execution
from .mutation_type import mutation

def from_string(value):
    if value is None:
        return None

    if value.startswith("{") and value.endswith("}"):
        return json.loads(value)

    if value.startswith("int:"):
        return int(value[4:])

    return value

@mutation.field("execute")
async def resolve_execute(
    obj,
    info,
    experiment,
    storage=None,
    engine=None,
    index=None,
    project=None,
    seed=None,
):
    execution = Execution(
        experiment=from_string(experiment),
        storage=from_string(storage),
        engine=from_string(engine),
        index=from_string(index),
        project=from_string(project),
        seed=from_string(seed),
    )
    execution.submit()
    return execution


from database_component import DataBase
from clustering_kmeans import ClusteringKMeans
from classifier_random_forest import ClassifierRandomForest
from logistic_regression import Logistic_Regression
from utils import select_option, show_select_project
import glob, os
import pandas as pd

'''
*   @author Evandro Luis da Rosa Fensterseifer Filho
'''

def read_csv():
    data = pd.read_csv("./data-all.csv", index_col=0)
    return data

# ARRUMAR PRA EXIBIR COLUNAS, LINHAS - CONJUNTOS E VALORES ESPECIFICOS
def show_csv(row=0, col=0):
    print(data)

def import_data_for_database(db):
    my_list = []
    os.chdir("./dataB")
    my_list = glob.glob("*.bson")

    for aux in my_list:
        name = aux.split(".")[0]
        db.insertIntoCollection(aux, name)
    os.chdir("../")

def select_project(model, data, db):
    my_data = show_select_project(data)
    model.get_similars(my_data, db)

if __name__ == '__main__':
    data = read_csv()
    db = DataBase()
    db.setDataBase("trabalho")

    import_data_for_database(db)

    choose = -1
    print("Selecione metodo <opcao numerica>")
    print("0 - KMeans (PCA)")
    print("1 - Random Forest")
    print("2 - Logistic Regression")
    print("3 - Show dataset")
    while choose > 3 or choose < 0:
        choose = int(input("Opcao => "))

    if(choose == 0):
        kmeans = ClusteringKMeans()
        option = select_option(choose)
        kmeans.executeKM(data, option)
        select_project(kmeans, data, db)
    elif(choose == 1):
        randomForest = ClassifierRandomForest()
        option = select_option(choose)
        randomForest.executeRF(data, option)
        select_project(randomForest, data, db)
    elif(choose == 2):
        logiReg = Logistic_Regression()
        option = select_option(choose)
        logiReg.executeLogReg(data)
        select_project(logiReg, data, db)
    elif(choose == 3):
        show_csv(data)

    print("Operacao finalizada")


"""Draw maps using folium."""

from IPython.core.display import HTML
import json
import folium
import functools
import random

#########################
# Exposed Functionality #
#########################

def draw_map(obj=None, **kwargs):
    """Draw a map with center & zoom containing all points and regions that
    displays points as circles and regions as polygons.

    obj -- a coordinate for the center, a MapRegion, or a MapPoint
    see help(Map) for more parameters to pass to draw_map
    """
    return get_map(obj, **kwargs).to_html()

def get_map(obj=None, **kwargs):
    """Get a map object to manipulate before drawing it. Use to_html() to draw.

    obj -- a coordinate for the center, a MapRegion, or a MapPoint
    see help(Map) for more parameters to pass to draw_map
    """
    return Map(location=obj, **kwargs).map()

class MapData:
    """A geoJSON string, object, or file."""

    _data = None

    def __init__(self, path_or_json_or_string = None):
        """Loads geoJSON"""
        jsons = path = string = path_or_json_or_string
        if not string:
            return
        if isinstance(jsons, (dict, list)):
            self._data = jsons
        try:
            self._data = json.loads(string)
        except ValueError:
            pass
        try:
            self._data = json.loads(open(path, 'r').read())
        except FileNotFoundError:
            pass
        if not self._data:
            raise MapError('MapData accepts a valid geoJSON object'
            'geoJSON string, or path to a geoJSON file')
        self._process_data()

    def _process_data(self):
        """Processes the data, and makes each list coordinates accessible via
        the ID of the feature.
        """
        self._features = self._data['features']
        self._data = {}
        for i, feature in enumerate(self._features):
            key, val = feature.get('id', i), MapRegion(
                locations=feature['geometry']['coordinates'],
                geo_formatted=True)
            self._data[key] = val
            setattr(self, key, val)

    def to_dict(self):
        return self._data.copy()

    def to_list(self):
        return list(self._data.values())

##################
# Implementation #
##################

def _map_to_defaults(defaults, params, excludes=[], includes=[]):
    """Squishes together two dictionaries, with one containing defaults."""
    rval = defaults.copy()
    rval.update(params)
    return {
        k: v for k, v in rval.items()
        if k not in excludes
        and k in (includes or rval.keys())
    }

def _hook(f):
    """Decorator for obj method, checks for and invokes PREFIX_before and
    PREFIX_after a method is called
    """

    def call_hook(self, template, *args, **kwargs):
        fn = getattr(self, template % f.__name__, None)
        if callable(fn):
            fn(*args, **kwargs)

    @functools.wraps(f)
    def method(self, *args, **kwargs):
        call_hook(self, '_before_%s', *args, **kwargs)
        rval = f(self, *args, **kwargs)
        call_hook(self, '_after_%s', *args, **kwargs)
        return rval

    return method

class ProgrammerError(Exception):
    """Thrown when programmers make a mistake"""

    message = ''
    prefix = 'Please report this to your instructors'

    def get_message(self):
        return '%s: %s' % (self.prefix, self.message)

class MapError(Exception):
    """Thrown when users make a mistake"""
    pass

######################
# Folium Abstraction #
######################

class MapEntity:
    """A base utility for all entities related to map

    "Attributes" are arguments that will be passed to the object's associated
    Folium method (called, the "mapper")
    - get and set "attributes", using map[prop]
    - after initializing the object, all properties will either be filled with
     a specified default or be set to None, to accept Folium's default

    All other data are pieces of data that you may wish to associate with the
    object.
    - all other data get and set, using map.prop
    """

    _init = {}                                  # params for initial call
    _defaults = {}                              # defaults for attrs and data
    _attributes = None                          # only data passed to mapper
    _allowed_attributes = []                    # allowed attributes
    _mapper = None                              # default function to map obj
    _folium = None                              # access to the map object
    _allowed_collections = (list, set, tuple)   # for list of locations

    @_hook
    def __init__(self, **kwargs):
        """Saves attributes, with defaults"""
        self._init, self._attributes = kwargs, {}
        self._attributes = _map_to_defaults(
            self._defaults, kwargs, includes=self._allowed_attributes)
        self.set(**_map_to_defaults(self._defaults, kwargs))

    def __setitem__(self, k, v):
        self._attributes[k] = v

    def __getitem__(self, k):
        return self._attributes[k]

    def __delitem__(self, k):
        del self._attributes[k]

    def set(self, override=False, **kwargs):
        """Transform args into properties"""
        for k, v in kwargs.items():
            attr = getattr(self, k, None)
            if callable(attr) and not override:
                raise ProgrammerError('"%s" is an existing method' % k)
            setattr(self, k, v)
        return self

    def get(self, *args):
        """Get a set of variables - use pairs to set getattr default"""
        get = lambda val: getattr(self, val[0], val[1]) \
            if isinstance(val, tuple) else getattr(self, val)
        return (get(var) for var in args)

    @_hook
    def map(self, map=None):
        """Maps this object, with its own attributes"""
        if not callable(self._mapper):
            self._mapper = getattr(map, self._mapper)
        self._folium = self._mapper(**self._attributes)
        return self

class Map(MapEntity):
    """
    Represents a map, potentially with MapPoints and MapRegions

    location or center -- lat-long pair at the center of the map
    width -- pixel int or percentage string
    height -- pixel int or percentage string
    tiles -- map tileset, or "theme"
    API_key -- for Cloudmade or Mapbox tiles
    max_zoom -- maximum zoom, default 18
    zoom_start -- starting zoom level
    points -- a list of MapPoints
    regions -- a list of MapRegions
    *pass any other kwarg to save it with the object

    >>> map = Map().map()
    >>> map.to_html()
    <IPython.core.display.HTML object>
    """

    _mapper = folium.Map

    _allowed_attributes = [
        'location',
        'width',
        'height',
        'tiles',
        'API_key',
        'max_zoom',
        'zoom_start',
    ]

    _defaults = {
        'location': [45.5244, -122.6699],
        'tiles': 'Stamen Toner',
        'zoom_start': 17,
        'width': 960,
        'height': 500,
        'points': [],
        'regions': []
    }

    @_hook
    def to_html(self):
        """Outputs the HTML that IPython will display."""
        map = self._folium
        map._build_map()
        map_html = map.HTML.replace('"', '"')
        return HTML('<iframe srcdoc="%s" '
                    'style="width: %spx; height: %spx; '
                    'border: none"></iframe>'
                    % (map_html, map.width, map.height))

    def _before_to_html(self, *args, **kwargs):
        """Called before to_html; add points and regions before mapping."""
        for point in self.points:
            point.map(self._folium)

        for region in self.regions:
            region.map(self._folium)

        # uncomment the following with new Folium release
        # if 'zoom_start' not in self._init:
        #     self._autofit(self.bounds)

    def _before_map(self, *args, **kwargs):
        """Autozoom map where appropriate."""
        if self.location:
            self._dynamic_arg('location')

        self.bounds = self._autobounds()

        if not self.location:
            self._autocenter(self.bounds)

        # TODO(Alvin): remove with new Folium release
        if 'zoom_start' not in self._init:
            self._autofit(self.bounds)

    def _autobounds(self):
        """Simple calculation for bounds."""
        bounds = {}

        def check(prop, compare, extreme, val):
            opp = min if compare is max else max
            bounds.setdefault(prop, val)
            bounds[prop] = opp(compare(bounds[prop], val), extreme)

        def bound_check(coord):
            if not coord:
                return
            if isinstance(coord, self._allowed_collections):
                if isinstance(coord[0], (int, float)):
                    long, lat = coord
                    check('max_lat', max, 90, lat)
                    check('min_lat', min, -90, lat)
                    check('max_long', max, 180, long)
                    check('min_long', min, -180, long)
                else:
                    [bound_check(c) for c in coord]
            else:
                raise ProgrammerError('Ended up with %s.' % type(coord))

        bound_check([point.location for point in self.points])
        bound_check([region.to_polygon() for region in self.regions])
        return bounds

    def _autofit(self, bounds):
        """Automatically fit everything with the maximum zoom possible."""
        if not bounds:
            return

        # TODO(Alvin): uncomment with new Folium release
        # self._folium.fit_bounds(
        #     [bounds['min_long'], bounds['min_lat']],
        #     [bounds['max_long'], bounds['max_lat']]
        # )

        # remove the following with new Folium release
        # rough approximation, assuming max_zoom is 18
        import math
        try:
            factor = 1.2
            width = ((bounds['max_lat'] - bounds['min_lat']) or 2)
            height = ((bounds['max_long'] - bounds['min_long']) or 2)
            area, max_area = width*height, 180*360
            zoom = math.log(area/max_area)/-factor
            self['zoom_start'] = max(1, min(18, round(zoom)))
        except ValueError:
            raise MapError('Uh oh. Is the data formatted as long-lat '
            'pairs? (Check. If the smallest number is less than -90, '
            'those are long-lat pairs) If so, please add geo_formatted=True '
            'to create_map')

    def _autocenter(self, bounds):
        """Find the center."""
        if not bounds:
            return

        midpoint = lambda a, b: (a + b)/2
        self['location'] = (
            midpoint(bounds['min_lat'], bounds['max_lat']),
            midpoint(bounds['min_long'], bounds['max_long'])
        )

    def _dynamic_arg(self, key):
        """Check the first arg in check_map for other options."""
        val = getattr(self, key, None)
        if val:
            setattr(self, key, None)
            del self[key]
            if isinstance(val, MapRegion):
                self.regions.append(val)
            elif isinstance(val, MapPoint):
                self.points.append(val)
            else:
                setattr(self, key, val)
                self[key] = val

    @classmethod
    def reverse(cls, lst_or_coord_or_obj):
        """Reverse all coordinates in a (possibly nested) list.

        Effectively translates all EPSG-compliant coordinates into valid
        geoJSON coordinates.
        """
        lst = coord = obj = lst_or_coord_or_obj
        if not lst:
            return
        if isinstance(coord[0], (int, float)):
            return coord[::-1]
        if isinstance(lst, cls._allowed_collections):
            return [cls.reverse(item) for item in lst]
        return obj

class MapPoint(MapEntity):
    """
    A circle, wrapper for Folium's draw_circle. Draw by passing into draw_map.

    location -- lat-long pair at center of circle
    radius -- radius of circle on map
    popup -- text that pops up when circle is clicked
    line_color -- color of circle border
    fill_color -- color of circle within border
    fill_opacity -- opacity of circle fill
    """

    _mapper = 'circle_marker'

    _allowed_attributes = [
        'location',
        'radius',
        'popup',
        'line_color',
        'fill_color',
        'fill_opacity'
    ]

    _defaults = {
        'location': [],
        'radius': 10,
        'popup': '',
        'line_color': '#3186cc',
        'fill_color': '#3186cc',
        'fill_opacity': 0.6
    }

    def __init__(self, location_or_x, y=None, **kwargs):
        """
        Checks for coordinates passed in as two args, default is to pass as
        one arg
        """
        location = x = location_or_x
        if isinstance(x, (int, float)):
            location = (x, y)
        super().__init__(location=location, **kwargs)

    def __str__(self):
        return str(list(self.location))

    def _after___init__(self, *args, **kwargs):
        """
        Translates all lat-long pairs into long-lat pairs, from EPSG-compliant
        to GeoJSON format
        """
        if 'geo_formatted' not in kwargs \
                or not kwargs['geo_formatted']:
            self.location = Map.reverse(self.location)

class MapRegion(MapEntity):
    """
    A polygon, wrapper for Folium's geo_json method, with a few customizations.
    Draw by passing into draw_map.

    regions -- list of MapRegions (if present, locations and points not used)
    locations -- list of list of lat-long pairs for locations with multiple
                 polygons (if present, points are not used)
    points -- list of lat-long pairs for a normal polygon
    geo_formatted -- boolean, True if the data is formatted as long-lat pairs
    *pass any other kwarg to save it with the object

    Note: GeoJSON requires long-lat-alt triplets, or just long-lat pairs,
    although EPSG:4326 states that the coordinate order should be lat-long. For
    this reason, MapRegion has to flip incoming lat-long pairs to become
    long-lat pairs. By default, MapRegion accepts lat-long pairs, to match
    MapPoint's behavior.
    """

    # composite options

    CHILDREN = 'children'  # invoke map() for children
    GROUP = 'group'  # invoke to_json() for children
    ONE = 'one'  # invoke to_polygon() for children

    _mapper = 'geo_json'

    _allowed_attributes = [
        'geo_path',
        'geo_str',
        'data_out',
        'data',
        'columns',
        'key_on',
        'threshold_scale',
        'fill_color',
        'fill_opacity',
        'line_color',
        'line_weight',
        'line_opacity',
        'legend_name',
        'topojson',
        'reset'
    ]

    _defaults = {
        'composite': GROUP,
        'regions': [],
        'locations': [[]],
        'points': [],
        'fill_color': 'blue',
        'fill_opacity': 0.6,
        'line_color': 'black',
        'line_weight': 1,
        'line_opacity': 1
    }

    CACHE_DIR = 'cache'
    TEMP_FILE = None

    def _after___init__(self, *args, **kwargs):
        """
        Translates all lat-long pairs into long-lat pairs, from EPSG-compliant
        to GeoJSON format
        """
        if 'geo_formatted' not in kwargs or not kwargs['geo_formatted']:
            if 'locations' in kwargs:
                self.locations = Map.reverse(kwargs['locations'])
            if 'points' in kwargs:
                self.points = Map.reverse(kwargs['points'])

    def _before_map(self, *args, **kwargs):
        """Prepares for mapping by passing JSON representation"""
        self._validate()
        json_str = json.dumps(self._to_json()).replace(']]]}}', ']]]}}\n')

        # self.attributes['geo_str'] = json_str

        # TODO(Alvin): find soln to temporary workaround
        if self.locations[0] or self.points or self.regions:
            import os
            os.makedirs(self.CACHE_DIR, exist_ok=True)
            self.TEMP_FILE = os.path.join(
                self.CACHE_DIR, str(round(random.random()*1000))+'temp.json')
            open(self.TEMP_FILE, 'w').write(json_str)
            self._attributes['geo_path'] = self.TEMP_FILE

    def map(self, map=None):
        """Takes action, depending on composite

        CHILDREN -- invoke map() for each child
        GROUP -- invoke parent map() normally
        ONE -- invoke parent map() normally
        """
        composite = self.composite
        if composite in (MapRegion.GROUP, MapRegion.ONE):
            super().map(map)
        elif composite == MapRegion.CHILDREN:
            for region in self.regions:
                region.map(map)

    def _validate(self):
        """Checks for validity of data"""
        locations, points = self.get('locations', 'points')

        if not isinstance(points, self._allowed_collections):
            raise MapError('"Points" must be a %s' %
                ' or '.join(self._allowed_collections))

    @staticmethod
    def to_json(features, **kwargs):
        """Converts any features list to FeatureCollection JSON"""
        return _map_to_defaults({
            'type': 'FeatureCollection',
            'features': features
        }, kwargs)

    def _to_json(self):
        """Converts MapRegion to folium-ready geoJSON, depending on composite.

        CHILDREN -- none
        GROUP -- construct jsons with children
        ONE -- construct polygons with children
        """
        composite = self.composite
        if composite == MapRegion.GROUP:
            return MapRegion.to_json(
                self._to_feature_json())
        elif composite == MapRegion.ONE:
            return MapRegion.to_json(
                MapRegion.to_feature_json(
                    self._to_polygon_json()))
        return None

    @staticmethod
    def to_feature_json(feature, **kwargs):
        """Converts single feature into feature JSON"""
        return _map_to_defaults({
            'type': 'Feature',
            'properties': {'featurecla': 'Map'},
            'geometry': feature
        }, kwargs)

    def _to_feature_json(self):
        """Converts to list of feature JSONs"""
        return [MapRegion.to_feature_json(feat)
            for feat in self.to_feature()]

    def to_feature(self):
        """Converts to list of poylgon JSONs"""
        regions = self.regions

        if regions:
            rval = []
            for region in regions:
                rval += (region.to_feature() or [])
            return rval

        return [self._to_polygon_json()]

    @staticmethod
    def to_polygon_json(polygon, **kwargs):
        """Converts single polygon into polygon JSON"""
        return _map_to_defaults({
            'type': 'Polygon',
            'coordinates': polygon
        }, kwargs)

    def _to_polygon_json(self):
        """Converts to one polygon JSON"""
        return MapRegion.to_polygon_json(self.to_polygon())

    def to_polygon(self):
        """Converts to list of polygons, including that of descendants"""
        regions, locations, points = self.get('regions', 'locations', 'points')

        if regions:
            rval = []
            for region in regions:
                rval += region.to_polygon()
            return rval

        return self._to_locations()

    def _to_locations(self):
        """Converts to list of polygons"""
        locations, points = self.get('locations', 'points')
        if not locations[0]:
            return [points]
        return locations

    def __str__(self):
        return json.dumps(self._to_json())


import socket
import base64
import zipfile
import os

from p2p.connection_manager_4edge import ConnectionManager4Edge
from p2p.my_protocol_message_handler import MyProtocolMessageHandler
from p2p.message_manager import (
    MessageManager,
    RSP_FULL_CHAIN,
    MSG_ENHANCED,
)
from signature.generate_sigunature import DigitalSignature
STATE_INIT = 0
STATE_ACTIVE = 1
STATE_SHUTTING_DOWN = 2

dirname = os.path.dirname(__file__).replace("core", "")

class ClientCore:

    def __init__(self, my_port=50082, core_host=None, core_port=None):
        self.client_state = STATE_INIT
        print('Initializing ClientCore...')
        self.my_ip = self.__get_myip()
        print('Server IP address is set to ... ', self.my_ip)
        self.my_port = my_port
        self.my_core_host = core_host
        self.my_core_port = core_port
        self.cm = ConnectionManager4Edge(self.my_ip, self.my_port, core_host, core_port, self.__handle_message)
        self.mpm = MyProtocolMessageHandler()
        self.gs = DigitalSignature()
        self.my_protocol_message_store = []

    def start(self):
        self.client_state = STATE_ACTIVE
        self.cm.start()
        self.cm.connect_to_core_node()

    def shutdown(self):
        self.client_state = STATE_SHUTTING_DOWN
        print('Shutdown edge node ...')
        self.cm.connection_close()

    def get_my_current_state(self):
        return self.client_state

    def send_message_to_my_core_node(self, msg_type, msg):
        print("dirname:", dirname)
        msg_pub = self.gs.add_public_key(msg)
        print("msg_pub:", msg_pub)

        #txt化
        file_name = "msg_pub"
        f = open(dirname+"ZIP_busdata/send_busdata/" + str(file_name) + ".txt", 'w')
        f.write(msg_pub)
        f.close()
        print("txt")
        #zipを作成
        with zipfile.ZipFile(dirname+"ZIP_busdata/send_busdata/"+str(file_name) + ".zip" , "w", zipfile.ZIP_DEFLATED) as zf:
            zf.write(dirname+"ZIP_busdata/send_busdata/" + str(file_name) + ".txt", str(file_name) + ".txt")
        print("zip")
        #file_open

        f = open(dirname+"ZIP_busdata/send_busdata/"+str(file_name) + ".zip","rb")
        readdata = f.read()

        d = {}
        d["bin"] = base64.b64encode(readdata).decode("utf-8") #バイナリ化
        f.close
        new_msg = d

        msg_txt = self.cm.get_message_text(msg_type, new_msg)

        self.cm.send_msg((self.my_core_host, self.my_core_port), msg_txt)
        print("============ send_msg =============================")

    def __client_api(self, request, message):

        if request == 'pass_message_to_client_application':
            self.my_protocol_message_store.append(message)
        elif request == 'api_type':
            return 'client_core_api'
        else:
            print('not implemented api was used')

    def get_my_protocol_messages(self):

        if self.my_protocol_message_store != []:
            return self.my_protocol_message_store
        else:
            return None

    def __handle_message(self, msg):
        if msg[2] == RSP_FULL_CHAIN:
            # TODO: ブロックチェーン送信要求に応じて返却されたブロックチェーンを検証する処理を呼び出す
            pass
        elif msg[2] == MSG_ENHANCED:
            # P2P Network を単なるトランスポートして使っているアプリケーションが独自拡張したメッセージはここで処理する。SimpleBitcoin としてはこの種別は使わない
            # self.mpm.handle_message(msg[4], self.__client_api, True)
            print("MSE_ENHANCED")

    def __get_myip(self):
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        s.connect(('8.8.8.8', 80))
        return s.getsockname()[0]


from django.forms import ModelForm
from .models import Question, Choice
from django.db import models

class QuestionForm(ModelForm):
    #question_text = forms.CharField(label='Название', max_length=200)

    class Meta:
        model = Question
        fields = ('question_text', 'pub_date')

class ChoiceForm(ModelForm):
    class Meta:
        model = Choice
        fields = ('question', 'choice_text')

form_question = QuestionForm()

form_choice = ChoiceForm()


import re

text = []

while True:
    try:
        inp = input()
        text.append(inp)
    except:
        break

text = " ".join(text)

text = text.strip()
text = re.sub(r"\s+", " ", text)

def replace_function(tag: re.Match):
    value = tag.group(0)

    if value == "\n
 " or value == "\n
 " or value == "\n
":
        return "\n" + ("-"*80) + "\n"
    if value == "
 " or value == "
 " or value == "
":
        return ("-"*80) + "\n"
    if value == "
 " or value == "
 " or value == "
":
        return "\n"

def fix_rule(match: re.Match):
    value = match.group(0)
    parts = re.split(r"\b", value)

    if len(parts) == 1:
        parts[0] = parts[0][:-1] + "\n" + parts[0][-1]
        return "".join(parts)

    last_part = parts[-1]

    if last_part == "":
        parts[-3] = "\n"
    else:
        parts[-1] = "\n"+parts[-1]

    return "".join(parts)

text = re.sub(r"(\n\s?
\s?)|(\s?
\s?)|(\s?
\s?)", replace_function, text)
text = re.sub(r"[^\n]{81}", fix_rule, text)
text = re.sub(r"\s+$", "", text)
text = re.sub(r"\n ", "\n", text)

print(text)


#!/usr/bin/env python
# -*- coding:utf-8 _*-
# Author: Wengs
# Time  : 2/24/2019 10:30 AM
# File  : sideshot_bullet.py
# IDE   : PyCharm

# 12-5

import pygame
from pygame.sprite import Sprite

class Bullet(Sprite):
    """一个对飞船发射的子弹进行管理的类"""

    def __init__(self, ss_settings, screen, ship):
        """在飞船所处的位置创建一个子弹对象"""
        super().__init__()
        self.screen = screen

        # 在(0, 0)处创建一个表示子弹的矩阵，再设置正确的位置
        self.rect = pygame.Rect(0, 0, ss_settings.bullet_width,
                                ss_settings.bullet_height)
        self.rect.centery = ship.rect.centery
        self.rect.right = ship.rect.right

        # 存储用小数表示的子弹设置
        self.x = float(self.rect.x)

        self.color = ss_settings.bullet_color
        self.speed_factor = ss_settings.bullet_speed_factor

    def update(self):
        """向上移动子弹"""
        # 更新表示子弹位置的小数值
        self.x += self.speed_factor
        # 更新表示子弹的rect的位置
        self.rect.x = self.x

    def draw_bullet(self):
        """在屏幕上绘制子弹"""
        pygame.draw.rect(self.screen, self.color, self.rect)


import requests
import json

key = 'apikey=plzBanMe' # deve ir ao site http://www.omdbapi.com/ e pegar uma nova key
r = '\033[31m'  # fonte vermelha (red)
n = '\033[m'  # Cor 'normal'

def existefilmes(titulo):
    quant = 0

    try:
        req = requests.get(f"http://www.omdbapi.com/?s={titulo}&type=movie&{key}")
        resposta = json.loads(req.text)
        if resposta['Response'] == 'True':
            quant = resposta['totalResults']
    except:
        print('Conexao falhou')
    return quant

def lista_filmes(titulo):
    lista = []
    for i in range(1, 101):
        print('Pesquisando na pagina', i)
        try:
            req = requests.get(f"http://www.omdbapi.com/?s={titulo}&type=movie&page={str(i)}&{key}")
            resposta = json.loads(req.text)

            if resposta['Response'] == 'False':
                break
            else:
                for filme in resposta['Search']:
                    title = filme['Title']
                    ano = filme['Year']
                    string = f"{title} ({ano})"
                    lista.append(string)
        except:
            print('Conexao falhou')
    return lista

sair = False
while not sair:
    op = input('Pesquise por um filme ou digite SAIR: ')
    if op.lower() == 'sair':
        sair = True
    else:
        lista_de_filmes = lista_filmes(op)
        print(f'Filmes encontrados: {len(lista_de_filmes)}')
        for filme in lista_de_filmes:
            print(filme)

Curso Python Básico - SOLYD/Aula 13 - API, JSON e consultando listas de filmes/exercicio13.py


# 09_math_demo.py
# 张量数学计算示例
import tensorflow as tf

x = tf.constant([[1, 2],
                 [3, 4]], dtype=tf.float32)
y = tf.constant([[4, 3],
                 [3, 2]], dtype=tf.float32)
x_add_y = tf.add(x, y) # 张量相加, 对应元素相加
x_mul_y = tf.matmul(x, y) # 按照矩阵规则相乘
log_x = tf.log(x)
x_sum = tf.reduce_sum(x, axis=1) # 0-列方向 1-行方向

data = tf.constant([1,2,3,4,5,6,7,8,9,10], dtype=tf.float32)
segment_idx = tf.constant([0,0,0,1,1,2,2,2,2,2], dtype=tf.int32)
x_seg_sum = tf.segment_sum(data, segment_idx)#分段求和

with tf.Session() as sess:
    print(x_add_y.eval())
    print(x_mul_y.eval())
    print(log_x.eval())
    print(x_sum.eval())
    print(x_seg_sum.eval())

5. Python 数据分析+人工智能/3. 深度学习/2. Deep Learning/day04/day04/09_math_demo.py


from Util.Config import Config
from Util.Log import Log
from Util.Data import Data

class Module:

    def start(self, gconfig, file):
        # Global config
        self.gconfig = gconfig

        # Load config
        self.config = Config(gconfig.getValue("srcPath"), file)

        # Create log file
        self.log = Log(self.config)
        self.log.open()
        self.log.printStart()

        # Create data folder
        self.data = Data(self.config)

    def stop(self):
        self.log.printStop()
        self.log.close()


# 경우의수 반복되는 문제이므로 DP
# 해당 수를 1,2,3 빼서 가각ㄱ의 값으로 더하면 됨
# 그럼 중복되는 경우는 없을까? >> 없어

bd = []
for i in range(int(input())) :
    bd.append(int(input()))

m = [0,1,2,4]
if (max(bd) >3) :
    for i in range(4,max(bd)+1) :
        mv = m[i-1] + m[i-2] + m[i-3]
        m.append(mv)

for i in bd :
    print(m[i])

# 마지막에 적은수도 생각해주기


#!/home/xuser/anaconda3/bin/python3

import xmlrpc.client

def main():
    print("This is a client!")

    client = xmlrpc.client.ServerProxy("http://localhost:8080")
    print(client.add(10, 20))

    client.get_image()

if __name__ == "__main__":
    main()


#!/usr/bin/env python
################################################################################
# MIT License
#
# Copyright (c) 2021 Yoshifumi Asakura
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
################################################################################

################
### packages
###############
'''
write basic tools for calculation, and for directory or run here.
'''

################
### packages
################
import numpy as np
#from scipy.optimize import minimize
#from scipy import stats
import time
import sys
import re
import pandas as pd

import os
import shutil

import pyper

#import math
#import matplotlib.pyplot as plt
#import seaborn as sns

import platform   as plf
import subprocess as sbp

import datetime as dtt

import traceback as tb

### handwrite

################
### field diff
################
def field_d1(field, xy, dx = 1, perio = True):
    ### field is an array
    ### xy is a directio x or y to diff
    ### dx should be 1
    ### periodic or not
    if xy == "x":
        pass
    elif xy == "y":
        field = field.T
        #
    else:
        print("error, not defined direction.")
        sys.exit()
    #
    if field.shape[0] == 1:
        return(np.zeros((1, 1)))
    #
    if perio:
        K = np.zeros((field.shape[0], field.shape[0]))\
            + np.diag(np.ones((field.shape[0] - 1)), k = 1)\
            - np.diag(np.ones((field.shape[0] - 1)), k = -1)\
            + np.diag((1.0,), k = (1 - field.shape[0]))\
            - np.diag((1.0,), k = (field.shape[0] - 1))
        #
        #print(K)
        out = K.dot(field)
        #
    else:
        K = np.zeros((field.shape[0], field.shape[0]))\
            + np.diag(np.ones((field.shape[0] - 1)), k = 1)\
            - np.diag(np.ones((field.shape[0] - 1)), k = -1)\
            + np.diag((1.0,), k = (1 - field.shape[0]))\
            - np.diag((1.0,), k = (field.shape[0] - 1))
        K[0, 1] = 0.0
        K[field.shape[0] - 1, field.shape[0] - 2] = 0.0
        out = K.dot(field)
        #
    #
    if xy == "y":
        out = out.T
        #
    #
    return(out)
    #
    #

def field_d2(field, xy, dx = 1, perio = True):
    ### field is an array
    ### xy is a directio x or y to diff
    ### dx should be 1
    ### periodic or not
    if xy == "x":
        pass
    elif xy == "y":
        field = field.T
        #
    else:
        print("error, not defined direction.")
        sys.exit()
    #
    if field.shape[0] == 1:
        return(np.zeros((1, 1)))
    #
    if perio:
        K = np.zeros((field.shape[0], field.shape[0]))\
            - np.diag(np.ones((field.shape[0])), k = 0) * 2.0\
            + np.diag(np.ones((field.shape[0] - 1)), k = 1)\
            + np.diag(np.ones((field.shape[0] - 1)), k = -1)\
            + np.diag((1.0,), k = (1 - field.shape[0]))\
            + np.diag((1.0,), k = (field.shape[0] - 1))
        #
        #print(K)
        out = K.dot(field)
        #
    else:
        K = np.zeros((field.shape[0], field.shape[0]))\
            - np.diag(np.ones((field.shape[0])), k = 0) * 2.0\
            + np.diag(np.ones((field.shape[0] - 1)), k = 1)\
            + np.diag(np.ones((field.shape[0] - 1)), k = -1)\
            + np.diag((1.0,), k = (1 - field.shape[0]))\
            + np.diag((1.0,), k = (field.shape[0] - 1))
        K[0, 1] = 2.0
        K[field.shape[0] - 1, field.shape[0] - 2] = 2.0
        out = K.dot(field)
    #
    if xy == "y":
        out = out.T
        #
    #
    return(out)
    #
    #

def di1(field, xy, dx = 1, perio = True):
    ### this calculates differential of the field, instead of just diff
    return(0.5 * field_d1(field, xy, dx, perio) / dx)
    #

def di2(field, xy, dx = 1, perio = True):
    return(1.0 * field_d2(field, xy, dx, perio) / (dx**2.0))
    #

################
###
################

################
###
################

################
###
################

################
### directory setting
################
def dir_reset(dirname, options = True):
    if os.path.exists(dirname) and options:
        shutil.rmtree(dirname)
        os.mkdir(dirname)
        #
    elif not os.path.exists(dirname):
        os.mkdir(dirname)
        #
    #

################
###
################
class Name_Maker:
    def __init__(self, outdir, outhead, ext, reg_com = None, sim_com = None):
        self.outdir  = outdir
        self.outhead = outhead
        self.ext     = ext
        self.reg_com = reg_com
        self.sim_com = sim_com
        #
        ### versions information
        filename = "%s/versions_info.txt" % self.outdir
        dir_reset(self.outdir, False)
        with open(filename, mode = "w") as f:
            tmp = sbp.run("pip list --format columns".split(" "), stdout = sbp.PIPE)
            mes = [
                "conducted in",
                os.getcwd(),
                "\nenvironments",
                sys.version,
                "\n",
                plf.platform(),
                "\n",
                tmp.stdout.decode()
            ]
            f.write("\n".join(mes))
        #
        reg = "%s/regression" % self.outdir
        sim = "%s/simulation" % self.outdir
        dir_reset(reg, False)
        dir_reset(sim, False)
        self.head_d_f_reg = "%s/%s" % (reg, self.outhead)
        self.head_d_f_sim = "%s/%s" % (sim, self.outhead)
        #
        ### markdown using list
        self.mdlist = [
            filename, ### txt file
            "%s/summary_reg.csv" % self.outdir,
            "%s/summary_sim.csv" % self.outdir
        ]
        self.mdlist2 = [
            filename, ### txt file
            "%s/summary_reg.csv" % self.outdir,
            "%s/summary_sim.csv" % self.outdir
        ]
        self.comp_only_head = [
            filename, ### txt file
            "%s/summary_reg.csv" % self.outdir,
            "%s/summary_sim.csv" % self.outdir
        ]
        self.comp_label = []
        self.comp_only  = []
        #
        self.prev = "yet"
        #
        #
    def get_passed_paras(self):
        out = "%s/passed_parameters.csv" % self.outdir
        return(out)
        #
    def get_sim_sum(self):
        return(self.mdlist[2])
        #
        #
    def set_sim(self, paras, mod_choice, ind_level):
        self.paras  = paras
        self.mod_choice = mod_choice
        self.i_paras, self.i_mod_choice = ind_level
        #
        ### prepare
        self.param_num_str = "m%02d_p%02d" %(self.i_mod_choice, self.i_paras)
        #
        #
    def get_in_reg(self):
        return(self.tablepaths, self.methods_df, self.i_meth, self.i_input)
        #
    def get_in_sim(self):
        return(self.paras, self.mod_choice, self.i_paras, self.i_mod_choice)
        #
        #
    def names_sim(self, another_color = []):
        out = {}
        out["heat_name"]     = "%s_%s_field_Euler.png"          %(self.head_d_f_sim, self.param_num_str)
        out["Vxname"]        = "%s_%s_field_Vx.png"             %(self.head_d_f_sim, self.param_num_str)
        out["Vyname"]        = "%s_%s_field_Vy.png"             %(self.head_d_f_sim, self.param_num_str)
        out["Rhoname"]       = "%s_%s_field_Rho.png"            %(self.head_d_f_sim, self.param_num_str)
        out["ERKname"]       = "%s_%s_field_ERK.png"            %(self.head_d_f_sim, self.param_num_str)
        out["part_name"]     = "%s_%s_particles_track.png"      %(self.head_d_f_sim, self.param_num_str)
        out["part_table"]    = "%s_%s_particles_track.csv"      %(self.head_d_f_sim, self.param_num_str)
        out["Lag_Rdata"]     = "%s_%s_sim/Lagrange.Rdata"       %(self.head_d_f_sim, self.param_num_str)
        out["integ_place"]   = "%s_%s_sim"                      %(self.head_d_f_sim, self.param_num_str)
        out["integ_value"]   = "%s_%s_sim/integrated.npy"       %(self.head_d_f_sim, self.param_num_str)
        out["integ_time"]    = "%s_%s_sim/timerange.npy"        %(self.head_d_f_sim, self.param_num_str)
        out["integ_coord"]   = "%s_%s_sim/coord.npy"            %(self.head_d_f_sim, self.param_num_str)
        out["animation"]     = "%s_%s_animation"                %(self.head_d_f_sim, self.param_num_str)
        out["ani_capture"]   = "%s_%s_ani_cap_last.png"         %(self.head_d_f_sim, self.param_num_str)
        out["source"]        = "%s_%s_sim/source.npy"           %(self.head_d_f_sim, self.param_num_str)
        out["erks_name"]     = "%s_%s_ERK_last.png"             %(self.head_d_f_sim, self.param_num_str)
        out["Rdata"]         = "%s_%s_sim/fields.Rdata"         %(self.head_d_f_sim, self.param_num_str)
        out["another_color"] = {
            "all":             "%s_%s_field_color2_Euler.png"   %(self.head_d_f_sim, self.param_num_str),
            "Vx":              "%s_%s_field_color2_Vx.png"      %(self.head_d_f_sim, self.param_num_str),
            "Vy":              "%s_%s_field_color2_Vy.png"      %(self.head_d_f_sim, self.param_num_str),
            "Rho":             "%s_%s_field_color2_Rho.png"     %(self.head_d_f_sim, self.param_num_str),
            "ERK":             "%s_%s_field_color2_ERK.png"     %(self.head_d_f_sim, self.param_num_str),
            "part":            "%s_%s_particles_color2.png"     %(self.head_d_f_sim, self.param_num_str)
        }
        out["pmodel_track"]  = "%s_%s_Pmodel_track.png"         %(self.head_d_f_sim, self.param_num_str)
        out["pmodel_color"]  = "%s_%s_Pmodel_track_color2.png"  %(self.head_d_f_sim, self.param_num_str)
        out["pmodel_Rdata"]  = "%s_%s_sim/Pmodel.Rdata"         %(self.head_d_f_sim, self.param_num_str)
        #
        out["comp_R"]        = "%s_%s_sim/compare.Rdata"        %(self.head_d_f_sim, self.param_num_str)
        out["comp_gif"]      = "%s_%s_particles_compare.gif"    %(self.head_d_f_sim, self.param_num_str)
        out["comp_all"]      = "%s_%s_p_track_compare.png"      %(self.head_d_f_sim, self.param_num_str)
        #
        out["param_num"]     = self.param_num_str
        #
        self.mdlist.append(out["heat_name"])
        self.mdlist.append(out["part_name"])
        #
        line  = os.getcwd()
        ### select save fig
        tmp_h = "%s/%s" % (line, out["heat_name"])
        if "heat" in another_color:
            tmp_h = "%s/%s" % (line, out["another_color"].get("all"))
        tmp_p = "%s/%s" % (line, out["part_name"])
        if "part" in another_color:
            tmp_p = "%s/%s" % (line, out["another_color"].get("part"))
        #
        table_md = [
            "|%s_field_Euler|%s_particles_track|" % (self.param_num_str, self.param_num_str),
            "|---|---|",
            "|![](%s)|![](%s)|\n"                   % (tmp_h, tmp_p)
        ]; #print("\n".join(table_md))
        if not self.sim_com is None:
            if self.i_mod_choice < len(self.sim_com):
                table_md = [self.sim_com[self.i_mod_choice]] + table_md
                #self.mdlist2.append("\n".join([self.sim_com[self.i_mod_choice] + "\n"]))
        #print("\n".join(table_md))
        ### select save fig 2nd row
        tmp_v = "%s/%s" % (line, out["Vxname"])
        if "Vx" in another_color:
            tmp_v = "%s/%s" % (line, out["another_color"].get("Vx"))
        tmp_a = "%s/%s" % (line, out["ani_capture"])
        #
        table_md2 = [
            "|%s_field_Vx|%s_ani_capture|" % (self.param_num_str, self.param_num_str),
            "|---|---|",
            "|![](%s)|![](%s)|\n"          % (tmp_v, tmp_a)
        ]
        #
        ### select save fig 3rd row
        #tmp_r = "%s/%s" % (line, out["Rhoname"]) ### later, overwrite instead
        tmp_r = "%s/%s" % (line, out["comp_all"])
        tmp_p = "%s/%s" % (line, out["pmodel_track"])
        #
        table_md3 = [
            "|%s_compare|%s_Pmodel_track|" % (self.param_num_str, self.param_num_str),
            "|---|---|",
            "|![](%s)|![](%s)|\n"          % (tmp_r, tmp_p)
        ]
        #
        if (not self.prev == "sim") and (not self.prev == "yet"):
            #table_md = ["

\n"] + table_md
            self.mdlist2.append("\n

\n")
        self.mdlist2.append("\n".join(table_md))
        self.mdlist2.append("\n".join(table_md2))
        self.mdlist2.append("\n".join(table_md3))
        self.mdlist2.append("\n

\n")
        self.prev = "sim"
        #
        self.comp_label.append("%s_compare" % self.param_num_str)
        self.comp_only.append(tmp_r)
        #
        return(out)
        #
    def get_mdlist(self):
        '''
        returns a list to include in markdown summary
        '''
        time_stmp = dtt.datetime.today().strftime("%Y%m%d_%H%M")[2:]
        out = "%s/%s_summary.md" % (self.outdir, time_stmp)
        return([out, self.mdlist])
        #
    def get_mdlist2(self, insertion = []):
        '''
        returns a list to include in markdown summary with figures in tables
        '''
        time_stmp = dtt.datetime.today().strftime("%Y%m%d_%H%M")[2:]
        out = "%s/%s_summary.md" % (self.outdir, time_stmp)
        if len(insertion) > 0:
            for j in range(0, len(insertion)):
                self.mdlist2.insert((3 + j), insertion[j])
        return([out, self.mdlist2])
        #
    def get_comp_only(self, insertion = []):
        #comp_only = self.comp_only
        ### put all figures into tables
        #
        if len(self.comp_label) <= 1:
            out = []
            if len(insertion) > 0:
                out = self.comp_only_head
                for j in range(0, len(insertion)):
                    out.insert((3 + j), insertion[j])
            return(out)
        #
        comp_only = []
        #
        pages = divmod(len(self.comp_only), 6)
        if pages[1] == 1:
            pages = [pages[0] - 1, 7]
            listk = [4, 3]
        else:
            listk = [pages[1]]
        #
        def table_command(j):
            indice = [6 * j + k for k in range(0, 6)]
            for k, ind in enumerate(indice):
                if ind >= len(self.comp_only):
                    indice[k] = 0
            return([
                "\n

\n",                       #1
                "|%s|%s|"           % (self.comp_label[indice[0]], self.comp_label[indice[1]]),     #2
                "|---|---|",                                                              #3
                "|![](%s)|![](%s)|" % (self.comp_only[ indice[0]], self.comp_only[ indice[1]]),     #4
                "|%s|%s|"           % (self.comp_label[indice[2]], self.comp_label[indice[3]]),     #5
                "|![](%s)|![](%s)|" % (self.comp_only[ indice[2]], self.comp_only[ indice[3]]),     #6
                "|%s|%s|"           % (self.comp_label[indice[4]], self.comp_label[indice[5]]),     #7
                "|![](%s)|![](%s)|" % (self.comp_only[ indice[4]], self.comp_only[ indice[5]]),     #8
                "\n"
            ])
        def odd_table(j, remainder):
            if remainder % 2 == 1:
                return([
                    "|%s|end|" % self.comp_label[(6 * j + remainder - 1)],
                    "|![](%s)|end|" % self.comp_only[ (6 * j + remainder - 1)],
                    "\n"
                ])
            else:
                return(["\n"])
                #
            #
        #
        for j in range(0, pages[0]):
            comp_only.append("\n".join(table_command(j)))
        try:
            j += 1; #print(j)
        except:
            j = 0
        try:
            for k in range(0, len(listk)):
                if listk[k] == 5:
                    comp_only.append("\n".join(table_command(j)[:6] + odd_table(j, listk[k])))
                elif listk[k] == 4:
                    comp_only.append("\n".join(table_command(j)[:6] + odd_table(j, listk[k])))
                elif listk[k] == 3:
                    comp_only.append("\n".join(table_command(j)[:4] + odd_table(j, listk[k])))
                elif listk[k] == 2:
                    comp_only.append("\n".join(table_command(j)[:4] + odd_table(j, listk[k])))
                    #
                #
            #
        except:
            print("ERROR occured in fun_dir.Name_Maker.get_comp_only")
            print(pages)
            print(listk)
            print(len(self.comp_label))
            print(len(self.comp_only))
            tb.print_exc()
        #
        #
        if len(insertion) > 0:
            out = self.comp_only_head + comp_only
            for j in range(0, len(insertion)):
                out.insert((3 + j), insertion[j])
            #
        else:
            out = comp_only
            #
        return(out)
        #
    def save_fields(self, fields):
        dir_f = "%s_%s_fields" %(self.head_d_f_sim, self.param_num_str)
        files = ["%s/array%02d.npy" % (dir_f, j) for j in range(0, len(fields))]
        dir_reset(dir_f, False)
        #
        for j, field_j in enumerate(fields):
            np.save(file = files[j], arr = field_j)
            #
        #
    def load_fields(self):
        dir_f  = "%s_%s_fields" %(self.head_d_f_sim, self.param_num_str)
        files  = os.listdir(dir_f)
        #for file in files:
        #    print(file)
        files2 = []
        for file in files:
            if ".npy" in file:
                print("    loading %s/%s" % (dir_f, file))
                files2.append("%s/%s" % (dir_f, file))
        out = [np.load(file) for file in files2]
        return(out)
        #
        #

class Data_Arrange:
    def __init__(self, resultdir):
        self.table = {"compare": [], "euler": [], "lagrange": [], "spring": []}
        self.label = []
        #
        self.dir       = resultdir
        self.tablename = "%s/paths_to_Rdata.csv" % resultdir
        #
    def add_row(self, names_sim):
        self.label.append(names_sim["param_num"])
        self.table["compare" ].append(names_sim["comp_R"])
        self.table["euler"   ].append(names_sim["Rdata"])
        self.table["lagrange"].append(names_sim["Lag_Rdata"])
        self.table["spring"  ].append(names_sim["pmodel_Rdata"])
        #
    def get_Rdata_paths(self):
        out0 = pd.DataFrame({"label": self.label})
        out1 = pd.DataFrame(self.table)
        out  = pd.concat([out0, out1], axis = 1)
        return(out)
        #
    def save_Rdata_paths(self, file = None, one_Rdata = True):
        if file is None:
            out = self.tablename
        else:
            out = file
        df = self.get_Rdata_paths()
        df.to_csv(out, header = True, index = False)
        #
        ### make all Rdata onto one / setting
        if one_Rdata:
            dir = "%s/graphs_Rdata" % self.dir
            dir_reset(dir)
            for j, label in enumerate(self.label):
                filej = "%s/%s.Rdata" % (dir, label)
                rowj  = df.iloc[j, 1:].tolist()
                #
                r = pyper.R()
                for k, path in enumerate(rowj):
                    r("load('%s')" % path)
                    #
                #
                r("save.image('%s')" % filej)
                if os.path.exists(filej):
                    print("saved %s" % filej)
                else:
                    print("failed %s" % filej)
                    #
                #
            #
        #
    def draw_pub_fig(self):
        path = "%s/sub_pub_graph.R" % os.path.dirname(__file__)
        with open(path, mode = "r") as f:
            cmd = f.read()
        cmd = re.sub("__result__", self.dir, cmd)
        r   = pyper.R()
        r(cmd)
        #

################
### path
################
def find_up(path):
    if "/" in path:
        path0 = path.split("/")[0]
    else:
        path0 = path
    #print(path0)
    path1 = "../"
    while len(path1) < 30:
        if any([path0 in j for j in os.listdir(path1)]):
            pathout = path1 + path
            break
        else:
            path1 = "../" + path1
        #
    #
    return(pathout)
    #

################
### time counter
################
class Time_keeper:
    def __init__(self):
        self.start_t = []
        self.start_t.append(time.time())
        #
    def start_count(self, print_i = True):
        self.start_t.append(time.time())
        if print_i:
            print(len(self.start_t))
        #
    def get_indice(self):
        return(len(self.start_t))
        #
    def get_elapsed(self, index = 0, seconds = False):
        took_time = int(time.time() - self.start_t[index])
        if seconds:
            out = took_time
            #
        else:
            el_hr = divmod(took_time, 3600)
            el_mi = divmod(el_hr[1],  60)
            out   = "%d hr %02d min %02d sec" % (el_hr[0], el_mi[0], el_mi[1])
            #
            #
        #
        return(out)
        #
        #

################
###
################
def main():
    pass

################
###
################
if __name__ == '__main__':
    main()

###


import numpy as np
import scipy as sp
import matplotlib.pyplot as plt
import time
from .utility import comp_data_covariances, progprint_xrange
from .ssid_loss import f_l2_Hankel_nl
from .ssid_gradients import g_l2_Hankel_sgd_nl_rnd, g_l2_Hankel_sgd_ln_rnd
from .ssid_gradients import g_l2_Hankel_sgd_nl_sso, g_l2_Hankel_sgd_ln_sso
from .ssid_gradients import g_l2_Hankel_sgd_nl, g_l2_Hankel_sgd_ln

###########################################################################
# stochastic gradient descent: gradients w.r.t. C,R,A,B,X=cov(x_{t+m},x_t)
###########################################################################

def main(lag_range,n,y,obs_scheme,
         Qs=None, Om=None,
         parametrization='nl',
         sso=False, W=None,
         idx_a=None,idx_b=None,
         pars_init='default', aux_init=None,
         alpha=0.001, b1=0.9, b2=0.99, e=1e-8, a_decay = 1.,
         max_iter=100, max_epoch_size=np.inf, eps_conv=0.99999,
         batch_size=1,save_every=np.inf,data_path=None,mmap=False,
         return_aux=False, verbose=False, pars_track=None,
         dtype=np.float):

    """ Fits LDS in multiple stages, commonly seperated by different batch sizes.

    Parameters
    ----------
    lag_range : array
        array of increasing time-lags to consider
    n : int
        latent dimensionality
    y : array
        T-by-p array of observed data (unobserved entries will not be queried)
    obs_scheme : ObservationScheme object
        block-wise partial observation scheme
    Qs : list of arrays
        (masked) pairwise time-lagged covariances. Will be computed if None
    Om : list of arrays
        masks for pairwise time-lagged covariances. Will be computed if None
    parametrization : str
        str giving parametrization of latent dynamics model .
        supported parametrizations: 'ln' linear, 'nl' agnostic
    sso : boolean
        if True, assumes block-wise observation scheme (faster computations)
    W :  list or arrays
        list of (inverse) co-observation counts. Will be computed if None
    idx_a, idx_b : arrays
        index arrays for which (variable pairs) to track pairwise covariances
    pars_init: str or dict
        initial parameters. 'default' for default initialization scheme
    aux_init: list of arrays or None
        optional ADAM auxiliary variables (m,v) for initialization of SGD
    alpha, b1, b2, e: floats
        ADAM parameters
    a_decay : float
        decay parameter of ADAM learning rate: a = alpha* a_decay**epoch
    max_iter : int
        maximum number of epochs
    max_epoch_size : int
        maxmimum epoch length
    eps_conv: float
        convergence criterion. Currently not used with minibatch-SGD !
    batch_size : int
        batch size for minibatch stochastic gradient descent
    save_every: int
        counter for epochs, how often to save intermediate results to data_path
    mmap: boolean
        flag if y is stored as memory-mapped array (see numpy.memmap)
    return_aux: boolean
        optionally returns ADAM auxiliary variables (m,v) for later reuse
    verbose : boolean
        verbosity flag. Defines whether to print intermediate losses
    pars_track: None or function
        optional function to track quality of emission matrix C across epochs
    dtype : numpy datatype
        datatype for numpy arrays

    Output
    ----------
    pars_est : dict
        parameter estimate
    pars_init : dict
        initial parameter estimate before first SGD step
    traces : list
        list of fit diagnostics (loss & correlation of covariances over time)
    Qs : list of arrays
        (masked) pairwise time-lagged covariances. Same as input Qs if provided
    Om : list of arrays
        mask for pairwise time-lagged covariances. Same as input Om if provided
    W :  list or arrays
        list of (inverse) co-observation counts. Same as input W if provided
    t_descts : float
        total computation time
    """

    T, p = y.shape

    assert parametrization in ['nl', 'ln']
    num_pars = 3 if parametrization=='nl' else 4

    idx_a = np.arange(p) if idx_a is None else idx_a
    idx_b = idx_a if idx_b is None else idx_b
    assert np.all(idx_a == np.sort(idx_a))
    assert np.all(idx_a == np.sort(idx_a))

    if W is None:
        W = obs_scheme.comp_coocurrence_weights(lag_range=lag_range,
                                                sso=sso,
                                                idx_a=idx_a,
                                                idx_b=idx_b)

    if Qs is None or Om is None:
        Qs, Om = comp_data_covariances(n=n,y=y,lag_range=lag_range,sso=sso,
                                mmap=mmap,data_path=data_path,
                                obs_scheme=obs_scheme,
                                idx_a=idx_a,idx_b=idx_b,W=W)

    if not save_every == np.inf:
        assert not data_path is None

    alpha = alpha * np.ones(num_pars) if np.asarray(alpha).size==1 else alpha
    b1    =  b1   * np.ones(num_pars) if np.asarray(  b1 ).size==1 else b1
    b2    =  b2   * np.ones(num_pars) if np.asarray(  b2 ).size==1 else b2
    e     =   e   * np.ones(num_pars) if np.asarray(  e  ).size==1 else e

    T,p = y.shape
    kl = len(lag_range)
    assert np.all(lag_range == np.sort(lag_range))

    if isinstance(pars_init, dict):
        assert 'C' in pars_init.keys()
        pars_init = pars_init.copy()

    if pars_init =='default':
        pars_init = {'A'  : np.diag(np.linspace(0.89, 0.91, n)),
             'Pi' : np.eye(n,dtype=dtype),
             'B'  : np.eye(n,dtype=dtype),
             'C'  : np.random.normal(size=(p,n)).astype(dtype=dtype),
             'R'  : np.zeros(p,dtype=dtype),
             'X'  : np.zeros(((kl)*n, n),dtype=dtype)}

    f,g,batch_draw,track_corrs,converged,save_interm = sis_setup(
                                           lag_range=lag_range,n=n,T=T,
                                           parametrization=parametrization,
                                           sso=sso,
                                           y=y,Qs=Qs,Om=Om,
                                           idx_a=idx_a, idx_b=idx_b,
                                           obs_scheme=obs_scheme, W=W,
                                           batch_size=batch_size,
                                           mmap=mmap, data_path=data_path,
                                           save_every=save_every,
                                           max_iter=max_iter, eps_conv=eps_conv)
    if verbose:
        print('starting descent')

    t_desc = time.time()
    pars_est, traces = adam_main(f=f,g=g,kl=kl,num_pars=num_pars,
                                 pars_init=pars_init,aux_init=aux_init,
                                 alpha=alpha,b1=b1,b2=b2,e=e, a_decay=a_decay,
                                 batch_draw=batch_draw,converged=converged,
                                 track_corrs=track_corrs,save_interm=save_interm,
                                 max_iter=max_iter,max_epoch_size=max_epoch_size,
                                 batch_size=batch_size,return_aux=return_aux,
                                 verbose=verbose,pars_track=pars_track)

    t_desc = time.time() - t_desc

    if parametrization=='nl':

        pars_est = {'C': pars_est[0],
                    'X': pars_est[1],
                    'R': pars_est[2],
                    'A': None,
                    'Pi': pars_est[1][:n,:].copy(),
                    'B': None }

    elif parametrization=='ln':

        X, Pi = np.zeros((len(lag_range)*n, n)), pars_est[2].dot(pars_est[2].T)
        for m in range(len(lag_range)):
            m_ = lag_range[m]
            X[m*n:(m+1)*n,:] = np.linalg.matrix_power(pars_est[1],m_).dot(Pi)

        pars_est = {'C': pars_est[0],
                    'A': pars_est[1],
                    'B': pars_est[2],
                    'R': pars_est[3],
                    'Pi': Pi,
                    'X': X}

    return pars_est, pars_init, traces, Qs, Om, W, t_desc

# decorations

def sis_setup(lag_range,T,n,y,Qs,Om,obs_scheme,
              idx_a=None, idx_b=None, W=None,
              parametrization='nl', sso=False,
              max_iter=np.inf, eps_conv=0.99999, save_every=np.inf,
              batch_size=None, mmap=False, data_path=None):
    "returns error function and gradient for use with gradient descent solvers"

    sub_pops= obs_scheme.sub_pops
    idx_grp = obs_scheme.idx_grp
    obs_idx = obs_scheme.obs_idx
    obs_pops= obs_scheme.obs_pops
    obs_time= obs_scheme.obs_time

    T,p = y.shape
    kl = len(lag_range)
    kl_ = np.max(lag_range)+1

    fn = 'p'+str(p)+'n'+str(n)+'T'+str(T)+'subpops'+str(len(sub_pops))

    if sso and obs_scheme.use_mask and len(idx_grp) == 1:
        #print('using rnd gradients')
        g_l2_nl, g_l2_ln = g_l2_Hankel_sgd_nl_rnd, g_l2_Hankel_sgd_ln_rnd
    elif sso:
        #print('using sso gradients')
        g_l2_nl, g_l2_ln = g_l2_Hankel_sgd_nl_sso, g_l2_Hankel_sgd_ln_sso
    else:
        g_l2_nl, g_l2_ln = g_l2_Hankel_sgd_nl, g_l2_Hankel_sgd_ln

    anb = np.intersect1d(idx_a, idx_b)
    idx_Rb = np.where(np.in1d(idx_b,idx_a))[0]
    idx_Ra = np.where(np.in1d(idx_a,idx_b))[0]

    if parametrization=='nl':

        def g(pars, ts, ms):
            return  g_l2_nl(C=pars[0],X=pars[1],R=pars[2],y=y,
                lag_range=lag_range,ts=ts,ms=ms,obs_scheme=obs_scheme,W=W)

        def f(pars):
            return f_l2_Hankel_nl(C=pars[0],X=pars[1],R=pars[2],Qs=Qs,Om=Om,
                lag_range=lag_range, ms=range(kl),idx_a=idx_a,idx_b=idx_b,
                anb=anb, idx_Ra=idx_Ra, idx_Rb=idx_Rb)

        def track_corrs(pars) :
            return track_correlations(C=pars[0],A=None,B=None,X=pars[1],
                            R=pars[2], Qs=Qs, p=p, n=n, lag_range=lag_range,
                            idx_a=idx_a, idx_b=idx_b, mmap=mmap,
                            data_path=data_path)

        def save_interm(pars, t):

            if np.mod(t, save_every) == 0:

                try:
                    print('saving intermediate results...')
                    save_dict = {'T' : T,
                                 'lag_range' : lag_range,
                                 'C' : pars[0],
                                 'X' : pars[1],
                                 'R' : pars[2],
                                 'mmap' : mmap,
                                 'data_path' : data_path
                                }

                    np.savez(data_path +  fn + '_interm_i'+str(t), save_dict)

                except:
                    print('failed to save interm. results! continuing fit ...')

    elif parametrization=='ln':

        def g(pars, ts, ms):
            return  g_l2_ln(C=pars[0],A=pars[1],B=pars[2],R=pars[3],y=y,
                lag_range=lag_range,ts=ts,ms=ms,obs_scheme=obs_scheme,W=W)

        def f(pars):
            X, Pi = np.zeros((kl_*n,n)), pars[2].dot(pars[2].T)
            for m in lag_range:
                X[m*n:(m+1)*n,:] = np.linalg.matrix_power(pars[1],m).dot(Pi)
            return f_l2_Hankel_nl(C=pars[0],X=X,R=pars[3],Qs=Qs,Om=Om,
                lag_range=lag_range, ms=range(kl),idx_a=idx_a,idx_b=idx_b,
                anb=anb, idx_Ra=idx_Ra, idx_Rb=idx_Rb)

        def track_corrs(pars) :
            return track_correlations(C=pars[0],A=pars[1],B=pars[2],
                            X=None,R=pars[3],
                            Qs=Qs, p=p, n=n, lag_range=lag_range,
                            idx_a=idx_a, idx_b=idx_b, mmap=mmap,
                            data_path=data_path)

        def save_interm(pars, t):

            if np.mod(t, save_every) == 0:

                try:
                    print('saving intermediate results...')
                    save_dict = {'T' : T,
                                 'lag_range' : lag_range,
                                 'C' : pars[0],
                                 'A' : pars[1],
                                 'B' : pars[2],
                                 'R' : pars[3],
                                 'mmap' : mmap,
                                 'data_path' : data_path
                                }

                    np.savez(data_path +  fn + '_interm_i'+str(t), save_dict)

                except:
                    print('failed to save interm. results! continuing fit ...')

    # setting up the stochastic batch selection:
    batch_draw, g_sgd = sgd_draw(p, T, lag_range, batch_size, g)

    # set convergence criterion
    if batch_size is None:
        def converged(t, fun):
            if t>= max_iter:
                return True
            elif t > 100 and fun[t-1] > eps_conv * np.min(fun[t-100:t-1]):
                return True
            else:
                return False

    else:
        def converged(t, fun):
            return True if t >= max_iter else False

    return f, g_sgd, batch_draw, track_corrs, converged, save_interm

def sgd_draw(p, T, lag_range, batch_size, g):
    "returns sequence of indices for sets of neuron pairs for SGD"

    kl = len(lag_range)
    kl_ = np.max(lag_range)+1
    if batch_size is None:

        def batch_draw():
            ts = (np.random.permutation(np.arange(T - kl_)) , )
            ms = (lag_range, )
            return ts, ms
        def g_sgd(pars, ts, ms, i):
            return g(pars, ts[i], ms[i])

    elif batch_size == 1:

        def batch_draw():
            ts = np.random.permutation(np.arange(T - kl_))
            ms = np.random.randint(0, kl, size=(len(ts),))
            return ts, ms
        def g_sgd(pars, ts, ms, i):
            return g(pars, (ts[i],), (ms[i],))

    elif batch_size > 1:

        def batch_draw():
            ts = np.random.permutation(np.arange(T - (kl_) ))
            ms = np.random.randint(0,kl,size=len(ts)//batch_size)
            rnge = range(len(ts)//batch_size)
            return ([ts[i*batch_size:(i+1)*batch_size] for i in rnge],ms)

        def g_sgd(pars, ts, ms, i):
            return g(pars, ts[i], (ms[i],))

    return batch_draw, g_sgd

# main optimiser

def adam_main(f,g,pars_init,aux_init,num_pars,kl,alpha,b1,b2,e,a_decay,
            batch_draw,track_corrs,converged,save_interm,
            return_aux,max_iter,batch_size,max_epoch_size,
            verbose,pars_track,dtype=np.float):
    " main function for ADAM optimization "
    # initialise pars
    p, n = pars_init['C'].shape
    pars = init_adam(pars_init, p, n, kl, num_pars)

    # setting up Adam
    b1,b2,e,vp,mp = pars_adam(batch_size,p,n,kl,num_pars,b1,b2,e,dtype)
    t_iter, t, ct_iter = 0, 0, 0
    corrs  = np.zeros((kl, 12))

    def epoch_range(epoch_size):
        if p > 1e4:
            return progprint_xrange(epoch_size, perline=100)
        else:
            return range(epoch_size)

    # trace function values
    fun = np.empty(max_iter)

    corrs[:,ct_iter] = track_corrs(pars)
    ct_iter += 1

    while not converged(t_iter, fun):

        ts, ms = batch_draw()
        epoch_size = get_epoch_size(batch_size, p, ts, max_epoch_size)

        for idx_epoch in epoch_range(epoch_size):
            t += 1

            grads = g(pars, ts, ms, idx_epoch)
            pars, mp, vp = adam_step(pars,grads,mp,vp,alpha,b1,b2,e,t_iter+1)

        if t_iter < max_iter:          # really expensive!
            fun[t_iter] = f(pars)
            if not pars_track is None:
                pars_track(pars,t_iter)
        if verbose and p > 1e4:
            print('f = ', fun[t_iter])
        if verbose and np.mod(t_iter,max_iter//10) == 0:
            print('finished %', 100*t_iter/max_iter+10)
            print('f = ', fun[t_iter])
            corrs[:,ct_iter] = track_corrs(pars)
            ct_iter += 1

        t_iter += 1
        alpha *= a_decay

        save_interm(pars, t_iter)

    corrs[:,ct_iter] = track_corrs(pars)
    fun = fun[:t_iter]

    if verbose:
        print('total iterations: ', t)

    traces = (fun,corrs,(mp,vp)) if return_aux else (fun,corrs)
    return pars, traces

def pars_adam(batch_size,p,n,kl,num_pars,b1,b2,e,dtype=np.float):
    " returns initial ADAM parameters "

    if batch_size is None:
        print('using batch gradients - switching to plain gradient descent')
        b1 = np.zeros(num_pars,dtype=dtype)
        b2 = np.ones(num_pars, dtype=dtype)
        e  = np.zeros(num_pars,dtype=dtype)
    elif not batch_size >= 1:
        raise Exception('cannot handle selected batch size')

    if num_pars == 3:     # (C, X, R)
        vp_0 = [np.ones((p,n),dtype=dtype),
                np.ones((kl*n,n),dtype=dtype),
                np.ones(p,dtype=dtype)]
        mp_0 = [np.zeros((p,n),dtype=dtype),
                np.zeros((kl*n,n),dtype=dtype),
                np.zeros(p,dtype=dtype)]
    elif num_pars == 4:   # (C, A, B, R)
        vp_0 = [np.ones((p,n),dtype=dtype),
                np.ones((n,n),dtype=dtype),
                np.ones((n,n),dtype=dtype),
                np.ones(p,dtype=dtype)]
        mp_0 = [np.zeros((p,n),dtype=dtype),
                np.zeros((n,n),dtype=dtype),
                np.zeros((n,n),dtype=dtype),
                np.zeros(p,dtype=dtype)]

    return b1,b2,e,vp_0,mp_0

def adam_step(pars,g,m,v,a,b1,b2,e,t):

    for i in range(len(m)):

        m[i] = (b1[i] * m[i] + (1-b1[i]) * g[i])
        v[i] = (b2[i] * v[i] + (1-b2[i]) * g[i]**2)

        mh,vh = adam_norm(m[i],v[i],b1[i],b2[i],t)

        pars[i] -= a[i] * mh / (np.sqrt(vh) + e[i])

    return pars, m, v

def adam_norm(m, v, b1, b2,t):

    m = m / (1-b1**t) if b1 != 1 else m.copy()
    v = v / (1-b2**t) if b2 != 1 else v.copy()

    return m,v

def init_adam(pars_0, p, n, kl, num_pars):
    " returns initial LDS parameters where not provided "

    dtype = pars_0['C'].dtype

    if num_pars == 3:
        C = pars_0['C'].copy()
        if 'R' in pars_0.keys():
            R = pars_0['R'].copy()
        else:
            R = np.zeros(p,dtype=dtype)
        if 'X' in pars_0.keys():
            X = pars_0['X'].copy()
        else:
            X = np.zeros((n*kl, n),dtype=dtype)
        pars = [C,X,R]

    elif num_pars == 4:
        C = pars_0['C'].copy()
        if 'A' in pars_0.keys():
            A = pars_0['A'].copy()
        else:
            A = np.zeros((n, n),dtype=dtype)
        if 'B' in pars_0.keys():
            B = pars_0['B'].copy()
        else:
            B = np.zeros((n, n),dtype=dtype)
        if 'R' in pars_0.keys():
            R = pars_0['R'].copy()
        else:
            R = np.zeros(p,dtype=dtype)
        pars = [C,A,B,R]

    return pars

def get_epoch_size(batch_size, p=None, a=None, max_epoch_size=np.inf):

    if batch_size is None:
        epoch_size = len(a)
    elif batch_size >= 1:
        epoch_size = len(a)

    return int(np.min((epoch_size, max_epoch_size)))

def track_correlations(C, A, B, X, R, Qs, p, n, lag_range,
    idx_a=None, idx_b=None, mmap = False, data_path=None):
    """ Computes correlation of covariances.
        Uses all given covariance entries (no masking)!
    """
    Pi = None if B is None else B.dot(B.T)

    if X is None:
        X = np.vstack([np.linalg.matrix_power(A,m) for m in lag_range]).dot(Pi)

    kl = len(lag_range)
    corrs = np.nan * np.ones(kl)
    if not Qs is None:

        idx_a = np.arange(p) if idx_a is None else idx_a
        idx_b = idx_a if idx_b is None else idx_b
        idx_ab = np.intersect1d(idx_a, idx_b)
        idx_a_ab = np.where(np.in1d(idx_a, idx_ab))[0]
        idx_b_ab = np.where(np.in1d(idx_b, idx_ab))[0]

        assert (len(idx_a), len(idx_b)) == Qs[0].shape
        for m in range(kl):
            m_ = lag_range[m]
            Qrec = C[idx_a,:].dot(X[m*n:(m+1)*n, :]).dot(C[idx_b,:].T)
            if m_==0:
                Qrec[np.ix_(idx_a_ab, idx_b_ab)] += np.diag(R[idx_ab])

            if mmap:
                Q = np.load(data_path+'Qs_'+str(m_)+'.npy')
            else:
                Q = Qs[m]

            corrs[m] = np.corrcoef( Qrec.reshape(-1), Q.reshape(-1) )[0,1]

            if mmap:
                del Q

    return corrs


"""Django ZTaskQ based on a fork of Django ZTask."""

VERSION = (0, 3, 0)

__version__ = ".".join([ str(c) for c in VERSION[0:3] ]) + "".join(VERSION[3:])
__author__ = "Drew Bryant, Jason Allum and Dave Martorana"
__contact__ = "drew.h.bryant@gmail.com"
__homepage__ = "https://github.com/awesomo/django-ztaskq"
__docformat__ = "markdown"
__license__ = "BSD (3 clause)"


from django.db import transaction

class TransactionMiddleware(object):
    """
    Transaction middleware. If this is enabled, each view function will be run
    with commit_on_response activated - that way a save() doesn't do a direct
    commit, the commit is done when a successful response is created. If an
    exception happens, the database is rolled back.
    """
    def process_request(self, request):
        """Enters transaction management"""
        transaction.enter_transaction_management()
        transaction.managed(True)

    def process_exception(self, request, exception):
        """Rolls back the database and leaves transaction management"""
        if transaction.is_dirty():
            transaction.rollback()
        transaction.leave_transaction_management()

    def process_response(self, request, response):
        """Commits and leaves transaction management."""
        if transaction.is_managed():
            if transaction.is_dirty():
                transaction.commit()
            transaction.leave_transaction_management()
        return response


from django.template import RequestContext
from django.shortcuts import render_to_response
import os
from root.fEPSPanalyser import fepspAnalyser
#===============================================================================
# import root.filtering_lib2
# import root.dbAccess #good
# import root.externalFunctions #good
# import root.checkResult #good
# import root.clussterization #good
# import root.graph #good
# import root.objects #good
# import root.onClick #good
# import root.rInterface #good
# import root.simple #good
# import root.simple_gui #good
# import root.interpolation #good
# import scipy #good
# from scipy import polyval, polyfit #good
# from scipy import interpolate #good
#===============================================================================
from sheduler.main import shedule

def individual(request,path):
    test_filelist=os.listdir("/%s" % path)
    input=request.POST
    dict={                                              'list': test_filelist,
                                                        'path': path,
                                                        'frequency': '200000',
                                                        'tags': 'not defined2',
                                                        #'debug': '0',
                                                        #'cluster':'0',
                                                        'db':'0',
                                                        'check':'0',
                                                        'dict':input,}
    for i in input:
        try:
            dict[i]=input[i]
        except:
            pass
    if dict['check']=='1':
        analyserObject=fepspAnalyser([0,dict['path'],dict['frequency'],"data","1",dict['tags'],0,dict['db'],1,0])
    return render_to_response('web_fEPSPA/individual.html', dict, context_instance=RequestContext(request))

def sheduler(request,path):
    test_filelist=os.listdir(path)
    input=request.POST
    dict={                                              'list': test_filelist,
                                                        'path': path,
                                                        'frequency': '200000',
                                                        'tags': 'not defined2',
                                                        'check':'0',
                                                        'dict':input,}
    for i in input:
        try:
            dict[i]=input[i]
        except:
            pass
    print(dict['check'])
    if dict['check']=='1':
        shedule(dict['path'],dict['tags'],dict['frequency'])
    return render_to_response('web_fEPSPA/sheduler.html', dict, context_instance=RequestContext(request))