import collections
from functools import lru_cache, reduce
from operator import getitem
import pandas as pd
from jira import JIRA
from jira.client import translate_resource_args, ResultList
from jira.resources import GreenHopperResource, Comment, Issue
import copy
import numpy as np
import json
import re
from box import Box
import logging

class JIRA_EXT(JIRA):
    def __init__(self, *args, logger=logging.getLogger('jira_ext'), **kwargs):
        super(JIRA_EXT, self).__init__(*args, **kwargs)
        self._init_class_properties(logger)

    def _init_class_properties(self, logger=logging.getLogger('jira_ext')):
        """
        dirty workaround since child class will need to init the added properties but no super.__init__ can be called
        """
        self.__logger = logger
        self.__issue_cache = collections.OrderedDict()
        self._field_mappings = collections.defaultdict(lambda: collections.defaultdict(lambda: None))

    @translate_resource_args
    def sprint_scope_and_burndown_chart(self, board_id, sprint_id):
        data = {}
        data['rapidViewId'] = board_id
        data['sprintId'] = sprint_id

        if self._options['agile_rest_path'] == GreenHopperResource.GREENHOPPER_REST_PATH:
            # Old, private API did not support pagination, all records were present in response,
            #   and no parameters were supported.
            r_json = self._get_json('rapid/charts/scopechangeburndownchart', params=data, base=self.AGILE_BASE_URL)
            # boards = [Board(self._options, self._session, raw_boards_json) for raw_boards_json in r_json['views']]
            # return ResultList(boards, 0, len(boards), len(boards), True)
            return r_json
        else:
            # return self._fetch_pages(Board, 'values', 'board', startAt, maxResults, params, base=self.AGILE_BASE_URL)
            print("Not implemented for this API Version!!")
            pass

    def sprint_report(self, board_id, sprint_id):
        """Return the total incompleted points this sprint."""
        return self._get_json('rapid/charts/sprintreport?rapidViewId=%s&sprintId=%s' % (board_id, sprint_id),
                              base=self.AGILE_BASE_URL)

    def get_linked_issues(self, issue, link_type="is precondition for", regex=".*", generic_filter=None,
                          load_linked_issues=True):
        '''
            Returns all issues linked to issue which:
                * have a link matching the regex type=x
                * linked issue's summary matches regex=x
        '''
        import re
        if not issue:
            print("issues is empty")
        result = ResultListExt()

        links = issue.fields.issuelinks
        # if they have already linked features check if the naming is okay and all the defined features are there
        # print(i.fields.summary)
        # feature_search_str=".*"+get_epic_full_name(i.fields.summary)
        # print(links)
        if links:
            for idx, l in enumerate(links):
                if hasattr(l, 'outwardIssue') and hasattr(l.type, 'outward'):
                    # print('outward: '+ l.type.outward+" "+l.outwardIssue.fields.summary)
                    # check if link is of right type
                    if re.search(link_type, l.type.outward):
                        # check linked issue name
                        if re.search(regex, l.outwardIssue.fields.summary, re.IGNORECASE):
                            # print('Outward: Feature found'+ i.fields.summary)
                            result.append(l.outwardIssue)

                if hasattr(l, 'inwardIssue') and hasattr(l.type, 'inward'):
                    # print('inward: '+ l.type.inward+" "+l.inwardIssue.fields.summary)
                    # check if link is of right type
                    if re.search(link_type, l.type.inward):
                        # check linked issue name
                        if re.search(regex, l.inwardIssue.fields.summary):
                            # print('Inward: Feature found'+ i.fields.summary)
                            result.append(l.inwardIssue)

        if result:
            cached = self.get_from_issue_cache(result)
            uncached = self.get_uncached_issues(result)
            if load_linked_issues and uncached:
                result = self.search_issues(self.create_jql_for_issuelist(uncached))
                self.update_issue_cache(result)
                result += cached
            else:
                dict_result = dict(np.array([[i.key for i in result], result]).transpose())
                result = [dict_result.get(i) for i in uncached]
                result += cached

            if generic_filter:
                result = [i for i in filter(generic_filter, result)]

        return result

    def search_issue_chunks_with_issuekeys(self, issue_keys, chunk_size=None):
        result = ResultListExt()
        if issue_keys is None:
            raise IOError("A list of issue_keys needs to be passed!")

        if issue_keys:
            if chunk_size:
                iks = np.array_split(issue_keys, np.ceil(len(issue_keys) / chunk_size))
                for i in iks:
                    jql = self.create_jql_for_issuelist(i.tolist())
                    result += self.search_issues(jql, maxResults=chunk_size)
            else:
                jql = self.create_jql_for_issuelist(issue_keys)
                result = self.search_issues(jql, maxResults=chunk_size)
            self.update_issue_cache(result)
        return result

    def search_issue_chunks_with_jql(self, jql, chunk_size=None):
        result = ResultListExt()
        if jql is None:
            raise IOError("An jql expression needs to be passed!")

        if not isinstance(jql, str):
            raise IOError("An string needs to be passed")

        if jql:
            if chunk_size:
                num_curr = chunk_size
                cnt = 0
                while chunk_size == num_curr:
                    i = self.search_issues(jql, startAt=cnt, maxResults=chunk_size)
                    result += i
                    num_curr = len(i)
                    cnt += len(i)
            else:
                result = self.search_issues(jql, maxResults=chunk_size)
            self.update_issue_cache(result)
        return result

    @staticmethod
    def parse_greenhopper_sprints(sprints):
        '''
        Parses the sprint string list and returns a dict with key:value
        '''
        sprints_parsed = []

        # if sprints were assigned parse them
        if sprints:
            for s in sprints:
                s = re.search('\[(.*)\]', s).group(1)
                # str_list = s.split(',')
                sprint_dict = Box({})
                while True:
                    m = re.match('(^[a-z,A-Z]*=).*?(?=,[a-z,A-Z]*=|$)', s)
                    if not m:
                        break

                    e = re.split('=', m.group())
                    s = s[m.end() + 1::]
                    # print(e)
                    if e[1] == '<null>':
                        e[1] = None
                    sprint_dict[e[0]] = e[1]
                sprints_parsed.append(sprint_dict)
        return sprints_parsed

    def issue_sprints(self, issue):
        '''
        Parses the sprint string list and returns a dict with key:value
        '''
        sprint = []

        # if issue has not the field included load it from server
        if 'customfield_10004' not in issue.raw['fields']:
            issue = self.issue(issue.key)

        return self.parse_greenhopper_sprints(issue.fields["customfield_10004"])

    def issue(self, id, fields=None, expand=None, cache=False, translate_custom_field_name=True):
        """Get an issue Resource from the server.

        :param id: ID or key of the issue to get
        :param fields: comma-separated string of issue fields to include in the results
        :param expand: extra information to fetch inside each resource
        """
        i = super(JIRA_EXT, self).issue(id, fields, expand)
        if translate_custom_field_name:
            i = IssueExt(i, field_mapping=self.get_field_mapping(i), jira_ext=self)
        else:
            i = IssueExt(i, jira_ext=self)
        if cache:
            self.update_issue_cache([i])
        return i

    def search_issues(self, *args, cache_result=True, translate_custom_field_name=True, **kwargs):
        issues = super(JIRA_EXT, self).search_issues(*args, **kwargs)
        if translate_custom_field_name:
            issues = ResultListExt(
                [IssueExt(i, field_mapping=self.get_field_mapping(i), jira_ext=self) for i in issues])
        else:
            issues = ResultListExt([IssueExt(i, jira_ext=self) for i in issues])
        if cache_result:
            self.update_issue_cache(issues)
        return issues

    def get_field_mapping(self, issue):
        if not ("project" in issue.raw["fields"] and "issuetype" in issue.raw["fields"]):
            i = self.issue(issue.key, fields=["project", "issuetype"])
        else:
            i = issue
        if self._field_mappings[i.fields.project.key][i.fields.issuetype.name]:
            return self._field_mappings[i.fields.project.key][i.fields.issuetype.name]
        else:
            meta = self.editmeta(i)["fields"]
            meta_mod = copy.deepcopy(meta)
            for k, v in meta.items():
                sanitized_name = re.sub(r'[*-+/\\.,~\- ]', r'_', v["name"].casefold())
                sanitized_name = re.sub(r'(_){2,}', r'_', sanitized_name)
                meta_mod[k]["sanitized_name"] = sanitized_name
                meta_mod[sanitized_name] = k
            self._field_mappings[i.fields.project.key][i.fields.issuetype.name] = meta_mod
            return meta_mod

    @property
    def issue_cache(self):
        return self.__issue_cache

    def update_issue_cache(self, issues):
        if not issues:
            return

        issues = self._listify(issues)
        d = dict(np.array([[i.key for i in issues], issues]).transpose())
        self.__issue_cache = collections.OrderedDict({**self.__issue_cache, **d})

    def clear_issue_cache(self, target_size=None):
        if target_size:
            for i in range(0, min([target_size, len(self.__issue_cache)])):
                self.__issue_cache.popitem(last=False)
        else:
            self.__issue_cache.clear()

    def get_from_issue_cache(self, key_list):
        if not key_list:
            return ResultListExt()
        key_list = self._listify(key_list)

        if isinstance(key_list[0], Issue):
            key_list = [i.key for i in key_list]

        result_in_cache = ResultListExt()
        for k in key_list:
            if k in self.__issue_cache:
                result_in_cache.append(self.__issue_cache[k])
        return result_in_cache

    def get_uncached_issues(self, key_list):
        if not key_list:
            return ResultListExt()
        key_list = self._listify(key_list)
        if isinstance(key_list[0], Issue):
            key_list = [i.key for i in key_list]
        result = ResultListExt()
        for k in key_list:
            if k not in self.__issue_cache:
                result.append(k)
        return result

    def save_cache_to_file(self, path=r".issue_cache.json"):
        with open(path, 'w') as f:
            cache = [i.raw for i in list(self.issue_cache.values())]
            json.dump(cache, f)

    def load_cache_from_file(self, path=r".issue_cache.json"):
        with open(path, 'r') as f:
            raw = json.load(f)
            issues = [Issue(self._options, self._session, r) for r in raw]
            self.update_issue_cache(issues)

    def save_issues_to_file(self, issue_list, path):
        with open(path, 'w') as f:
            cache = [i.raw for i in issue_list]
            json.dump(cache, f)

    def load_issues_from_file(self, path):
        with open(path, 'r') as f:
            raw = json.load(f)
            return ResultListExt([Issue(self._options, self._session, r) for r in raw])

    def get_uncached_issues_jql_request(self, issue_list):
        issues = self.get_uncached_issues(issue_list)
        return self.create_jql_for_issuelist(issues)

    def create_jql_for_issuelist(self, issuelist):
        jql = ""
        issuelist = self._listify(issuelist)
        if not issuelist:
            return ""
        if isinstance(issuelist[0], Issue):
            jql = [i.key for i in issuelist]
        jql = list(dict.fromkeys(issuelist))
        jql = " OR ".join("issuekey = {0}".format(i) for i in jql)
        return jql

    def create_recursive_issue_list(self, issues, link_type_regex, depth=None, generic_filter=None):
        '''
        Crawls all the issues from the given list of issues and searches all linked issues
        '''
        issue_list = ResultListExt()

        linked_issues = ResultListExt()
        # get all links of first level
        for i in issues:
            linked_issues += self.get_linked_issues(i, link_type=link_type_regex, load_linked_issues=False)

        # print("{0} {1}:".format(i.fields.summary,link_type_regex))
        if linked_issues:
            # remove duplicates
            linked_issues = list(dict(np.array([[k.id for k in linked_issues], linked_issues]).transpose()).values())
            # load from server all the fields of the linked issue. retrieve them with the created jql query
            cached = self.get_from_issue_cache([i.key for i in linked_issues])
            linked_issues = self.get_uncached_issues([i.key for i in linked_issues])

            linked_issues = self.search_issue_chunks_with_issuekeys(issue_keys=linked_issues, chunk_size=50)
            self.update_issue_cache(linked_issues)
            linked_issues += cached
            issue_list = linked_issues
            if bool(depth):
                depth_loc = depth - 1
                if depth_loc > 0:
                    issue_list += self.create_recursive_issue_list(linked_issues, link_type_regex,
                                                                   depth=depth_loc)
            else:
                issue_list += self.create_recursive_issue_list(linked_issues, link_type_regex, depth=depth)
            # for li in linked_issues:
            # print( "- {0}".format(i.fields.summary))
        if generic_filter:
            issue_list = filter(generic_filter, issue_list)
            issue_list = [i for i in issue_list]
        return issue_list

    def create_issue_graph(self, issues, link_type_regex, depth=None):
        '''
        Crawls all the issues from the given list of issues and searches all linked issues
        '''
        issue_hierarchy = ResultListExt()
        issue_cache = ResultListExt()

        for i in issues:
            li = [li for li in self.get_linked_issues(i, link_type=link_type_regex, load_linked_issues=False)]
            issue_cache += li
            issue_dict = {"issue": i, "linked_issues": li}
            issue_hierarchy.append(issue_dict)

        if issue_cache:
            cached = self.get_from_issue_cache(issue_cache)
            issue_cache = self.get_uncached_issues(issue_cache)

            issue_cache = self.search_issue_chunks_with_issuekeys(issue_keys=issue_cache, chunk_size=100)
            self.update_issue_cache(issue_cache)
            issue_cache += cached

            issue_cache = dict(np.array([[k.key for k in issue_cache], issue_cache]).transpose())

        for d in issue_hierarchy:

            linked_issues = [issue_cache[i.key] for i in d["linked_issues"]]
            # print("{0} {1}:".format(i.fields.summary,link_type_regex))
            if linked_issues:

                d["linked_issues"] = linked_issues
                if bool(depth):
                    depth_loc = depth - 1
                    if depth_loc > 0:
                        d["linked_issues"] = self.create_issue_graph(linked_issues, link_type_regex,
                                                                     depth=depth_loc)
                else:
                    d["linked_issues"] = self.create_issue_graph(linked_issues, link_type_regex, depth=depth)
                # for li in linked_issues:
                # print( "- {0}".format(i.fields.summary))
        return issue_hierarchy

    def _delistify(self, object):
        if isinstance(object, list):
            if np.size(object) == 1:
                return object[0]
        return object

    def _listify(self, object):
        '''
        Return a list of the passed object
        '''
        if isinstance(object, list):
            return object
        elif isinstance(object, type(np.array([]))):
            return object.tolist()
        elif not object:
            return []
        else:
            return [object]

    def move_to_sprint(self, issues, sprint, recursion_association=None, generic_filter=None, simulate=False):
        '''
        Move all issue with recursion relation to given sprint. Generic filter is a function which allows to filter
        the result of the recursion individually.

        Parameters:
        issues (list): a list of jira issues
        sprint (int): sprint id to which issues should be moved
        recursion_association (string): regular expression to follow linked relation (e.g. "Is mother of")
        generic_filer (func): function to filter recursion results before issues are moved

        Example usage:
        move_to_sprint(issues_ppc,j.get_sprint("11"),"Is mother of",lambda x: not bool(j.issue_curr_sprint(x)))
        '''
        issues_to_move = copy.copy(self._listify(issues))

        if recursion_association:
            issues_to_move += self.create_recursive_issue_list(issues_to_move, recursion_association)

        if generic_filter:
            f = filter(generic_filter, issues_to_move)
            issues_to_move = [i for i in f]

        self.__logger.info(f"The following issues were identified {self.sprint(sprint).name}:\n")
        [print(f"{i.key}\t{i.fields.summary}") for i in issues_to_move]

        if issues_to_move and not simulate:
            self.__logger.info(f"Moving now the issues")
            issues_to_move = [i.key for i in issues_to_move]
            self.add_issues_to_sprint(sprint, issues_to_move)
        else:
            self.__logger.info(f"Simulation active, nothing executed")

    @lru_cache(maxsize=32)
    def get_boards_by_name(self, board_regex):
        f = filter(lambda x, _board_regex=board_regex: re.search(_board_regex, x.name), self.boards())
        return [i for i in f]

    @lru_cache(maxsize=32)
    def get_all_sprints_from_boards(self, board_regex="(AR.*HUD|Audi HCP3 SAFe.*Capability)"):
        boards = self.get_boards_by_name(board_regex)
        sprints = []
        for b in boards:
            sprints += self.sprints(b.id)
        # remove duplicates
        sprints = list(dict(np.array([[k.id for k in sprints], sprints]).transpose()).values())
        return sprints

    @lru_cache(maxsize=32)
    def get_sprints_by_name(self, sprint_regex, board_regex="(AR.*HUD|Audi HCP3 SAFe.*Capability)"):
        sprints = self.get_all_sprints_from_boards(board_regex)
        f = filter(lambda x, _sprint_regex=sprint_regex: re.search(_sprint_regex, x.name), sprints)
        return [sprint for sprint in f]

    def get_sprint(self, pi, sprint="", team="(AR.*HUD)"):
        search_string = ""
        if not sprint:
            search_string = f"AHCP3.*(PI)*{pi}"
        else:
            search_string = f"(PI)*{pi}.*(S)*[0-9]{{0,1}}{sprint}.*{team}"

        self.__logger.debug(f"Search string: {search_string}")
        # print(search_string)
        sprints = self.get_sprints_by_name(search_string)

        if len(sprints) != 1:
            sprints = None
        return self._delistify(sprints)

    def comments(self, issue):
        return CommentList([CommentExt(i) for i in super(JIRA_EXT, self).comments(issue)])

    def comment(self, issue, comment):
        return CommentExt(super(JIRA_EXT, self).comment(issue, comment))

class IssueExt(Issue):
    """
    Convenience class for GCIP issues.
    +   print function shows also summary
    +   customfield names are translated to sanitized given name (works only with GCIPJira)
    """

    def __init__(self, issue=None, field_mapping=None, jira_ext=None, *args, **kwargs):
        if isinstance(issue, Issue):
            kwargs = {'options': issue._options,
                      'raw': issue.raw,
                      'session': issue._session}
        if field_mapping:
            self.__field_mapping = collections.defaultdict(lambda: None, field_mapping)
        else:
            self.__field_mapping = collections.defaultdict(lambda: None)
        super(IssueExt, self).__init__(*args, **kwargs)
        self._parse_raw(self.raw)
        self.__jira = jira_ext

    def __str__(self):
        return f"<{self.key}>\t\t<{self.fields.summary}>"

    def __repr__(self):
        return f"<{self.key}>\t\t<{self.fields.summary}>"

    def _parse_raw(self, raw):
        # add custom fieldnames to the dict if they exist
        raw["clear2custom"] = collections.defaultdict(lambda: None)
        for k, v in self.__field_mapping.items():
            if "fields" in raw:
                if k in raw["fields"] and k.startswith("customfield_"):
                    sanitized_name = v["sanitized_name"]
                    raw["fields"][sanitized_name] = raw["fields"][k]
                    raw["clear2custom"][sanitized_name] = k

        super(IssueExt, self)._parse_raw(raw)

    @property
    def sprints(self):
        ret = {}
        # if issue has not the field included load it from server
        if 'customfield_10004' not in self.raw['fields']:
            self.update()
        return JIRA_EXT.parse_greenhopper_sprints(self.fields.customfield_10004)

    @property
    def current_sprint(self):
        if self.sprints:
            return self.sprints[-1]
        return None

    def to_df(self):
        df = pd.DataFrame(self.raw["fields"])
        for k, v in self.raw.items():
            if k != "fields":
                df[k] = v
        return df

    def move_to_sprint(self, sprint):
        """
        Move Feature to given sprint and update issue fields
        @param sprint: id
        @return: None
        """
        self.__jira.add_issues_to_sprint(sprint, [self.key])
        self.update()

    @property
    def comments(self):
        return self.__jira.comments(self.key)

    def add_comment(self, body, visibility=None, is_internal=False):
        """
        Add a comment from the current authenticated user on the specified issue and return a Resource for it.
        The issue identifier and comment body are required.
        :param issue: ID or key of the issue to add the comment to
        :param body: Text of the comment to add
        :param visibility: a dict containing two entries: "type" and "value".
            "type" is 'role' (or 'group' if the JIRA server has configured
            comment visibility for groups) and 'value' is the name of the role
            (or group) to which viewing of this comment will be restricted.
        :param is_internal: defines whether a comment has to be marked as 'Internal' in Jira Service Desk
        Type:      method
        """
        self.__jira.add_comment(self.key, body, visibility=visibility, is_internal=is_internal)

class ResultListExt(ResultList):
    def __init__(self, iterable=None, mapping=None, **kwargs):
        super(ResultListExt, self).__init__(iterable, **kwargs)
        self.mapping = mapping

    def __getitem__(self, index):
        retval = super(ResultListExt, self).__getitem__(index)
        retval = type(self)(retval)
        return retval

    def raw(self):
        """
        export raw content of each issue
        """
        return [i.raw for i in self]

    def raw_flattened(self):
        return [self.flatten_fields(i.raw) for i in self]

    @staticmethod
    def flatten_fields(raw):
        ret_dict = {**raw.get("fields")}
        for key, value in raw.items():
            if key != "fields":
                ret_dict[key] = value
        return ret_dict

    def normalized_json(self, _mapping=None):
        """
        Export normalized json. If mapping is explicitly given, self.mapping property is ignored

        mapping must have structure:
        "normalized_name": "name",
        "function": function,
        """

        def getitem_from_dict(dataDict, mapList):
            """Iterate nested dictionary"""
            return reduce(getitem, mapList, dataDict)

        mapping = self.mapping
        if _mapping:
            mapping = _mapping
        if not mapping:
            return self.raw_flattened()

        normalized_list = []
        for i in self:
            d = {}
            i_flat_raw = self.flatten_fields(i.raw)
            for key, properties in mapping.items():
                field = properties.get("field")
                normalized_name = properties.get("normalized_name", key)
                fun = properties.get("function")
                field_content = getitem_from_dict(i_flat_raw, field.split(",./"))
                if field and fun:
                    d[normalized_name] = fun(field_content)
                elif field and not fun:
                    d[normalized_name] = field_content
                elif not field and fun:
                    d[normalized_name] = fun(i)
                else:
                    raise ValueError("Mapping input doesnt contain field nor function")
            normalized_list.append(d)
        return normalized_list

    def normalized_df(self, _mapping=None):
        normalized_json = self.normalized_json(_mapping=_mapping)
        return pd.DataFrame(normalized_json)

class CommentExt(Comment):
    def __init__(self, comment):
        super(CommentExt, self).__init__(options=comment._options, session=comment._session, raw=comment.raw)

    def __str__(self):
        return f"From: {self.author.displayName} {self.updated}\n{self.body}\n____END____"

    def __repr__(self):
        return f"From: {self.author.displayName} {self.updated}\n{self.body}\n____END____\n"

class CommentList(list):
    def __setitem__(self, key, value):
        comment = self[key]
        comment.update(body=value)


from flask import Flask, render_template, request, jsonify, make_response
import csv
import json
import sqlite3
import data.statements as sql_statement
import data.students as students_dictionary

app = Flask(__name__)

def create_table():
    connection = sqlite3.connect('training.db')
    mycursor = connection.cursor()
    mycursor.execute(sql_statement.sqlite_create_student_table)
    connection.commit()
    connection.close()

def reset():
    try:
        connection = sqlite3.connect('training.db')
        mycursor = connection.cursor()
        mycursor.execute(sql_statement.sqlite_drop_student_table)
        connection.commit()
        connection.close()
    except Exception as e:
        print("Table does not exist: ", e)

def insert_students():
    try:
        list_of_students_for_sql_insert = []
        for student in students_dictionary.list_students:
            """
            As we have a NOT NULL Constraint on given_name and family_name
            we need to compensate for this in our application logic since
            the student data has missing values,
            or in some way not adhering to our database design
            """
            if student["family_name"] is None:
                student["family_name"] = "Unknown"
            tuple_object = (student["given_name"], student["family_name"])
            list_of_students_for_sql_insert.append(tuple_object)
        print(list_of_students_for_sql_insert)
        connection = sqlite3.connect('training.db')
        mycursor = connection.cursor()
        mycursor.executemany('INSERT INTO python_students (given_name,family_name) VALUES (?,?)', list_of_students_for_sql_insert)
        connection.commit()
        connection.close()
        return {
            "ok": True,
            "message": "Students successfully inserted into database"
        }
    except Exception as e:
        text = "Something went wrong: " + str(e)
        print(text)
        return {
            "ok": False,
            "message": text
        }

def get_all_students():
    result = []
    try:
        connection = sqlite3.connect('training.db')
        mycursor = connection.cursor()
        query_result = mycursor.execute('SELECT given_name, family_name FROM python_students ORDER BY given_name')
        for row in query_result.fetchall():
            dictionary_object = {
                "given_name": row[0],
                "family_name": row[1],
            }
            result.append(dictionary_object)
        connection.close()
    except Exception as e:
        print("Student table does not exist: ", e)
    return result

""" ROUTES """

@app.route('/')
def index():
    # TODO: Create table
    return render_template('base.html', student_list=get_all_students())

@app.route('/hello')
def hello_world():
    return 'Hello, World!'

@app.route('/create/stuff', methods=["POST"])
def create_stuff():
    request_data = request.get_json()
    action = request_data['action']
    result = {
        "ok": False,
        "action": action,
    }
    if action == "make a table for our students":
        create_table()
        result["ok"] = True
    elif action == "insert students":
        insert_response = insert_students()
        result["ok"] = insert_response["ok"]
        if not insert_response["ok"]:
            result["ok"] = False
        result["message"] = insert_response["message"]
    elif action == "delete everything":
        reset()
        result["ok"] = True
        result["message"] = "Ready to start over"
    return jsonify(result)

@app.route('/export/<export_type>', methods=["GET"])
def export(export_type):
    if not export_type:
        return "Try json or csv"
    result = get_all_students()
    if export_type == "json":
        with open('students.json', "w") as file:
            dump = json.dumps(result, indent=4)
            file.write(dump)
        return jsonify(result)
    if export_type == "csv":
        with open('students.csv', 'w', newline='') as csvfile:
            spamwriter = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
            spamwriter.writerow(["First name", "Last name"])
            for student in result:
                spamwriter.writerow([student["given_name"], student["family_name"]])
        file = open('students.csv', 'r', newline='')
        return "<pre>" + file.read() + "</pre>"


#!/usr/binY/env python

## Program:   VMTK
## Module:    $RCSfile: vmtkpointselector.py,v $
## Language:  Python
## Date:      $Date: 2006/07/17 09:52:56 $
## Version:   $Revision: 1.20 $

##   Copyright (c) Luca Antiga, David Steinman. All rights reserved.
##   See LICENCE file for details.

##      This software is distributed WITHOUT ANY WARRANTY; without even
##      the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
##      PURPOSE.  See the above copyright notices for more information.

import vtk
import sys
from os import path

import vtkvmtk
import vmtkrenderer
import pypes

vmtkpointselector = 'vmtkPointselector'

class vmtkSeedSelector():

    def __init__(self):
        self._Surface = None
        self._SeedIds = None
        self._SourceSeedIds = vtk.vtkIdList()
        self._TargetSeedIds = vtk.vtkIdList()
        self.PrintError = None
        self.PrintLog = None
        self.InputText = None
        self.OutputText = None
        self.InputInfo = None

    def SetSurface(self,surface):
        self._Surface = surface

    def GetSurface(self):
        return self._Surface

    def GetSourceSeedIds(self):
        return self._SourceSeedIds

    def GetTargetSeedIds(self):
        return self._TargetSeedIds

    def Execute(self):
        pass

class vmtkPickPointSeedSelector(vmtkSeedSelector):

    def __init__(self):
        vmtkSeedSelector.__init__(self)
        self.PickedSeedIds = vtk.vtkIdList()
        self.PickedSeeds = vtk.vtkPolyData()
        self.vmtkRenderer = None
        self.OwnRenderer = 0
        self.Script = None

    def UndoCallback(self, obj):
        self.InitializeSeeds()
        self.PickedSeeds.Modified()
        self.vmtkRenderer.RenderWindow.Render()

    def PickCallback(self, obj):
        picker = vtk.vtkCellPicker()
        picker.SetTolerance(1E-4 * self._Surface.GetLength())
        eventPosition = self.vmtkRenderer.RenderWindowInteractor.GetEventPosition()
        result = picker.Pick(float(eventPosition[0]),float(eventPosition[1]),0.0,self.vmtkRenderer.Renderer)
        if result == 0:
            return
        pickPosition = picker.GetPickPosition()
        pickedCellPointIds = self._Surface.GetCell(picker.GetCellId()).GetPointIds()
        minDistance = 1E10
        pickedSeedId = -1
        for i in range(pickedCellPointIds.GetNumberOfIds()):
            distance = vtk.vtkMath.Distance2BetweenPoints(pickPosition,self._Surface.GetPoint(pickedCellPointIds.GetId(i)))
            if distance < minDistance:
                minDistance = distance
                pickedSeedId = pickedCellPointIds.GetId(i)
        if pickedSeedId == -1:
            pickedSeedId = pickedCellPointIds.GetId(0)
        self.PickedSeedIds.InsertNextId(pickedSeedId)
        point = self._Surface.GetPoint(pickedSeedId)
        self.PickedSeeds.GetPoints().InsertNextPoint(point)
        self.PickedSeeds.Modified()
        self.vmtkRenderer.RenderWindow.Render()

    def InitializeSeeds(self):
        self.PickedSeedIds.Initialize()
        self.PickedSeeds.Initialize()
        seedPoints = vtk.vtkPoints()
        self.PickedSeeds.SetPoints(seedPoints)

    def Execute(self):

        if (self._Surface == None):
            self.PrintError('vmtkPickPointSeedSelector Error: Surface not set.')
            return

        self._SourceSeedIds.Initialize()
        self._TargetSeedIds.Initialize()

        if not self.vmtkRenderer:
            self.vmtkRenderer = vmtkrenderer.vmtkRenderer()
            self.vmtkRenderer.Initialize()
            self.OwnRenderer = 1

        self.vmtkRenderer.RegisterScript(self.Script)

        glyphs = vtk.vtkGlyph3D()
        glyphSource = vtk.vtkSphereSource()
        glyphs.SetInput(self.PickedSeeds)
        glyphs.SetSource(glyphSource.GetOutput())
        glyphs.SetScaleModeToDataScalingOff()
        glyphs.SetScaleFactor(self._Surface.GetLength()*0.01)
        glyphMapper = vtk.vtkPolyDataMapper()
        glyphMapper.SetInput(glyphs.GetOutput())
        self.SeedActor = vtk.vtkActor()
        self.SeedActor.SetMapper(glyphMapper)
        self.SeedActor.GetProperty().SetColor(1.0,0.0,0.0)
        self.SeedActor.PickableOff()
        self.vmtkRenderer.Renderer.AddActor(self.SeedActor)

        ##self.vmtkRenderer.RenderWindowInteractor.AddObserver("KeyPressEvent", self.KeyPressed)
        self.vmtkRenderer.AddKeyBinding('u','Undo.',self.UndoCallback)
        self.vmtkRenderer.AddKeyBinding('space','Add points.',self.PickCallback)
        surfaceMapper = vtk.vtkPolyDataMapper()
        surfaceMapper.SetInput(self._Surface)
        surfaceMapper.ScalarVisibilityOff()
        surfaceActor = vtk.vtkActor()
        surfaceActor.SetMapper(surfaceMapper)
        surfaceActor.GetProperty().SetOpacity(1.0)

        self.vmtkRenderer.Renderer.AddActor(surfaceActor)

        self.InputInfo('Please position the mouse and press space to add source points, \'u\' to undo\n')

        any = 0
        while any == 0:
            self.InitializeSeeds()
            self.vmtkRenderer.Render()
            any = self.PickedSeedIds.GetNumberOfIds()
        self._SourceSeedIds.DeepCopy(self.PickedSeedIds)

        self.InputInfo('Please position the mouse and press space to add target points, \'u\' to undo\n')

        any = 0
        while any == 0:
            self.InitializeSeeds()
            self.vmtkRenderer.Render()
            any = self.PickedSeedIds.GetNumberOfIds()
        self._TargetSeedIds.DeepCopy(self.PickedSeedIds)

        if self.OwnRenderer:
            self.vmtkRenderer.Deallocate()

class vmtkNonManifoldSurfaceChecker(object):

    def __init__(self):

        self.Surface = 0

        self.NumberOfNonManifoldEdges = 0
        self.Report = 0
        self.NonManifoldEdgePointIds = vtk.vtkIdList()

        self.PrintError = None

    def Execute(self):

        if (self.Surface == 0):
            self.PrintError('NonManifoldSurfaceChecker error: Surface not set')
            return

        self.NonManifoldEdgesFound = 0
        self.Report = ''
        self.NonManifoldEdgePointIds.Initialize()

        neighborhoods = vtkvmtk.vtkvmtkNeighborhoods()
        neighborhoods.SetNeighborhoodTypeToPolyDataManifoldNeighborhood()
        neighborhoods.SetDataSet(self.Surface)
        neighborhoods.Build()

        neighborCellIds = vtk.vtkIdList()
        cellPointIds = vtk.vtkIdList()

        self.Surface.BuildCells()
        self.Surface.BuildLinks(0)
        self.Surface.Update()

        numberOfNonManifoldEdges = 0

        for i in range(neighborhoods.GetNumberOfNeighborhoods()):

            neighborhood = neighborhoods.GetNeighborhood(i)

            for j in range(neighborhood.GetNumberOfPoints()):

                neighborId = neighborhood.GetPointId(j)

                if (i<neighborId):

                    neighborCellIds.Initialize()
                    self.Surface.GetCellEdgeNeighbors(-1,i,neighborId,neighborCellIds)

                    if (neighborCellIds.GetNumberOfIds()>2):

                        numberOfNonManifoldEdges = numberOfNonManifoldEdges + 1

                        self.Report = self.Report +  "Non-manifold edge found" + str(i) + ' ' + str(neighborId) + '.\n'

                        self.NonManifoldEdgePointIds.InsertNextId(i)
                        self.NonManifoldEdgePointIds.InsertNextId(neighborId)

class vmtkPointselector(pypes.pypeScript):

    def __init__(self):

        pypes.pypeScript.__init__(self)

        self.Surface = None
        self.Output = None
        self.Centerlines = None
        self.SeedSelector = None
        self.SeedSelectorName = 'pickpoint'
        self.FlipNormals = 0
        self.CapDisplacement = 0.0
        self.RadiusArrayName = 'MaximumInscribedSphereRadius'
        self.CostFunction = '1/R'
        self.AppendEndPoints = 0
        self.CheckNonManifold = 0

        self.Resampling = 0
        self.ResamplingStepLength = 1.0
        self.SimplifyVoronoi = 0

        self.EikonalSolutionArrayName = 'EikonalSolution'
        self.EdgeArrayName = 'EdgeArray'
        self.EdgePCoordArrayName = 'EdgePCoordArray'
        self.CostFunctionArrayName = 'CostFunctionArray'

        self.UseTetGen = 0
        self.TetGenDetectInter = 1

        self.DelaunayTessellation = None
        self.VoronoiDiagram = None
        self.PoleIds = None

        self.DelaunayTolerance = 0.001

        self.SourceIds = []
        self.TargetIds = []
        self.SourcePoints = []
        self.TargetPoints = []

        self.vmtkRenderer = None
        self.OwnRenderer = 0

        self.SetScriptName('vmtkpointselector')
        self.SetScriptDoc('Choose points on a geometry that will be stored for later use')
        self.SetInputMembers([['Surface', 'i', 'vtkPolyData', 1, '',
                               'the input surface', 'vmtksurfacereader'],
                               ["Output", 'opath', 'str', 1, "", "Filepath to store points"]])
        self.SetOutputMembers([['Centerlines','o','vtkPolyData',1,'','the output centerlines',
                                'vmtksurfacewriter']])

    def PrintProgress(self,obj,event):
        self.OutputProgress(obj.GetProgress(),10)

    def Execute(self):
        self.SeedSelectorName = "pickpoint"

        if self.Surface == None:
            self.PrintError('Error: No input surface.')

        if self.CheckNonManifold:
            self.PrintLog('NonManifold check.')
            nonManifoldChecker = vmtkNonManifoldSurfaceChecker()
            nonManifoldChecker.Surface = self.Surface
            nonManifoldChecker.PrintError = self.PrintError
            nonManifoldChecker.Execute()

            if (nonManifoldChecker.NumberOfNonManifoldEdges > 0):
                self.PrintLog(nonManifoldChecker.Report)
                return

        self.vmtkRenderer = vmtkrenderer.vmtkRenderer()
        self.vmtkRenderer.Initialize()
        self.OwnRenderer = 1

        self.PrintLog('Cleaning surface.')
        surfaceCleaner = vtk.vtkCleanPolyData()
        surfaceCleaner.SetInput(self.Surface)
        surfaceCleaner.Update()

        self.PrintLog('Triangulating surface.')
        surfaceTriangulator = vtk.vtkTriangleFilter()
        surfaceTriangulator.SetInput(surfaceCleaner.GetOutput())
        surfaceTriangulator.PassLinesOff()
        surfaceTriangulator.PassVertsOff()
        surfaceTriangulator.Update()

        self.PrintLog('Capping surface.')
        surfaceCapper = vtkvmtk.vtkvmtkCapPolyData()
        surfaceCapper.SetInput(surfaceTriangulator.GetOutput())
        surfaceCapper.SetDisplacement(self.CapDisplacement)
        surfaceCapper.SetInPlaneDisplacement(self.CapDisplacement)
        surfaceCapper.Update()
        centerlineInputSurface = surfaceCapper.GetOutput()
        capCenterIds = surfaceCapper.GetCapCenterIds()

        self.SeedSelector = vmtkPickPointSeedSelector()
        self.SeedSelector.vmtkRenderer = self.vmtkRenderer
        self.SeedSelector.Script = self

        self.SeedSelector.SetSurface(centerlineInputSurface)
        self.SeedSelector.InputInfo = self.InputInfo
        self.SeedSelector.InputText = self.InputText
        self.SeedSelector.OutputText = self.OutputText
        self.SeedSelector.PrintError = self.PrintError
        self.SeedSelector.PrintLog = self.PrintLog
        self.SeedSelector.Execute()

        inletSeedIds = self.SeedSelector.GetSourceSeedIds()
        outletSeedIds = self.SeedSelector.GetTargetSeedIds()

        point_source = []
        points_target = []
        s = ""
        text_target = "target%s: %s\n"
        text_source = "source: %s\n"

        for i in range(inletSeedIds.GetNumberOfIds()):
            point_source.append(centerlineInputSurface.GetPoint(inletSeedIds.GetId(i)))
            s += text_source % (str(point_source[-1]))

        for i in range(outletSeedIds.GetNumberOfIds()):
            points_target.append(centerlineInputSurface.GetPoint(outletSeedIds.GetId(i)))
            s += text_target % (i, points_target[-1])

        f = open(path.join(self.Output, "points"), "w")
        f.write(s)
        f.close()

if __name__=='__main__':
    main = pypes.pypeMain()
    main.Arguments = sys.argv
    main.Execute()


import logging
import pickle
import os
import glob

from concurrent.futures import ThreadPoolExecutor

from tqdm.auto import tqdm

from eolearn.core import EOPatch

from eoflow.data_loader import EOMultiTempDataGenerator
from eoflow.utils import process_config
from eoflow.utils import create_dirs
from eoflow.utils import get_args

logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(levelname)s %(message)s')

def main():
    # capture the config path from the run arguments
    # then process the json configuration file
    try:
        args = get_args()
        config = process_config(args.config)
    except RuntimeError:
        print("missing or invalid arguments")
        exit(0)

    # create the experiments dirs
    create_dirs([config.log_dir, config.checkpoint_dir])

    data_creator = EOMultiTempDataGenerator(config)

    def save_patchlets(filename):
        eopatch = EOPatch.load(filename, lazy_loading=True)
        patch_name = filename.split('/')[-1]

        if not os.path.isdir(f'{config.save_patchlet_dir}{patch_name}/'):
            os.system(f'mkdir {config.save_patchlet_dir}{patch_name}/')

        if len(glob.glob(f'{config.save_patchlet_dir}{patch_name}/*')) == config.n_patchlets:
            return None

        for index in range(config.n_patchlets):
            seed = data_creator.get_seed_for_patch(filename, index)
            patchlet = data_creator.read_eopatch(filename, seed, eopatch)
            if patchlet is not None:
                pickle.dump(patchlet,
                            open(f'{config.save_patchlet_dir}{patch_name}/patchlet_{patch_name}_{index}.pkl', 'wb'))

    with open(os.path.join(config.log_dir, config.train_cval_log), 'rb') as f:
        train_set, _ = pickle.load(f)

    with ThreadPoolExecutor(max_workers=config.workers) as executor:
        _ = list(tqdm(executor.map(save_patchlets, train_set), total=len(train_set)))

if __name__ == '__main__':
    main()


from tqdm import tqdm
import sys
from pathlib import Path
import matplotlib.pyplot as plt
from vlib.image import draw_box_plt
from utils.train_utils import *

def test(cfgs):
    dcfg, mcfg, cfg = cfgs
    n_coop = cfg.TEST['n_coop'] if 'n_coop' in list(cfg.TEST.keys()) else 0
    com_range = cfg.TEST['com_range'] if 'com_range' in list(cfg.TEST.keys()) else 0

    test_dataloader = build_dataloader(dcfg, cfg, train=False)
    print(len(test_dataloader))
    # train on the GPU or on the CPU, if a GPU is not available
    device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
    # device = torch.device('cpu')

    model = build_model(mcfg, cfg, dcfg).to(device)

    log_path = Path(cfg.PATHS['run'])
    test_out_path = log_path / 'test_result_{:2d}m'.format(com_range)
    test_out_path.mkdir(exist_ok=True)
    # get metric
    from utils.eval import MetricAP
    metric = MetricAP(cfg.TEST, test_out_path, device='cuda', bev=cfg.TEST['bev'])
    thrs = cfg.TEST['ap_ious']

    if metric.has_test_detections:
        aps = [metric.cal_ap_all_point(IoU_thr=thr)[0] for thr in thrs]
        with open(test_out_path / 'thr{}_ncoop{}.txt'.format(cfg.TEST['score_threshold'], n_coop), 'w') as fh:
            for thr,  ap in zip(thrs, aps):
                fh.writelines('mAP@{}: {:.2f}\n'.format(thr, ap * 100))

    # load checkpoint
    model.load_state_dict(torch.load(str(log_path / 'epoch{:03}.pth'
                                         .format(cfg.TRAIN['total_epochs']))))
    # dir for save test images
    images_path = (test_out_path / 'images_{}_{}'.format(cfg.TEST['score_threshold'], n_coop))
    images_path.mkdir(exist_ok=True)
    load_data_to_device = load_data_to_gpu if device.type == 'cuda' else load_data_as_tensor
    model.eval()
    direcs = []
    i = 0
    with torch.no_grad():
        for batch_data in tqdm(test_dataloader):
            i += 1
            points = batch_data['points']
            points = points[points[:, 0] == 0, 1:3]
            boxes = batch_data['gt_boxes'][0]
            load_data_to_device(batch_data)

            # Forward pass
            batch_data = model(batch_data)
            predictions_dicts = model.post_processing(batch_data, cfg.TEST)
            pred_boxes = [pred_dict['box_lidar'] for pred_dict in predictions_dicts]
            scores = [pred_dict['scores'] for pred_dict in predictions_dicts]
            direcs.extend([boxes[:, -1] for boxes in pred_boxes])
            if i % 1 == 0:
                ax = plt.figure(figsize=(8, 8)).add_subplot(1, 1, 1)
                ax.set_aspect('equal', 'box')
                ax.set(xlim=(dcfg.pc_range[0], dcfg.pc_range[3]),
                       ylim=(dcfg.pc_range[1], dcfg.pc_range[4]))
                ax.plot(points[:, 0], points[:, 1], 'y.', markersize=0.3)
                ax = draw_box_plt(boxes, ax, color='green', linewidth_scale=2)
                ax = draw_box_plt(pred_boxes[0], ax, color='red')
                plt.xlabel('x')
                plt.ylabel('y')
                plt.savefig(str(images_path / '{}.png'.format(batch_data['frame'][0])))
                plt.close()

            metric.add_samples(batch_data['frame'], pred_boxes, batch_data['gt_boxes'], scores)

    metric.save_detections()
    aps = [metric.cal_ap_all_point(IoU_thr=thr)[0] for thr in thrs]
    with open(test_out_path / 'thr{}_ncoop{}.txt'.format(cfg.TEST['score_threshold'], n_coop), 'w') as fh:
        for thr, ap in zip(thrs, aps):
            fh.writelines('mAP@{}: {:.2f}\n'.format(thr, ap * 100))


import numpy as np

class Network(object):

    def __init__(self, layers):
        self.num_layers = len(layers)
        self.sizes = layers

        # Biases are the same per neuron and do not depend on the inputs from the previous layer.
        self.biases = [np.random.randn(outputs, 1) for outputs in layers[1:]]

        # Nielsens implementation used zip. Not very comfortable with the way he initialized.
        # np.random.randn(x,y) initilzes an x by y dimensional array with normally sampled elements.
        # self.weights = [np.random.randn(outputs, inputs) for outputs, inputs in zip(layers[1:], layers[:-1])]

        self.weights = [np.random.randn(layers[index + 1], layers[index]) for index in range(len(layers) - 1)]

        # for the sake of clarity on matrix multiplication front.
        self.matrix_sizes = [this_layer.shape for this_layer in self.weights]


from os.path import join, expanduser

RASA_TRAIN_PATH = 'data/rasa/'
RASA_TST_PATH = 'data/test/rasa/'
NLU_TRAIN_CONFIG_FILE = "nlu_model_config.json"
NLU_MODEL_PATH = 'saved_models/nlu/'
DIALOGUE_T5_MODEL_PATH = 'saved_models/dialogue_t5/'
DIALOGUE_T6_MODEL_PATH = 'saved_models/dialogue_t6/'
NLU_TEST_CONFIG_FILE = 'tests/data/nlu_model_test_config.json'
DSTC2_DATA_PATH = 'data/dstc2/data/'
DSTC2_ONTOLOGY_FILE = 'data/dstc2/config/ontology_dstc2.json'
DSTCT2_TRN_LIST_FILE = 'data/dstc2/config/dstc2_train.flist'
DSTCT2_DEV_LIST_FILE = 'data/dstc2/config/dstc2_dev.flist'
DSTCT2_TST_LIST_FILE = 'data/dstc2/config/dstc2_test.flist'
DOMAIN_T6_FILE = 'fabot_t6_domain.yml'
DOMAIN_T5_FILE = 'fabot_t5_domain.yml'
BABI_PATH = 'data/dialog-bAbI-tasks/'
BABI_T6_TRN_FILE = join(BABI_PATH, 'dialog-babi-task6-dstc2-trn.txt')
BABI_T6_DEV_FILE = join(BABI_PATH, 'dialog-babi-task6-dstc2-dev.txt')
BABI_T6_TST_FILE = join(BABI_PATH, 'dialog-babi-task6-dstc2-tst.txt')
BABI_T6_KB_FILE = join(BABI_PATH, 'dialog-babi-task6-dstc2-kb.txt')
BABI_T5_TRN_FILE = join(BABI_PATH, 'dialog-babi-task5-full-dialogs-trn.txt')
BABI_T5_DEV_FILE = join(BABI_PATH, 'dialog-babi-task5-full-dialogs-dev.txt')
BABI_T5_TST_FILE = join(BABI_PATH, 'dialog-babi-task5-full-dialogs-tst.txt')
BABI_T5_TST_OOV_FILE = join(BABI_PATH, 'dialog-babi-task5-full-dialogs-tst-OOV.txt')
BABI_T5_KB_FILE = join(BABI_PATH, 'dialog-babi-kb-all.txt')
BABI_T5_KB_OOV_FILE = join(BABI_PATH, 'dialog-babi-kb-oov.txt')
NLU_T5_MODEL_NAME = 't5'
NLU_T6_MODEL_NAME = 't6new'
W2VEC_MODEL_PATH = expanduser('~/GoogleNews-vectors-negative300.bin')
NLU_T6_MODEL_PATH = join(NLU_MODEL_PATH, NLU_T6_MODEL_NAME)
NLU_T5_MODEL_PATH = join(NLU_MODEL_PATH, NLU_T5_MODEL_NAME)
PERSISTED_MEMNET_PATH = 'saved_models/memnet_{bot_prev}/t{task}_{ent}_{feats}/'
PERSISTED_LSTM_PATH = 'saved_models/lstm_{bot_prev}/t{task}_{ent}_{feats}/'
PERSISTED_ENSEMBLE_T6 = 'saved_models/ensemble/t6/{ent}/{feats}/'


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import math

import draw_basic as db
import settings

#_______________________________________________________________________________
def draw(scale, wire_spacing, shield_wire=None, track=False, abc_label=True):
  settings.set_scale(scale)
  r = wire_spacing/math.sqrt(3)
  width = wire_spacing*4
  xcenter = settings.a4size[0]/2
  ycenter = settings.a4size[1]/2 - wire_spacing
  n = int(width/wire_spacing)+1
  for i in range(8):
    for w in range(n):
      x = xcenter-width/2+(w+0.5*(i%2))*wire_spacing
      y = ycenter+i*r*0.5
      if i%3 == 0:
        db.draw_line([x, y], wire_spacing/2, r/2)
      if i%3 == 1:
        db.draw_line([x, y], wire_spacing/2, -r/2)
        if i != 7:
          db.draw_line([x, y], 0, r)
  for i in range(8):
    for w in range(n):
      x = xcenter-width/2+(w+0.5*(i%2))*wire_spacing
      y = ycenter+i*r*0.5
      db.draw_circle([x, y], 1, 0 if i%3 == 2 else 1.0)
      if settings.target == 'SDC3' and i%3 == 2:
        x = xcenter-width/2+(w+0.5*((i+1)%2))*wire_spacing
        db.draw_circle([x, y], 1, 1)
  db.draw_text([xcenter+width/2+wire_spacing/2+4, ycenter+3.5*r-1],
               'Potential wire', centering=False)
  db.draw_text([xcenter+width/2+wire_spacing/2+4, ycenter+2.5*r-1],
               'Anode wire', centering=False)
  db.draw_line_with_scale([xcenter-width/2, ycenter-1], wire_spacing, -4)
  db.draw_line_with_scale([xcenter-width/2+wire_spacing, ycenter-1],
                          wire_spacing/2, -4)
  db.draw_line_with_scale([xcenter-width/2-1, ycenter], -4, r, True)
  db.draw_line_with_scale([xcenter-width/2-1, ycenter+r], -4, 1.5*r, True)
  if shield_wire is not None:
    for i in range(2):
      for w in range(int(width/shield_wire[0])+1):
        x = xcenter-width/2+wire_spacing/4+w*shield_wire[0]
        y = ycenter - shield_wire[1]
        if i == 1: y += 3.5*r + 2*shield_wire[1]
        db.draw_circle([x, y], 1, 0.7)
    db.draw_line_with_scale([xcenter-width/2+wire_spacing/4,
                             ycenter+3.5*r+shield_wire[1]+1],
                            shield_wire[0], 4)
    db.draw_line_with_scale([xcenter-width/2+wire_spacing/4-1,
                             ycenter+3.5*r],
                            -4, shield_wire[1], True)
    db.draw_text([xcenter+width/2+wire_spacing/4+4,
                  ycenter+3.5*r+shield_wire[1]-1],
                  #ycenter-shield_wire[1]-1],
                 'Shield wire', centering=False)
    # db.draw_text([xcenter+width/2+r*3.5, ycenter-shield_wire[1]-6], '[mm]')
  else:
    pass
    # db.draw_text([xcenter+width/2+r*3.5, ycenter-wire_spacing/2], '[mm]')
  if track:
    db.draw_text([xcenter+wire_spacing/4+22, ycenter+r*4.5], 'Charged particle')
    db.draw_arrow([xcenter+wire_spacing/8, ycenter-r], wire_spacing/4, 5.5*r, 2)
    db.draw_text([xcenter+r, ycenter+2.5*r+7.5], 'Drift', False, 4)
    db.draw_text([xcenter+r, ycenter+2.5*r+3.5], 'length', False, 4)
    db.draw_text([xcenter-0.5, ycenter+r+7.5], 'Drift', False, 4)
    db.draw_text([xcenter-0.5, ycenter+r+3.5], 'length', False, 4)
    length = math.sqrt(pow(5.5*r, 2) + pow(wire_spacing/4, 2))/(1.05*scale)
    normal_vector = [5.5*r/length, -wire_spacing/4/length]
    db.draw_arrow([xcenter+0.25*wire_spacing+0.2*scale, ycenter+2.5*r+0.1*scale],
               normal_vector[0], normal_vector[1])
    db.draw_arrow([xcenter+1+0.01*scale, ycenter+r-0.01*scale],
               normal_vector[0], normal_vector[1])
  if abc_label:
    db.draw_text([xcenter-width/2-wire_spacing, ycenter+r*4.5],
                 ('(a)' if settings.target == 'SDC1' else
                  '(b)' if settings.target == 'SDC2' else '(c)'))

#_______________________________________________________________________________
def draw_sdc1():
  scale = 4.0
  wire_spacing = 6*scale
  shield_wire = [wire_spacing, (20*scale-3.5*wire_spacing/math.sqrt(3))/2]
  draw(scale, wire_spacing, shield_wire)

#_______________________________________________________________________________
def draw_sdc2():
  scale = 24/9
  wire_spacing = 9*scale
  shield_wire = [wire_spacing, (10-7/4*6/math.sqrt(3))*scale]
  draw(scale, wire_spacing, shield_wire)

#_______________________________________________________________________________
def draw_sdc3():
  scale=24/20
  wire_spacing = 20*scale
  shield_wire = [2*wire_spacing, 6*scale]
  draw(scale, wire_spacing, shield_wire)


import os
import hickle
import numpy as np
import cv2

target_size = (192, 224)

parent = '/export/public/ukita/prednet/kitti_data/'

filenames = ['X_train', 'X_val', 'X_test', 'sources_train', 'sources_val', 'sources_test']

for filename in filenames:
    tmp = hickle.load(parent + filename + '.hkl')

    if 'sources' in filename:
        tmp_converted = tmp
    else:
        tmp_converted = np.zeros((len(tmp), ) + target_size + (3, ), dtype=tmp.dtype)
        for i in range(len(tmp)):
            tmp_converted[i] = cv2.resize(tmp[i], (target_size[1], target_size[0]))

    hickle.dump(tmp_converted, parent + filename + str(target_size[0]) + 'x' +
                               str(target_size[1]) + '.hkl', mode='w')


'''
Code for Credit Probe Network Embedding.

Author: Yu Han (yhanthu@gmail.com)

For more details, refer to the paper:
Network Embedding on Evolving Networks
Yu Han, Jie Tang and Zhibin Liu
'''

import os
import argparse
import scipy as sp
from scipy.sparse import csc_matrix
from scipy.sparse import linalg
from scipy.sparse.linalg import norm as snorm
from scipy.linalg import norm
from scipy.linalg import svd
from scipy.linalg import pinv
import networkx as nx
import matplotlib.pyplot as plt
import heapq
from collections import defaultdict
import random
import gzip

def parse_args():
    parser = argparse.ArgumentParser(description="Run CPNE.")
    parser.add_argument('--input', nargs='?', default='email-Eu-core.txt.gz', help='Input graph path')
    parser.add_argument('--output', nargs='?', default='result-', help='Result path')
    parser.add_argument('--proximity', nargs='?', default='neighborship', help='Proximity used. Default is "neighborship"')
    parser.add_argument('--K', nargs='?', type = int, default=100, help='Value of K. Default is 100')
    parser.add_argument('--dimensions', type=int, default=10, help='Number of dimensions. Default is 10.')
    parser.add_argument('--lambd', type=float, default=0.8, help='Value of lambda. Default is 0.8.')
    parser.add_argument('--weighted', dest='weighted', action='store_true', help='Boolean specifying (un)weighted. Default is unweighted.')
    parser.add_argument('--unweighted', dest='unweighted', action='store_false')
    parser.set_defaults(weighted=False)
    parser.add_argument('--directed', dest='directed', action='store_true', help='Graph is (un)directed. Default is undirected.')
    parser.add_argument('--undirected', dest='undirected', action='store_false')
    parser.set_defaults(directed=False)

    return parser.parse_args()

def read_graph(args):

    G = []
    day = 0
    start = 0
    g = nx.Graph()
    inputfile = gzip.open(args.input,'r')
    for line in inputfile:
        line = line.strip().split()
        if line[0] != '#' and len(line) == 3:
            if int(line[2]) < start:
                continue
            if (int(line[2]) - start) >= 86400 * 7:
                G.append(g)
                nodes = g.nodes()
                g = nx.Graph()
                g.add_nodes_from(nodes)
                start += 86400 * 7
                day += 1
                if day >= 10:
                    break
            if g.has_edge(int(line[0]), int(line[1])):
                g[int(line[0])][int(line[1])]['weight'] += 1
            else:
                g.add_edge(int(line[0]), int(line[1]), weight = 1)

    inputfile.close()

    nodes = set(g.nodes())
    for g in G:
        g.add_nodes_from(nodes)

    return G

def calc_M(args, G):
    M = []
    if args.proximity == 'neighborship':
        for g in G:
            M.append(nx.adjacency_matrix(g))
    return M

def learn_emb(args, G):
    M = calc_M(args, G)
    X = []
    S = []
    U = []
    V = []
    for m in M:
        u, s, vt = sp.sparse.linalg.svds(m.astype(float), args.dimensions)
        for i in range(args.dimensions):
            if u[0,i] < 0:
                u[:,i] = -u[:,i]
        x = u.dot(sp.diag(sp.sqrt(s)))
        X.append(x)
        S.append(s)
        U.append(u)
        V.append(sp.transpose(vt))
    return X, S, U, V

def compare_sval(args, S, S_hat, S_hat_rand, S_hat_rr, S_hat_degree, S_hat_close):
    L2 = []
    L2_rand = []
    L2_rr = []
    L2_degree = []
    L2_close = []

    for i in range(len(S)):
        L2.append(norm(S[i] - S_hat[i]))
        L2_rand.append(norm(S[i] - S_hat_rand[i]))
        L2_rr.append(norm(S[i] - S_hat_rr[i]))
        L2_degree.append(norm(S[i] - S_hat_degree[i]))
        L2_close.append(norm(S[i] - S_hat_close[i]))

    MG = []
    MG_rand = []
    MG_rr = []
    MG_degree = []
    MG_close = []

    MG.append(L2[1])
    for i in range(len(L2) - 2):
        MG.append(L2[i+2] + MG[i])
    MG_rand.append(L2_rand[1])
    for i in range(len(L2_rand) - 2):
        MG_rand.append(L2_rand[i+2] + MG_rand[i])
    MG_rr.append(L2_rr[1])
    for i in range(len(L2_rr) - 2):
        MG_rr.append(L2_rr[i+2] + MG_rr[i])
    MG_degree.append(L2_degree[1])
    for i in range(len(L2_degree) - 2):
        MG_degree.append(L2_degree[i+2] + MG_degree[i])
    MG_close.append(L2_close[1])
    for i in range(len(L2_close) - 2):
        MG_close.append(L2_close[i+2] + MG_close[i])

    outputfile = open(args.output + '1', 'w')

    print >> outputfile, ' '.join('%s' %d for d in MG)
    print >> outputfile, ' '.join('%s' %d for d in MG_rand)
    print >> outputfile, ' '.join('%s' %d for d in MG_rr)
    print >> outputfile, ' '.join('%s' %d for d in MG_degree)
    print >> outputfile, ' '.join('%s' %d for d in MG_close)

    outputfile.close()

    x = [i for i in range(len(MG))]

    fig = plt.figure('fig1')
    plt.plot(x, MG, 'b*-', label = 'CPNE', linewidth = 2.0)
    plt.plot(x, MG_rand, 'ro-', label = 'Random', linewidth = 2.0)
    plt.plot(x, MG_rr, 'gp-', label = 'Round Robin', linewidth = 2.0)
    plt.plot(x, MG_degree, 'y^-', label = 'Degree Centrality', linewidth = 2.0)
    plt.plot(x, MG_close, 'kh-', label = 'Closeness Centrality', linewidth = 2.0)
    plt.xlabel('Time Stamps')
    plt.ylabel('Magnitude Gap')
    plt.legend(loc = 'upper left')

def compare_svec(args, U, U_hat, U_hat_rand, U_hat_rr, U_hat_degree, U_hat_close, V, V_hat, V_hat_rand, V_hat_rr, V_hat_degree, V_hat_close):
    L = []
    L_rand = []
    L_rr = []
    L_degree = []
    L_close = []

    for i in range(len(U)):
        L.append(calc_gap(U[i], U_hat[i], V[i], V_hat[i]))
        L_rand.append(calc_gap(U[i], U_hat_rand[i], V[i], V_hat_rand[i]))
        L_rr.append(calc_gap(U[i], U_hat_rr[i], V[i], V_hat_rr[i]))
        L_degree.append(calc_gap(U[i], U_hat_degree[i], V[i], V_hat_degree[i]))
        L_close.append(calc_gap(U[i], U_hat_close[i], V[i], V_hat_close[i]))

    AG = []
    AG_rand = []
    AG_rr = []
    AG_degree = []
    AG_close = []

    AG.append(L2[1])
    for i in range(len(L2) - 2):
        AG.append(L2[i+2] + AG[i])
    AG_rand.append(L2_rand[1])
    for i in range(len(L2_rand) - 2):
        AG_rand.append(L2_rand[i+2] + AG_rand[i])
    AG_rr.append(L2_rr[1])
    for i in range(len(L2_rr) - 2):
        AG_rr.append(L2_rr[i+2] + AG_rr[i])
    AG_degree.append(L2_degree[1])
    for i in range(len(L2_degree) - 2):
        AG_degree.append(L2_degree[i+2] + AG_degree[i])
    AG_close.append(L2_close[1])
    for i in range(len(L2_close) - 2):
        AG_close.append(L2_close[i+2] + AG_close[i])

    outputfile = open(args.output + '2', 'w')

    print >> outputfile, ' '.join('%s' %d for d in AG)
    print >> outputfile, ' '.join('%s' %d for d in AG_rand)
    print >> outputfile, ' '.join('%s' %d for d in AG_rr)
    print >> outputfile, ' '.join('%s' %d for d in AG_degree)
    print >> outputfile, ' '.join('%s' %d for d in AG_close)

    outputfile.close()

    x = [i for i in range(len(AG))]

    fig = plt.figure('fig2')
    plt.plot(x, AG, 'b*-', label = 'CPNE', linewidth = 2.0)
    plt.plot(x, AG_rand, 'ro-', label = 'Random', linewidth = 2.0)
    plt.plot(x, AG_rr, 'gp-', label = 'Round Robin', linewidth = 2.0)
    plt.plot(x, AG_degree, 'y^-', label = 'Degree Centrality', linewidth = 2.0)
    plt.plot(x, AG_close, 'kh-', label = 'Closeness Centrality', linewidth = 2.0)
    plt.xlabel('Time Stamps')
    plt.ylabel('Magnitude Gap')
    plt.legend(loc = 'upper left')

def calc_gap(U1, U2, V1, V2):
    return ((norm(calc_P(U1) - calc_P(U2)))**2 + (norm(calc_P(V1) - calc_P(V2)))**2) / 2

def calc_P(X):
    return X.dot(pinv(X))

def update_graph(args, G):
    G_hat = []
    G_hat.append(G[0])
    G_hat.append(G[1])
    R = {}
    T = {}
    nodes = sorted(nx.nodes(G[0]))
    Nodes_id = {}
    for node in nodes:
        Nodes_id[node] = nodes.index(node)
        if args.proximity == 'neighborship':
            R[node] = calc_R(args, Nodes_id[node], G[0], G[1])

        T[node] = 1
    for t in range(2, len(G)):
        nodes_chosen = heapq.nlargest(args.K, R, key = R.get) # nodes with highest rewards

        # get new graph
        G_hat.append(get_new_graph(G[t], G_hat[t-1], nodes_chosen))

        for node in nodes_chosen:
            # update means of chosen nodes
            if args.proximity == 'neighborship':
                R[node] = (R[node]*T[node] + calc_R(args, Nodes_id[node], G[t-1], G[t])) / (T[node] + 1)

        # update parameters of all nodes
        for node in nodes:
            R[node] += sp.sqrt(sp.log(t) / T[node]) * args.lambd
    return G_hat

def update_graph_rand(args, G):
    G_hat = []
    G_hat.append(G[0])
    G_hat.append(G[1])
    for t in range(2, len(G)):
        nodes_chosen = random.sample(G[0].nodes(), args.K)
        G_hat.append(get_new_graph(G[t], G_hat[t-1], nodes_chosen))
    return G_hat

def update_graph_rr(args, G):
    G_hat = []
    G_hat.append(G[0])
    G_hat.append(G[1])
    nodes = G[0].nodes()
    for t in range(2, len(G)):
        flag = args.K * (t - 1) % nx.number_of_nodes(G[0])
        nodes_chosen = nodes[flag - args.K: flag]
        G_hat.append(get_new_graph(G[t], G_hat[t-1], nodes_chosen))
    return G_hat

def update_graph_degree(args, G):
    G_hat = []
    G_hat.append(G[0])
    G_hat.append(G[1])
    for t in range(2, len(G)):
        nodes_chosen = heapq.nlargest(args.K, nx.degree(G_hat[t-1]), key = nx.degree(G_hat[t-1]).get)
        G_hat.append(get_new_graph(G[t], G_hat[t-1], nodes_chosen))
    return G_hat

def update_graph_close(args, G):
    G_hat = []
    G_hat.append(G[0])
    G_hat.append(G[1])
    for t in range(2, len(G)):
        nodes_chosen = heapq.nlargest(args.K, nx.closeness_centrality(G_hat[t-1]), key = nx.closeness_centrality(G_hat[t-1]).get)
        G_hat.append(get_new_graph(G[t], G_hat[t-1], nodes_chosen))
    return G_hat

def calc_R(args, node_id, gs, gt):
    return sp.sqrt((snorm(nx.adjacency_matrix(gt)[node_id,:] - nx.adjacency_matrix(gs)[node_id,:]))**2 + (snorm(nx.adjacency_matrix(gt)[:, node_id] - nx.adjacency_matrix(gs)[:, node_id]))**2)

def calc_R_trans(args, node_id, ts, tt):
    return snorm(tt[node_id, :] - ts[node_id, :])

def get_new_graph(G, G_hat_last, nodes_chosen):
    G_hat = G_hat_last.copy()
    for node in nodes_chosen:
        for neighbor in nx.all_neighbors(G_hat, node):
            if G_hat.has_edge(node, neighbor):
                G_hat[node][neighbor]['weight'] = 0
            if G_hat.has_edge(neighbor, node):
                G_hat[neighbor][node]['weight'] = 0
    for node in nodes_chosen:
        for neighbor in nx.all_neighbors(G, node):
            if G.has_edge(node, neighbor):
                G_hat.add_edge(node, neighbor, weight = G[node][neighbor]['weight'])
            if G.has_edge(neighbor, node):
                G_hat.add_edge(neighbor, node, weight = G[node][neighbor]['weight'])
    return G_hat

if __name__ == '__main__':
    args = parse_args()

    G = read_graph(args)
    G_hat = update_graph(args, G)
    G_hat_rand = update_graph_rand(args, G)
    G_hat_rr = update_graph_rr(args, G)
    G_hat_degree = update_graph_degree(args, G)
    G_hat_close = update_graph_close(args, G)

    EMB, S, U, V = learn_emb(args, G)
    EMB_hat, S_hat, U_hat, V_hat = learn_emb(args, G_hat)
    EMB_hat_rand, S_hat_rand, U_hat_rand, V_hat_rand = learn_emb(args, G_hat_rand)
    EMB_hat_rr, S_hat_rr, U_hat_rr, V_hat_rr = learn_emb(args, G_hat_rr)
    EMB_hat_degree, S_hat_degree, U_hat_degree, V_hat_degree = learn_emb(args, G_hat_degree)
    EMB_hat_close, S_hat_close, U_hat_close, V_hat_close = learn_emb(args, G_hat_close)

    compare_sval(args, S, S_hat, S_hat_rand, S_hat_rr, S_hat_degree, S_hat_close)
    compare_svec(args, U, U_hat, U_hat_rand, U_hat_rr, U_hat_degree, U_hat_close, V, V_hat, V_hat_rand, V_hat_rr, V_hat_degree, V_hat_close)


# -*- coding: utf-8 -*-
"""
Created on Tue Jul 23 16:07:12 2019

@author: LokalAdm
"""

# -*- coding: utf-8 -*-
"""
Created on Mon Jul 22 11:35:45 2019

@author: LokalAdm
"""

import math
from vincenty import vincenty
import matplotlib.pyplot as plt
import numpy as np
import cartopy.crs as ccrs
import cartopy.feature as cfeature
import scipy.optimize as opt
def distance2 (ll1,ll2):
    return vincenty(ll1,ll2)**2

languages = ('de', 'hu', 'sr', 'sk', 'other')

#input data loading process here
file = open("D:/finaldata2.txt")
data = []
for line in file:
    l=line.rstrip().split('\t')
    lp = []
    for i in l:
        lp.append(float(i))
    data.append(lp)
#for line in data:
#    settlements.append(settlement(line[1],line[0], line[3], line[2], line[4], line[5], line[6]+line[7], sum(line[2:8]), sum(line[8:13]), sum(line[13:18]), sum(line[18:23])))
data = np.array(data)
#D = {'de':14, 'hu':14, 'sr':14, 'sk':14, 'other':14} #Testing purpose, do not trust
#Dinit = np.array([  2.8796892,  106.4799405,   10.09502287,   0.61053932,   0.84524969])#3.06733121
#Dinit = np.array([  3.06733109, 121.91322502,   5.24350383,  16.59367886,   0.79454929])
#Dinit = np.array([ 4.24138189e+04,  6.90749573e-01,  1.48903524e+05,  1.68392701e+02,       -2.38063878e+02])
Dinit = np.array([ -5.50833113e-07, 0])
#Dinit = np.array([  6,6,6,6,6,6])
#Dinit = np.array([-1.91255781e-08,  1.93718199e+01,  -1.91255781e-08, -1.91255781e-08,
#  -1.91255781e-08,  2.77199829e-01])
settlements = []
speaker0 = []
for line in data:
    settlements.append([line[1],line[0]])
    speaker0.append([line[3], line[2], line[4], line[5], line[6]+line[7]])
settlements = np.array(settlements)
speaker0 = np.array(speaker0)
speakerModelling = speaker0[:, 0:3]
emptyplaces = []
for i in range(len(speakerModelling.sum(axis = 1))):
    if speakerModelling.sum(axis = 1)[i]==0:
        emptyplaces.append(i)

observed1910 = np.copy(data[:, 18:23])
observed1910 [:, [0,1]] = observed1910 [:, [1,0]]

observed1900 = np.copy(data[:, 13:18])
observed1900 [:, [0,1]] = observed1900 [:, [1,0]]

observed1891 = np.copy(data[:, 8:13])
observed1891 [:, [0,1]] = observed1891 [:, [1,0]]

observed = {1910:observed1910, 1891:observed1891, 1900: observed1900}

observedTest = {}
for year in observed.keys():
    observedTest[year] = observed[year][:, 0:3]
pop = {}
pop[1881] = data[:,2:5].sum(axis = 1)
pop[1891] = data[:,8:11].sum(axis = 1)
pop[1900] = data[:,13:16].sum(axis = 1)
pop[1910] = data[:,18:21].sum(axis = 1)

for year in observed.keys():
#    print(year)
    for i in range(len(speakerModelling.sum(axis = 1))):
        if (observed[year][:, 0:3].sum(axis = 1))[i]==0:
            if i not in emptyplaces:
                emptyplaces.append(i)

speakerModelling = np.delete(speakerModelling, emptyplaces, 0)
settlements0 = np.delete(settlements, emptyplaces, 0)
for i in observedTest.keys():
    observedTest[i]= np.delete(observedTest[i], emptyplaces, 0)
for i in pop.keys():
    pop[i]= np.delete(pop[i], emptyplaces, 0)

for year in range(1882,1891):
    pop[year]= (1/(1891-1881))*((year-1881)*pop[1891]+(1891-year)*pop[1881])
for year in range(1892,1900):
    pop[year]= (1/(1900-1890))*((year-1891)*pop[1900]+(1900-year)*pop[1891])
for year in range(1901,1910):
    pop[year]= (1/(1910-1900))*((year-1900)*pop[1910]+(1910-year)*pop[1900])

distmat = np.zeros((len(settlements0),len(settlements0)))
for i in range(len(settlements0)):
    for j in range(len(settlements0)):
        distmat[i,j] = distance2((settlements0[i,0], settlements0[i,1]), (settlements0[j,0], settlements0[j,1]))
def findsse (D):
    constant = {}
    speaker = np.copy(speakerModelling)
    sse = 0
    langnum = 3
#    for k in range(langnum):
#
#        d = abs(D[k])
#        if d != 0:
#            matrix = 1/(4*math.pi*d)*np.exp(-1*distmat/(4*d))
#        else:
#            matrix = np.zeros((len(settlements0),len(settlements0)))
#        np.fill_diagonal(matrix, 1)
#        constant[languages[k]] = matrix
    h = np.zeros(3)
    h[0] = D[0]
    h[-1] = D[-1]
    h[1] = -D[0]-D[-1]
    for year in range(1882,1911):
        for k in range(langnum):

#            d = abs(D[k])
#            if d != 0:
#                matrix = 1/(4*math.pi*d)*np.exp(-1*distmat/(4*d))
#            else:
            matrix = np.zeros((len(settlements0),len(settlements0)))
#            np.fill_diagonal(matrix, 1)
            for i in range(len(settlements0)):
                matrix[i,i]= math.exp(h[k]*pop[year-1][i])
            constant[languages[k]] = matrix
        score = []
        for i in range(langnum):
            score.append(constant[languages[i]] @ speaker[:,i])
        score = np.transpose(np.array(score))
        totalscore = score.sum(axis=1)
        score = score / totalscore[:,None]
        speaker = score * pop[year][:, None]
        if year in [1891,1900,1910]:
#            for i in range(len(speaker)):
#                for j in range(len(speaker[i])):
#                    if observedTest[year][i,j]:
#                        sse += ((observedTest[year][i,j] - speaker[i,j])/observedTest[year][i,j])**2
            sse += ((observedTest[year] - speaker)**2).sum()
#    print ('ha')
    return sse
res = opt.minimize(findsse, Dinit, method = 'Nelder-Mead', tol = 0.01)
print(res.x)
#print(res.x)
print(res.success)
print(res.message)
print(res.fun)

constant = {}
D = res.x
constant = {}
speaker = np.copy(speakerModelling)
sse = 0
langnum = 3
#    for k in range(langnum):
#
#        d = abs(D[k])
#        if d != 0:
#            matrix = 1/(4*math.pi*d)*np.exp(-1*distmat/(4*d))
#        else:
#            matrix = np.zeros((len(settlements0),len(settlements0)))
#        np.fill_diagonal(matrix, 1)
#        constant[languages[k]] = matrix
h = np.zeros(3)
h[0] = D[0]
h[-1] = D[-1]
h[1] = -D[0]-D[-1]
print(h)
for year in range(1882,1911):
    for k in range(langnum):

#            d = abs(D[k])
#            if d != 0:
#                matrix = 1/(4*math.pi*d)*np.exp(-1*distmat/(4*d))
#            else:
        matrix = np.zeros((len(settlements0),len(settlements0)))
#            np.fill_diagonal(matrix, 1)
        for i in range(len(settlements0)):
            matrix[i,i]= math.exp(h[k]*pop[year-1][i])
        constant[languages[k]] = matrix
    score = []
    for i in range(langnum):
        score.append(constant[languages[i]] @ speaker[:,i])
    score = np.transpose(np.array(score))
    totalscore = score.sum(axis=1)
    score = score / totalscore[:,None]
    speaker = score * pop[year][:, None]
print(np.sum(speaker, axis = 0))
print(np.sum(observedTest[1910], axis = 0))

comparison = speakerModelling/pop[1881][:, None]*pop[1910][:, None]
print(np.sum(comparison, axis = 0))
print(math.sqrt(((observedTest[1910] - speaker)**2).sum()/speaker.size))
print(math.sqrt(((observedTest[1910] - comparison)**2).sum()/speaker.size))
print((abs(observedTest[1910] - speaker)).sum()/speaker.size)
print((abs(observedTest[1910] - comparison)).sum()/speaker.size)
def add_bg(w,e,s,n):
    ax = plt.axes(projection=ccrs.Mercator())
    ax.set_extent([w, e, s, n], crs=ccrs.PlateCarree())
#    ax.set_extent([-2, 2, -2, 2], crs=ccrs.PlateCarree())
    land = cfeature.NaturalEarthFeature(
        category='physical',
        name='land',
        scale='10m',
        facecolor=np.array([0,0,0]))
    borders = cfeature.NaturalEarthFeature(
        category='cultural',
        name='admin_0_boundary_lines_land',
        scale='10m',
        facecolor='none',
        edgecolor='black')
    intbor = cfeature.NaturalEarthFeature(
        category='cultural',
        name='admin_1_states_provinces',
        scale='10m',
        facecolor=cfeature.COLORS['land'])
    rivers = cfeature.NaturalEarthFeature(
        category='physical',
        name='rivers_lake_centerlines',
        scale='10m',
        edgecolor=cfeature.COLORS['water'],
        facecolor='none')
    rivers1 = cfeature.NaturalEarthFeature(
        category='physical',
        name='rivers_europe',
        scale='10m',
        edgecolor=cfeature.COLORS['water'],
        facecolor='none')
    lakes = cfeature.NaturalEarthFeature(
        category='physical',
        name='lakes',
        scale='10m',
        facecolor=cfeature.COLORS['water'])
    lakes1 = cfeature.NaturalEarthFeature(
        category='physical',
        name='lakes_europe',
        scale='10m',
        facecolor=cfeature.COLORS['water'])
    ax.add_feature(land)
    #ax.add_feature(cfeature.OCEAN)
    #ax.add_feature(cfeature.COASTLINE)
    ax.add_feature(intbor)
    ax.add_feature(borders)

    #ax.add_feature(cfeature.LAKES, alpha=0.5)
    ax.add_feature(rivers)
    ax.add_feature(rivers1)
    ax.add_feature(lakes)
    ax.add_feature(lakes1)
    return 0
def color (de, hu, sr, other):
    Total = de+hu+sr+other
    if Total:
        red = de/Total
        green = hu/Total
        blue = sr/Total
#        x_color = de / Total
#        y_color = hu/ Total
#        z_color = sr/ Total
#        radius = x_color + y_color + z_color
##        if x_color + y_color + z_color:
##            lat = z_color/(x_color + y_color + z_color) * math.pi/2
##        else:
##            lat = 0
##        if x_color + y_color:
##            long = x_color / (x_color + y_color) * math.pi/2
##        else:
##            long = 0
##        if radius:
##            subzcolor = z_color/(x_color + y_color + z_color)# * math.pi/2
##            if subzcolor != 1 :
##                lat = math.atan(subzcolor/(1-subzcolor))
##            else:
##                lat = math.pi/2
##        else:
##            lat = 0
##        if x_color + y_color:
##            subxcolor = x_color / (x_color + y_color)
##            if subxcolor != 1 :
##                long = math.atan(subxcolor/(1-subxcolor))
##            else:
##                long = math.pi/2
###        long = x_color / (x_color + y_color) * math.pi/2
##        else:
##            long = 0
##        red = radius * math.sin(long)* math.cos(lat)
##        green = radius * math.cos(long)* math.cos(lat)
##        blue = radius * math.sin(lat)
#        k = 0.71
#        cyan = np.array([0,1,1])
#        mag = np.array([1,0,1])
#        yellow = np.array([1,1,0])
#        final = k*(cyan * z_color + mag * x_color + yellow * y_color)
#        red = final[0]
#        green = final[1]
#        blue = final[2]
        return Total/50, red, green, blue
    else:
        return 0,0,0,0
#speaker = comparison
emptyplaces.sort()
speakerplot = np.copy(speaker)
for i in emptyplaces:
    speakerplot = np.insert(speakerplot, i, 0, axis = 0)
color1 = []
for i in range(len(speakerplot)):
    l = speakerplot[i]
    k = observed[1910][i]
    other = k[3]+k[4]
    color1.append(color(l[0], l[1], l[2], other))

color1 = np.array(color1)
size = np.transpose(color1)[0]
color1 = np.transpose(np.transpose(color1)[1:])

west, east, south, north = 15.7,18.0,46.7,48.4
add_bg(west, east, south, north)
plt.scatter(settlements[:,1], settlements[:,0], size, color1, transform=ccrs.PlateCarree(), zorder=2)
plt.title('Simulation results')
plt.show()

plotted = (speaker[:,0]-observedTest[1910][:,0])/observedTest[1910].sum(axis = 1)
west, east, south, north = 15.7,18.0,46.7,48.4
add_bg(west, east, south, north)
#print(plotted)
plt.scatter(settlements0[:,1], settlements0[:,0], 5, c=plotted, vmin=-max(abs(plotted)), vmax=max(abs(plotted)), transform=ccrs.PlateCarree(), zorder=2)
plt.title('Plot of errors from simulation')
plt.colorbar()
plt.show()


#
# Test function control submodel
#
import pybamm
import tests
import unittest

def external_circuit_function(variables):
    I = variables["Current [A]"]
    V = variables["Terminal voltage [V]"]
    liion_param = pybamm.LithiumIonParameters()

    return (
        V
        + I
        - pybamm.FunctionParameter(
            "Current plus voltage function",
            {"Time [s]": pybamm.t * liion_param.timescale},
        )
    )

class TestFunctionControl(unittest.TestCase):
    def test_public_functions(self):
        param = pybamm.LithiumIonParameters()
        submodel = pybamm.external_circuit.FunctionControl(
            param, external_circuit_function
        )
        variables = {"Terminal voltage [V]": pybamm.Scalar(0)}
        std_tests = tests.StandardSubModelTests(submodel, variables)
        std_tests.test_all()

if __name__ == "__main__":
    print("Add -v for more debug output")
    import sys

    if "-v" in sys.argv:
        debug = True
    pybamm.settings.debug_mode = True
    unittest.main()


"""
@Time    : 2021/4/20 18:50
@Author  : Xiao Qinfeng
@Email   : qfxiao@bjtu.edu.cn
@File    : cmd.py
@Software: PyCharm
@Desc    :
"""

def l2diff(x1, x2):
    """
    standard euclidean norm
    """
    res = ((x1 - x2) ** 2).sum().sqrt()
    return res

def moment_diff(sx1, sx2, k):
    """
    difference between moments
    """
    ss1 = (sx1 ** k).mean(0)
    ss2 = (sx2 ** k).mean(0)

    return l2diff(ss1, ss2)

def cmd(x1, x2, n_moments=5):
    """
    central moment discrepancy (cmd)
    objective function for keras models (theano or tensorflow backend)

    - Zellinger, Werner et al. "Robust unsupervised domain adaptation
    for neural networks via moment alignment," arXiv preprint arXiv:1711.06114,
    2017.
    - Zellinger, Werner, et al. "Central moment discrepancy (CMD) for
    domain-invariant representation learning.", ICLR, 2017.
    """
    mx1 = x1.mean(0)
    mx2 = x2.mean(0)
    sx1 = x1 - mx1
    sx2 = x2 - mx2
    dm = l2diff(mx1, mx2)
    scms = dm
    for i in range(n_moments - 1):
        # moment diff of centralized samples
        scms = scms + moment_diff(sx1, sx2, i + 2)
    return scms


# -*- coding: utf-8 -*-
"""
Created on Wed Feb 11 09:34:06 2015

@author: Admin
"""
import logging
import sys

def add_logging_handler(formatter, handler, name):

    logger = logging.getLogger(name)
    handler.setLevel(logging.DEBUG)
    handler.setFormatter(formatter)
    logger.addHandler(handler)

def config_logging(logging_dict, logging_format):
    name = logging_dict["name"]
    logger = logging.getLogger(name)

    formatter = logging.Formatter(logging_format)

    add_logging_handler(formatter, logging.StreamHandler(), name)
    logger.setLevel(logging_dict["logging_level"])

    add_logging_handler(formatter, logging.FileHandler(logging_dict["name"]), name)
    return logger

def get_logging_data(logger_data):
    logging_levels_dict = {"INFO": logging.INFO,
                           "DEBUG": logging.DEBUG,
                           "WARNING": logging.WARNING,
                           "ERROR": logging.ERROR,
                           "CRITICAL": logging.CRITICAL
                           }
    logger_path = logger_data.get("path")
    logging_level = logger_data.get("level")
    if logging_level not in logging_levels_dict.keys():
        sys.stdout.write("logging level %s doesn't exist, choosing default INFO\n" % logging_level)
        logging_level = "INFO"

    logging_dict = {"name": logger_path,
                    "logging_level": logging_level}
    return logging_dict

def get_logger(logger_data, logging_format):
    logging_dict = get_logging_data(logger_data)
    return config_logging(logging_dict, logging_format)


"""A conversion module for googletrans"""
from __future__ import print_function
import re
import json

from googletrans.constants import DEFAULT_API_KEY

def build_params(src, dest, token, fmt, **kwargs) -> dict:
    return {
        'client': 'te_lib',
        'sl': src,
        'tl': dest,
        'tk': token,
        'format': fmt,
        'v': '1.0',
        'key': DEFAULT_API_KEY,
        **kwargs,
    }

def legacy_format_json(original):
    # save state
    states = []
    text = original

    # save position for double-quoted texts
    for i, pos in enumerate(re.finditer('"', text)):
        # pos.start() is a double-quote
        p = pos.start() + 1
        if i % 2 == 0:
            nxt = text.find('"', p)
            states.append((p, text[p:nxt]))

    # replace all weired characters in text
    while text.find(',,') > -1:
        text = text.replace(',,', ',null,')
    while text.find('[,') > -1:
        text = text.replace('[,', '[null,')

    # recover state
    for i, pos in enumerate(re.finditer('"', text)):
        p = pos.start() + 1
        if i % 2 == 0:
            j = int(i / 2)
            nxt = text.find('"', p)
            # replacing a portion of a string
            # use slicing to extract those parts of the original string to be kept
            text = text[:p] + states[j][1] + text[nxt:]

    converted = json.loads(text)
    return converted

def get_items(dict_object):
    for key in dict_object:
        yield key, dict_object[key]

def format_json(original):
    try:
        converted = json.loads(original)
    except ValueError:
        converted = legacy_format_json(original)

    return converted

def rshift(val, n):
    """python port for '>>>'(right shift with padding)
    """
    return (val % 0x100000000) >> n


import kivy

import os

from kivy.app import App
from kivy.uix.gridlayout import GridLayout
from kivy.config import Config
from kivy.uix.listview import ListItemButton, ListView
from kivy.adapters.listadapter import ListAdapter
import psutil

from kivy.uix.button import Button

class Folder(GridLayout):
    elements = 0
    folder = "c:\\"

    def __init__(self, **kwargs):
        kwargs['cols'] = 2
        super(Folder, self).__init__(**kwargs)

        self.show_list()

    def callback(self, obj):
        if obj.selection[0].text == '[..]':
            folder = ''

            i = 0
            for element in self.elements[0:-1]:
                if(i==0):
                    folder = folder + element
                else:
                    folder = folder + '/' + element
                i = i+1

            self.folder = folder

        else:
            self.folder = self.folder + '/' + obj.selection[0].text

        self.show_list()
    def changeDisk(self,obj):
        self.folder = obj.text
        self.show_list()

    def show_list(self):
        self.ids.folder_list.clear_widgets()
        self.ids.disk.clear_widgets()

        self.elements = self.folder.split("/")

        if len(self.elements)<=1:
            list = []
        else:
            list = ['[..]']

        list = list + os.listdir(self.folder)

        listA = ListAdapter(
            data=list,
            selection_mode='single',
            allow_empty_selection=False,
            cls=ListItemButton)

        lista = ListView(adapter=listA, size_hint=(.2, 1.0))

        listA.bind(on_selection_change=self.callback)

        dps = psutil.disk_partitions()
        for dp in dps:
            button = Button(text = dp.device)
            button.bind(on_press=self.changeDisk)

            self.ids.disk.add_widget(button)

        self.ids.folder_list.add_widget(lista)
class Program(App):
    title = 'Folder list'
    icon = 'icon.png'

    def build(self):
        return Folder()

Config.set('graphics', 'fullscreen', '0')

Program().run()


from itertools import product
from random import shuffle
from types import MappingProxyType
from typing import Callable, Iterable, NoReturn, Tuple, Type
from uuid import uuid4, UUID

random_bytes: Callable[[int], bytes]
try:
    from secrets import token_bytes as random_bytes
except ImportError:
    from os import urandom as random_bytes

__all__ = ['BaseCharsEncodeError', 'BaseCharsDecodeError', 'BaseCharsProduct']

class BaseCharsEncodeError(ValueError):
    pass

class BaseCharsDecodeError(ValueError):
    pass

class BaseCharsProduct:

    __slots__: Tuple[str, ...] = ('chars', 'repeat', 'ivmap', 'vimap')

    _expected_chars_count: Tuple[int, ...] = (1 << 1, 1 << 2, 1 << 4, 1 << 8)
    _expected_chars_repeat: Tuple[int, ...] = (8, 4, 2, 1)

    def __init__(self, chars: str, do_shuffle: bool = False):
        cls: Type[BaseCharsProduct] = type(self)
        chars_count: int = len(chars)
        if not chars_count == len(set(chars)):
            raise ValueError(
                'Expected a string with different characters')
        repeat: int
        try:
            repeat = self._expected_chars_repeat[
                cls._expected_chars_count.index(chars_count)]
        except ValueError as e:
            raise ValueError(
                'Expected a string within %r different characters, get %d'
                % (cls._expected_chars_count, chars_count)) from e
        self.chars: str
        super().__setattr__('chars', chars)
        self.repeat: int
        super().__setattr__('repeat', repeat)
        prod: Iterable = product(chars, repeat=repeat)
        if do_shuffle:
            prod = list(prod)
            shuffle(prod)
        self.ivmap: MappingProxyType
        super().__setattr__('ivmap', MappingProxyType(
            {i: ''.join(l) for i, l in enumerate(prod)}))
        self.vimap: MappingProxyType
        super().__setattr__('vimap', MappingProxyType(
            {v: k for k, v in self.ivmap.items()}))

    def __setattr__(self, attr, val) -> NoReturn:
        raise TypeError('setting properties is not allowed')

    def encode(self, b: bytes) -> str:
        try:
            return ''.join(self.ivmap[i] for i in b)
        except Exception as e:
            raise BaseCharsEncodeError from e

    def decode(self, s: str) -> bytes:
        repeat: int = self.repeat
        try:
            return bytes(self.vimap[s[i:i+repeat]]
                         for i in range(0, len(s), repeat))
        except Exception as e:
            raise BaseCharsDecodeError from e

    def random(self, bytes_len: int = 1) -> Tuple[bytes, str]:
        rb: bytes = random_bytes(bytes_len)
        return rb, self.encode(rb)

    def random_uuid(self) -> Tuple[UUID, str]:
        uuid: UUID = uuid4()
        return uuid, self.encode(uuid.bytes)


"""
    This spider is a OntarioJobs spider created on top of the ATSSpider
    scrapy crawl ontariojobs -a mining_job_id=9999 -a iteration=1 -a extract=1 -a url="http://www.ontariojobs.com/gosearch.cfm?language=English"

    sample job url:
        http://www.ontariojobs.com/gosearch.cfm?task=viewjob&postid=452155&language=English

    NOTE(ted) - we will probably not want to run this spider since the job quality is really low.
        1. Many job listings are to recruiting companies so the company is not accurate (e.g. regisandsmith)
        2. Some jobs do not have an apply url (e.g. Great Clips)
"""

from datetime import datetime
from re import compile

from scrapy.http import Request
from scrapy.selector import Selector
from urlparse import urlparse

from brightcorp.base.atsspiders import ATSSpider
from brightcorp.items import BrightcorpItemLoader

pattern = {
    'Ref_Number' : compile(r'postid=(\d+)'),
}

class OntarioJobs(ATSSpider):

    name = "ontariojobs"

    def parse(self, response):
        '''
            Iterating over the list of jobs available on a single page,
            All the nodes display the same pattern tags, So taking the in-depth node and
            moving on to parent node, then capturing the data.
        '''
        selector = Selector(response)
        domain_url = urlparse(response.url)

        # Set expected_job_count
        if not self.expected_job_count_set:
            expected_job_count = selector.xpath("//font[@face='geneva,arial']/p/b/text()").extract()
            if expected_job_count:
                self.expected_job_count = expected_job_count

        jobs = selector.xpath('//tr/td//b[contains(text(),"Title")]/../../../following-sibling::tr')
        for job in jobs:
            posting_date = job.xpath('./td[3]/font/text()').extract()[0]
            yield Request(
                callback=self.parse_job,
                meta = {
                    'posting_date': posting_date
                },
                url='%(scheme)s://%(domain)s/%(job_url)s' % {
                    'scheme': domain_url.scheme,
                    'domain': domain_url.netloc,
                    'job_url': job.xpath('./td[@valign="top"]//a/@href').extract()[0],
                },
            )

        # Using the next node and requesting for the next page and parsing the data.
        next_page_url = selector.xpath('//b/a[contains(text(), "next records")]/@href')
        if next_page_url:
            yield Request(
                callback=self.parse,
                url='%(scheme)s://%(domain)s/%(next_page_url)s' % {
                    'scheme': domain_url.scheme,
                    'domain': domain_url.netloc,
                    'next_page_url': next_page_url.extract()[0],
                },
            )

    def parse_job(self, response):
        '''
            Collecting all the information available for a single job.
        '''
        selector = Selector(response)
        loader = BrightcorpItemLoader(selector=selector)
        if 'posting_date' in response.meta:
                post_date = datetime.strptime(response.meta['posting_date'], '%m/%d/%y').strftime('%Y/%m/%d')
                loader.add_value('date', post_date)
        referencenumber = 'ontariojobs-%s' % pattern['Ref_Number'].search(response.url).group(1)

        jobcategory = selector.xpath('//tr/td[@valign="top"]//p/b[contains(text(), "Job Category")]/following-sibling::text()').extract()
        jobcategory = ', '.join(jobcategory)

        loader.add_xpath(
            'company',
            '//tr/td[@valign="top"]//p/b[contains(text(), "Job Posted by")]/following-sibling::text()'
        )
        loader.add_xpath(
            'description',
            '//tr/td[@valign="top"]//p'
        )
        loader.add_xpath(
            'jobtype',
            '//tr/td[@valign="top"]//p/b[contains(text(), "Job Type")]/following-sibling::text()'
        )
        loader.add_xpath(
            'location',
            '//tr/td[@valign="top"]//b[contains(text(),"Location")]/following-sibling::text()'
        )
        loader.add_xpath(
            'title',
            '//tr/td[@valign="top"]//b[contains(text(),"Position")]/following-sibling::text()'
        )
        loader.add_xpath('baseSalary',
                         '//p[contains(text(), "Salary:")]/text()',
                         lambda a: [x.replace('Salary:', '') for x in a]
        )

        loader.add_value('apply_url', response.url.replace('&language=English', ''))
        loader.add_value('url', response.url)
        loader.add_value('jobcategory', jobcategory)
        loader.add_value('referencenumber', referencenumber)

        yield loader.load_item()