"""My gradebook."""

import json
import math
import re

from .defaults import default_student_sort, DEFAULT_FORMULA_OUTOF
from .shared import _make_gf_header, _make_gf_student_line, _make_csv_header
from .students import Student, Students

SEARCH_BY = ('student_number', 'utorid', 'gitid')

DEBUG = True

class GradeBook:
    """My own gradebook."""

    def __init__(self, studentgrades=None, dict_key='student_number',
                 outofs=None, comments=None, sanity_check=True):
        """Init a Gradesfile given:

        outofs: Dict[str, float] maps assigment name to max possible grade.
        dict_key: a Student attribute; key into studentgrades and comments.
           Most likely student_number or utorid.
        studentgrades: Dict[dict_key, Tuple(Student, Grades)].
        comments: Dict[dict_key, str].
        sanity_check: check created GradeBook for inconsistencies or errors?

        """

        self.outofs = dict(outofs) if outofs else {}
        self.studentgrades = dict(studentgrades) if studentgrades else {}
        self.dict_key = dict_key
        self.comments = dict(comments) if comments else {}
        if sanity_check:
            self.sanity_check()

    def sanity_check(self):
        """Check validity of this GradeBook."""

        try:
            _validate(self.studentgrades, self.dict_key,
                      self.outofs, self.comments)
        except AssertionError as error:
            print('WARNING: Invalid GradeBook. {}'.format(error))
            return False
        return True

    def __iter__(self):
        return iter(self.studentgrades)

    def __str__(self):
        result = str(self.outofs) + '\n\n'

        student_list = _sorted_student_grades(self.studentgrades)
        for student, grades in student_list:
            key = getattr(student, self.dict_key)
            result += '{}: {},{},{}\n'.format(
                key,
                student,
                grades,
                self.comments.get(key, ''))
        return result

    def __len__(self):
        return len(self.studentgrades)

    def __eq__(self, other):

        if len(self.studentgrades) != len(other.studentgrades):
            return False

        if self.outofs != other.outofs:
            return False

        for (student, grades) in self.studentgrades.values():
            search_attrs = dict(
                (attr, getattr(student, attr))
                for attr in SEARCH_BY if getattr(student, attr) is not None)
            other_record = other.get_student_info(search_attrs)
            if not other_record:
                return False
            if grades != other_record[1]:
                if DEBUG:
                    print('different grades for {}: {} vs {}'.format(
                        student, grades, other_record[1]))
                return False
        return True

    def get_student_info(self, search_by):
        """search_by is a Dict[attribute, attr_value]. Return (Student,
        Grades) if a Student matches any of the search_by, or None if
        no such Student."""

        for student, grades in self.studentgrades.values():
            if any(getattr(student, attribute) == attr_value
                   for attribute, attr_value in search_by.items()):
                return (student, grades)
        return None

    def get_students(self):
        """Return a Students object with this Gradebook' students."""

        return Students(record[0] for record in self.studentgrades.values())

    def get_student_grades_by_utorid(self, utorid):
        """Return Grades of a Student with utorid. Raise NoSuch? if no such
        Student."""

        return self.get_student_grades_by_attribute('utorid', utorid)

    def get_student_grades_by_student_number(self, student_number):
        """Return Grades of a Student with student_number. Raise NoSuch? if no
        such Student."""

        return self.get_student_grades_by_attribute('student_number',
                                                    student_number)

    def get_student_grades_by_attribute(self, attribute, attr_value):
        """Return Grades of a Student with value of attribute equal to
        attr_value.  Raise NoSuch? if no such Student. Raise
        AttributeError if attribute is not present in at least one
        Student.

        """

        for student, grades in self.studentgrades.values():
            if getattr(student, attribute) == attr_value:
                return grades
        raise Exception('No student with {} value {}.'.format(
            attribute, attr_value))

    def get_dict(self, key):
        """Return a Dict[key, Tuple(Student, Grades).

        Useful when self.key = 'student_number' and we want the dict
        by 'utorid', for example.

        """

        if key == self.dict_key:
            return self.studentgrades

        new_key_to_student_grades = {}
        for student, grades in self.studentgrades.values():
            try:
                key = getattr(student, key)
                new_key_to_student_grades[key] = (student, grades)
            except AttributeError:
                print('WARNING: '
                      'Student does not have attribute {}:\n\t{}'.format(
                          key, student))
        return new_key_to_student_grades

    @staticmethod
    def load_gf_file(infile, dict_key='student_number', use_utorid=True):
        """Read gf grades file and create a new GradeBook. This GradeBook will
        have dict_key as its dictionary key.

        """

        lines = infile.readlines()
        sep = lines.index('\n')

        header = lines[:sep + 1]
        assts, outofs = _make_out_of_from_gf_header(header)

        stnum_to_student_grades = {}  # gf files are by student number
        stnum_to_comment = {}
        for line in lines[sep + 1:]:
            student = Student.make_student_from_gf_line(line, use_utorid)
            if not student:  # this is a comment/individual formula line
                stnum, comment = _make_comment_from_gf_line(line)
                stnum_to_comment[stnum] = comment
                continue

            grades = Grades.make_grades_from_gf_line(line, assts)
            stnum_to_student_grades[student.student_number] = (student, grades)

        gradebook = GradeBook(stnum_to_student_grades,
                              'student_number', outofs,
                              stnum_to_comment)

        gradebook.to_key(dict_key)
        return gradebook

    def to_key(self, key):
        """Convert this Gradebook's dictionaries of student_grades and
        comments to have the new key."""

        if key == self.dict_key:
            return

        dict_key_to_student_grades = {}
        dict_key_to_comments = {}

        for old_key, (student, grades) in self.studentgrades.items():
            try:
                new_key = getattr(student, key)
            except AttributeError:
                print('WARNING: '
                      'Student does not have attribute {}:\n\t{}'.format(
                          key, student))
                continue
            dict_key_to_student_grades[new_key] = (student, grades)
            if old_key in self.comments:
                dict_key_to_comments[new_key] = self.comments[old_key]
        self.studentgrades = dict_key_to_student_grades
        self.comments = dict_key_to_comments
        self.dict_key = key

    def write_gf(self, outfile, assts=None, utorid=True,
                 key=default_student_sort):
        """Write a gf file to outfile.

        assts is an iterable of asst names: the order in which they
        will appear in the gf.  If assts is None, the order will be
        alphabetical and all grades will be included.  utorid: Include
        a field for utorid?

        """

        outofs = _sort_outofs(self.outofs, assts)
        header = _make_gf_header(outofs, utorid)
        outfile.write(header + '\n')

        student_grades_list = _sorted_student_grades(self.studentgrades, key)
        for student, grades in student_grades_list:
            comment = self.comments.get(getattr(student, self.dict_key), '')
            line = _make_gf_student_line(
                student, utorid, grades, outofs, comment)
            outfile.write(line)

    def get_grades_dict(self, asst, key='student_number'):
        """Return Dict[key, grade:float] of grades for assignment asst.

        asst is the assignment name for which the grades will be collected.
        """

        return {getattr(student, key):
                self.get_student_grades_by_student_number(
                    student.student_number).get_grade(asst)
                for student in self.get_students()}

    def write_csv_grades_file(self, outfile, student_attrs=None,
                              header=True, comments=True, assts=None,
                              key=default_student_sort, names=None):
        """Write a csv grades file to outfile.

        student_attrs is an iterable of Student attributes to be
           included in the file, in the order in which they will
           appear.
        assts is an iterable of asst names: the order in which they
          will appear in the gf.  If assts is None, the order will be
          alphabetical and all grades will be included.
        comments: include comments as a last column?
        names is Dict[asst-or-attr, new-name] specifies how the header
           is created, in case we want it to be different from just
           the names of Student attributes and assignment names as
           stored. One of the keys can be 'comments' to replace the
           word 'comments' in the header.

        """

        if student_attrs is None:
            student_attrs = []
        if assts is None:
            assts = []

        if header:
            outfile.write(_make_csv_header(
                student_attrs, assts, comments, names))

        student_grades_list = _sorted_student_grades(self.studentgrades, key)
        for student, grades in student_grades_list:
            student_grades = ''.join(
                [',{}'.format(grades.get_grade(asst)) for asst in assts])
            student_comment = (',{}'.format(self.comments.get(
                getattr(student, self.dict_key), '')) if comments else '')
            outfile.write('{}{}{}\n'.format(
                student.full_str(student_attrs),
                student_grades,
                student_comment))

    def write_csv_submit_file(self, outfile, asst='all',
                              exam_no_shows=None, attribute='student_number'):
        """Write a CSV submit file for eMarks to outfile.

        asst is the name of the "final mark" assignment
        exam_no_shows is an iterable of values of attribute of Student's

        """

        if exam_no_shows is None:
            exam_no_shows = []

        for student, grades in self.studentgrades.values():
            try:
                grade = grades.get_grade(asst)
            except KeyError:
                print('WARNING: '
                      'No grade for assignment {} for student:\n\t{}'.format(
                          asst, student))
                continue

            no_show = getattr(student, attribute) in exam_no_shows
            outfile.write('{},{}{}\n'.format(student.student_number,
                                             # submit file needs integers
                                             min(math.ceil(grade), 100),
                                             ',y' if no_show else ''))

class Grades:
    """Essentially a dictionary of grades."""

    def __init__(self, grades=None):
        self.grades = {}
        if grades:
            for asst, grade in grades.items():
                self.add_grade(asst, grade)

    def __iter__(self):
        return iter(self.grades)

    def add_grade(self, assignment, grade=0):
        """Add/update grade for assignment. Raise TypeError if assignment is
        not a str or if grade cannot be converted to float.

        """

        grade = _clean_grade(grade)
        assignment = _clean_asst(assignment)
        self.grades[assignment] = grade

    def add_grades(self, grades):
        """Add/update grades from dictionary grades.

        """

        for assignment, grade in grades:
            self.add_grade(assignment, grade)

    def get_grade(self, assignment):
        """Return the grade for assignment. Raise KeyError if no such
        assignment.

        """

        try:
            return self.grades[assignment]
        except KeyError:
            raise KeyError('No such assignment: {}'.format(assignment))

    def get_assignments(self):
        """Return a set of assignments in these Grades."""

        return set(self.grades.keys())

    @staticmethod
    def make_grades_from_gf_line(line, assts):
        """Create Grades from a line in a gf file. Return None if line does
        not contain student grades (e.g., it is a comment or header line.)

        """

        fields = line.strip().split(',')
        match = re.fullmatch(
            r'(\d+) [ dx][ dx] ([^,]+)((\s+([^,]+))+)', fields[0])

        # this is not a gf line with student information
        if match is None:
            return None

        grades = Grades()

        if len(fields) == 1:
            return grades

        if not fields[1].isdigit():  # line contains utorid
            fields = fields[2:]
        else:
            fields = fields[1:]

        # pad with zeros for missing grades
        grades.add_grades(
            zip(assts, fields + [0] * (len(assts) - len(fields))))
        return grades

    def to_json(self):
        """Return a JSON of the grades."""

        return json.dumps(self.grades, sort_keys=True, indent=4)

    def __str__(self):
        return str(self.grades)

    def __eq__(self, other):

        if other is None:
            return False

        for asst, grade in self.grades.items():
            if asst not in other.grades:
                return False
            if not grades_equal(grade, other.grades[asst]):
                return False
        return True

def _clean_grade(grade):
    if grade == '' or str(grade).lower() == 'gwr':
        return 0.0
    try:
        return float(grade)
    except ValueError:
        raise TypeError('Invalid type for grade: {} of type {}.'.format(
            grade, type(grade)))

def _clean_asst(assignment):
    if not isinstance(assignment, str):
        raise TypeError('Invalid type for assignment: {} of type {}.'.format(
            assignment, type(assignment)))

    return assignment.strip()

def _make_out_of_from_gf_header(header):
    # TODO FIX collecting calculated grades
    outofs = {}
    assts = []
    for line in header:
        match = re.fullmatch(r'(\w+)\s*/\s*(\d+)\n', line)
        if match:
            asst = _clean_asst(match.group(1))
            outof = _clean_grade(match.group(2))
            outofs[asst] = outof
            assts.append(asst)
            continue
        match = re.match(r'(\w+)\s*=', line)  # calculated grade
        if match:
            asst = _clean_asst(match.group(1))
            outof = DEFAULT_FORMULA_OUTOF
            outofs[asst] = outof
            assts.append(asst)
            continue
    return (assts, outofs)

def _make_comment_from_gf_line(line):
    match = re.fullmatch(r'(\d+)[*]\s+(.+)', line.strip())
    if match:
        return match.group(1), match.group(2)
    return None

def _sort_outofs(outofs, asst_order):
    """Return a List[(asst, outof)] in the order in which asstignments
    appear in asst_order. If asst_order is None, sort in alphabetical
    order.
    outofs is a Dict{asst, outof}
    """

    if asst_order is None:
        result = sorted(outofs.items(), key=lambda pair: pair[0])
    else:
        try:
            result = [(asst, outofs[asst]) for asst in asst_order]
        except KeyError as error:
            raise KeyError(
                'Invalid assignment list {}: {}'.format(outofs, error))
    return result

def _sorted_student_grades(stnum_to_student_grades, key=default_student_sort):
    student_grades_list = list(stnum_to_student_grades.values())

    def sort_key(record):  # sort by student, according to key
        return key(record[0])

    student_grades_list.sort(key=sort_key)
    return student_grades_list

def _validate(student_grades, dict_key, outofs, comments):

    for key, (student, grades) in student_grades.items():
        assert getattr(student, dict_key) == key
        assert grades is None or grades.get_assignments() == outofs.keys()

    assert all(key in student_grades for key in comments)

def grades_equal(grade1, grade2):
    """Return whether grade1 and grade2 are the same grade, with 0.1
    precision."""

    return round(grade1, 1) == round(grade2, 1)


from pyspark_easy.utils import *
from pyspark.sql import functions as F
from texttable import Texttable
from pyspark.sql import DataFrame

class pyspark_easy(object):

    def __init__(self, df):
        if not (isinstance(df, DataFrame)):
            print("oops! It is not a dataframe")
            exit
        self.df = df
        self.dfcount = df.count()
        self.dfdistcount = df.distinct().count()
        self.columns = df.columns
        self.no_of_columns = len(df.columns)
        self.col_types = df.dtypes

    # pyspark summary - columns by types, observations, duplicate rows, no.of columns,summary stats and missing values stats
    def summary(self, summary_stats='N'):

        # count no.of rows, variables and columns

        t = Texttable()

        t.add_rows([['The number of observations/rows :', self.dfcount],
                    ['the number of variables/columns :', self.no_of_columns],
                    ["No of Duplicate rows :", str(self.dfcount - self.dfdistcount)],
                    ["% of Duplicate rows :", str(round((1 - ((self.dfdistcount / self.dfcount))) * 100, 10))]],
                   header=False)

        print(t.draw())

        # list out the columns by types

        string = 0
        numeric = 0
        date_time = 0
        other_type = 0
        string_columns = []
        numeric_columns = []
        date_time_columns = []
        other_type_columns = []
        for i in self.col_types:
            if len(i) > 1:
                if 'string' in i[1] or 'char' in i[1]:
                    string = string + 1
                    string_columns.append(list(i))
                elif 'time' in i[1] or 'date' in i[1]:
                    date_time = date_time + 1
                    date_time_columns.append(list(i))
                elif 'decimal' in i[1] or 'double' in i[1] or 'int' in i[1] or 'float' in i[1]:
                    numeric = numeric + 1
                    numeric_columns.append(list(i))
                else:
                    other_type = other_type + 1
                    other_type_columns.append(list(i))
        print("\n")
        print("The number of string columns are :", string)
        if string > 0:
            print("\n")
            print("The string columns are :")
            t = Texttable()
            string_columns.insert(0, ['Columns', 'Types'])
            t.add_rows(string_columns)
            print(t.draw())
        print("\n")
        print("The number of numerical columns are :", numeric)
        if numeric > 0:
            print("\n")
            print("The numerical columns are :")
            t = Texttable()
            numeric_columns.insert(0, ['Columns', 'Types'])
            t.add_rows(numeric_columns)
            print(t.draw())
        print("\n")
        print("The number of date/time columns are :", date_time)
        if date_time > 0:
            print("\n")
            print("The date/time columns are :")
            t = Texttable()
            date_time_columns.insert(0, ['Columns', 'Types'])
            t.add_rows(date_time_columns)
            print(t.draw())
        if other_type > 0:
            print("\n")
            print("The number of columns with other data types are :", other_type)
            print("\n")
            print("The other columns with data types are :")
            t = Texttable()
            other_type_columns.insert(0, ['Columns', 'Types'])
            t.add_rows(other_type_columns)
            print(t.draw())

        # summary stats for numerical columns
        if summary_stats == 'Y':
            numeric_cols = Extract(numeric_columns)[1:]
            if len(numeric_cols) > 0:
                print("\n")
                print("Summary statistics for numerical columns are :")
                print("\n")
                for i in range(0, len(numeric_cols), 5):
                    dff = self.df.select([F.col(c) for c in numeric_cols[i:i + 5]])
                    dff.summary().show()

        # missing values

        miss_values = self.df.select([F.count(F.when(F.col(c).isNull(), c)).alias(c) for c in self.columns]).toPandas()
        miss_values = miss_values.loc[:, (miss_values != 0).any(axis=0)]
        miss_c = list(miss_values)
        if len(miss_c) <= 0:
            print("\n")
            print("There are no columns with missing values")
        else:
            miss_v = list(miss_values.values)[0]
            percent_values = []
            miss_main_list = []
            for i in range(len(miss_c)):
                miss_sub_list = []
                miss_sub_list.append(miss_c[i])
                miss_sub_list.append(miss_v[i])
                miss_sub_list.append(round((miss_v[i] / self.dfcount) * 100, 2))
                miss_main_list.append(miss_sub_list)
            print("\n")
            print("Number of columns with missing values are :", len(miss_c))
            print("\n")
            print("The columns with missing values are :")
            print("\n")
            t = Texttable()
            miss_main_list.insert(0, ['Columns', 'No. of missing values', '% of missing values'])
            t.add_rows(miss_main_list)
            print(t.draw())


import oci
from ociextirpater.OCIClient import OCIClient

class oke( OCIClient ):
    service_name = "OKE"
    clientClass = oci.container_engine.ContainerEngineClient

    objects = [
        {
            "formatter"          : lambda cluster: "OKE Cluster with OCID {} / name '{}' is in state {}".format(cluster.id, cluster.name, cluster.lifecycle_state),
            "function_list"      : "list_clusters",
            "kwargs_list"        : {
                                        "lifecycle_state": [
                                            "ACTIVE",
                                            "FAILED",
                                            "UPDATING"
                                        ]
                                   },
            "function_delete"    : "delete_cluster",
            "name_singular"      : "OKE Cluster",
            "name_plural"        : "OKE Clusters",
        },

        # {
        #     # "formatter"          : lambda instance: "XXX instance with OCID {} / name '{}' is in state {}".format( instance.id, instance.name, instance.lifecycle_state ),
        #     "function_list"      : "list_xxx",
        #     "kwargs_list"        : {
        #                            },
        #     "function_delete"    : "delete_xxx",
        #     "name_singular"      : "XXX",
        #     "name_plural"        : "XXXXs",
        # },
    ]


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('agenda', '0010_auto_20171012_1549'),
    ]

    operations = [
        migrations.RemoveField(
            model_name='agenda',
            name='conta_core',
        ),
        migrations.AlterField(
            model_name='agenda',
            name='conta',
            field=models.ForeignKey(to='core.Conta', null=True),
        ),
    ]


from __future__ import annotations

import asyncio
import logging
from typing import TYPE_CHECKING, List, Optional, Type, Union

import aiohttp
import pendulum

from ._types import HolodexPaginatedVideo
from ._types import HolodexVideo as HolodexVideoPayload
from ._types import HolodexVideoStatus
from .models import HolodexVideo

if TYPE_CHECKING:
    from internals.vth import SanicVTHell

__all__ = ("HolodexAPI",)

logger = logging.getLogger("Internals.Holodex")

class HolodexAPI:
    BASE = "https://holodex.net/api/v2/"

    def __init__(self, api_key: Optional[str] = None, *, loop: Optional[asyncio.AbstractEventLoop] = None):
        self.api_key = api_key
        self._loop = loop or asyncio.get_event_loop()

        self.client: aiohttp.ClientSession = None
        self.__ready: bool = False

    @property
    def ready(self) -> bool:
        return self.__ready

    async def create(self):
        self.client = aiohttp.ClientSession(
            loop=self._loop,
            headers={
                "Content-Type": "application/json",
                "User-Agent": "VTHell/3.0.0 (+https://github.com/noaione/vthell)",
            },
        )
        if self.api_key:
            self.client.headers.update({"X-APIKEY": self.api_key})
        self.__ready = True

    @staticmethod
    def _convert_date_to_unix(date_time: Optional[str]) -> Optional[int]:
        if date_time is None:
            return None
        dt = pendulum.parse(date_time, tz="UTC")
        return dt.int_timestamp

    async def get_channel_videos(self, channel_id: str):
        params = {
            "limit": 50,
            "include": "live_info",
        }
        request_path = f"{self.BASE}channels/{channel_id}/videos"
        async with self.client.get(request_path, params=params) as response:
            if response.status != 200:
                return []
            response_json: List[HolodexVideoPayload] = await response.json()

        coerced_data: List[HolodexVideo] = []
        for video in response_json:
            video_type = video.get("type")
            if video_type != "stream":
                continue
            start_time = self._convert_date_to_unix(video.get("start_actual"))
            if start_time is None:
                start_time = self._convert_date_to_unix(video.get("start_scheduled"))
            channel_id = video.get("channel_id") or video.get("channel", {}).get("id")
            if channel_id is None:
                continue

            org_group = video.get("channel", {}).get("org")
            video_url = f"https://youtube.com/watch?v={video['id']}"
            is_member = "member" in video.get("topic_id", "").lower()
            coerced_data.append(
                HolodexVideo(
                    video["id"],
                    video["title"],
                    start_time,
                    channel_id,
                    org_group,
                    video["status"],
                    video_url,
                    is_member,
                )
            )

        return coerced_data

    async def close(self):
        if self.client and not self.client.closed:
            await self.client.close()

    @staticmethod
    def to_int(number: Union[str, int]):
        if isinstance(number, str):
            return int(number)
        return number

    async def _get_videos_paginated(self, status: HolodexVideoStatus, endpoint: str):
        sort_by = "available_at"
        if status == "upcoming":
            sort_by = "start_scheduled"
        elif status == "live":
            sort_by = "start_actual"
        limitation = 50
        params = {
            "type": "stream",
            "include": "live_info",
            "sort": sort_by,
            "order": "asc",
            "limit": limitation,
            "paginated": "true",
            "max_upcoming_hours": 48,
        }
        if status is not None:
            params["status"] = status
        collected_videos: List[HolodexVideoPayload] = []
        offset = 0
        while True:
            params["offset"] = offset
            async with self.client.get(f"{self.BASE}{endpoint}", params=params) as response:
                if response.status != 200:
                    break
                response_json: List[HolodexPaginatedVideo] = await response.json()

            total_item = self.to_int(response_json.get("total", "0"))
            if total_item < 1:
                break

            collected_videos.extend(response_json["items"])
            offset += limitation + 1
            if len(collected_videos) >= total_item:
                break

        return collected_videos

    async def get_lives(self):
        async with self.client.get(f"{self.BASE}live") as response:
            if response.status != 200:
                logger.error(f"Failed to get live videos: {response.status}")
                return []
            try:
                results: List[HolodexVideoPayload] = await response.json()
            except Exception:
                logger.exception("Failed to parse live videos")
                return []

        coerced_data: List[HolodexVideo] = []
        for video in results:
            video_type = video.get("type")
            if video_type != "stream":
                continue
            start_time = self._convert_date_to_unix(video.get("start_actual"))
            if start_time is None:
                start_time = self._convert_date_to_unix(video.get("start_scheduled"))
            channel_id = video.get("channel_id") or video.get("channel", {}).get("id")
            if channel_id is None:
                continue

            org_group = video.get("channel", {}).get("org")

            video_url = f"https://youtube.com/watch?v={video['id']}"
            is_member = "member" in video.get("topic_id", "").lower()
            coerced_data.append(
                HolodexVideo(
                    video["id"],
                    video["title"],
                    start_time,
                    channel_id,
                    org_group,
                    video["status"],
                    video_url,
                    is_member,
                )
            )

        return coerced_data

    async def get_video(self, video_id: str) -> Optional[HolodexVideo]:
        params = {
            "id": video_id,
            "include": "live_info",
        }

        async with self.client.get(f"{self.BASE}videos", params=params) as resp:
            if resp.status != 200:
                return None
            json_resp: List[HolodexVideoPayload] = await resp.json()

        if len(json_resp) < 1:
            return None

        selected_video = json_resp[0]

        if selected_video is None:
            return None

        video_type = selected_video.get("type")
        if video_type != "stream":
            # Dont archive clipper stuff because why would you do that?
            return None
        stream_status = selected_video.get("status")
        if stream_status is None:
            # Somehow it's missing the status? Remove it.
            return None
        if stream_status == "missing":
            # Stream is private, dont add.
            return None
        start_time = self._convert_date_to_unix(selected_video.get("start_actual"))
        if start_time is None:
            start_time = self._convert_date_to_unix(selected_video.get("start_scheduled"))
        channel_id = selected_video.get("channel_id") or selected_video.get("channel", {}).get("id")
        if channel_id is None:
            return None
        org_group = selected_video.get("channel", {}).get("org")

        video_url = f"https://youtube.com/watch?v={selected_video['id']}"
        is_member = "member" in selected_video.get("topic_id", "").lower()
        return HolodexVideo(
            selected_video["id"],
            selected_video["title"],
            start_time,
            channel_id,
            org_group,
            stream_status,
            video_url,
            is_member,
        )

    @classmethod
    def attach(cls: Type[HolodexAPI], app: SanicVTHell):
        config = app.config.get("HOLODEX_API_KEY")

        async def init_holodex_api(app: SanicVTHell):
            holodex = cls(config, loop=app.loop)
            logger.info("Initializing Holodex API")
            await holodex.create()
            app.holodex = holodex
            app.mark_holodex_ready()
            logger.info("Holodex API initialized")

        app.add_task(init_holodex_api)


"""
county_population.py creates a map of the population in each US county
It uses population data from https://www.kaggle.com/stevepalley/us-county-population/version/1, CO-EST2015-alldata.csv
"""
import pandas as pd
import numpy as np
from bokeh.plotting import figure
from bokeh.io import show
from bokeh.models import (
    ColumnDataSource,
    HoverTool,
    LogColorMapper,
    Range1d
)
from bokeh.palettes import brewer
import get_map

palette = brewer['YlGnBu'][9]
palette[8] = '#cccccc'

map_high_res = "cb_2015_us_county_500k"
map_low_res = "cb_2015_us_county_20m"
map_file = map_low_res

local_dir = "/home/alex/Documents/data/maps/"  # dir to place the data in
url = "http://www2.census.gov/geo/tiger/GENZ2015/shp/"
get_map.download_map_data(map_file, local_dir, url)
map_df = get_map.get_map_data(map_file, local_dir, 5, 0)
map_df.rename(columns={'province': 'county'}, inplace=True)
state_names = get_map.fix_state_names(map_df)

# population data for the counties
pop = pd.read_csv("/home/alex/Documents/data/CO-EST2015-alldata.csv", header=0, encoding="ISO-8859-1")
pop = pop[['STNAME', 'CTYNAME', 'POPESTIMATE2010']]
str_splitter = lambda x: x.rsplit(' ', 1)[0]
pop['CTYNAME'] = pop['CTYNAME'].apply(str_splitter)
# exclude Alaska
#pop = pop[pop['STNAME'] != 'Alaska'].reset_index()
# exclude Hawaii
#pop = pop[pop['STNAME'] != 'Hawaii'].reset_index()
# Select just one state
#pop = pop[pop['STNAME'] == 'California'].reset_index()

# match population and map shape data
x_lat = []
y_lon = []
state = []
county = []
population = []
for j in map_df.index:
    q = np.intersect1d(np.where(pop['CTYNAME'] == map_df.get_value(j, 'county'))[0],
                       np.where(pop['STNAME'] == map_df.get_value(j, 'state'))[0])
    if len(q) == 1:
        x_lat.append(map_df.get_value(j, 'x'))
        y_lon.append(map_df.get_value(j, 'y'))
        county.append(map_df.get_value(j, 'county'))
        state.append(map_df.get_value(j, 'state'))
        population.append(pop.get_value(q[0], 'POPESTIMATE2010'))
    if len(q) == 2:
        """
        The population data has rows that have the aggregate state population data.
        Many state have counties that are the same as the state.
        So in these cases's the correct county population should be the lesser population value.
        """
        x_lat.append(map_df.get_value(j, 'x'))
        y_lon.append(map_df.get_value(j, 'y'))
        county.append(map_df.get_value(j, 'county'))
        state.append(map_df.get_value(j, 'state'))
        county_population = np.min([pop.get_value(q[0], 'POPESTIMATE2010'), pop.get_value(q[1], 'POPESTIMATE2010')])
        population.append(county_population)

map_df = pd.DataFrame({'x': x_lat, 'y': y_lon, 'state': state, 'county': county, 'population': population})
cd_source = ColumnDataSource(map_df)

palette.reverse()
color_mapper = LogColorMapper(palette=palette)

tools = "pan,wheel_zoom,box_zoom,reset,hover,save"
left, right, bottom, top = -126, -67, 25, 50
#left, right, bottom, top = get_map.reasonable_bounds(map_df['x'], map_df['y'])
p = figure(
    title="Example County Plot", tools=tools, x_axis_location=None, y_axis_location=None,
    plot_width=950, plot_height=600, x_range=Range1d(left, right), y_range=Range1d(bottom, top))

p.grid.grid_line_color = None

p.patches('x', 'y', source=cd_source,
          fill_color={'field': 'population', 'transform': color_mapper},
          fill_alpha=0.7, line_color="white", line_width=0.5)

hover = p.select_one(HoverTool)
hover.point_policy = "follow_mouse"
hover.tooltips = [
    ("State", "@state"),
    ("County", "@county"),
    ("value", "@population"),
    ("(Long, Lat)", "($x, $y)")]
show(p)


import torch

from torch import optim
from torch.utils.data import DataLoader

from digitrecognizer.dataset import Mnist, mnist_dataset
from digitrecognizer.network import Net
import torch.nn as nn

def data(file, test_ratio):
    trainset, testset = mnist_dataset(file, test_ratio)

    trainloader = DataLoader(trainset)

    testloader = DataLoader(testset)

    return trainloader, testloader

if __name__ == '__main__':
    trainloader, testloader = data('input/train.csv', 0.1)

    net = Net()

    criterion = nn.CrossEntropyLoss()
    optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)

    for epoch in range(2):  # loop over the dataset multiple times

        running_loss = 0.0
        for i, data in enumerate(trainloader, 0):
            # get the inputs
            inputs, labels = data

            # zero the parameter gradients
            optimizer.zero_grad()

            # forward + backward + optimize
            outputs = net(inputs)
            loss = criterion(outputs, labels)
            loss.backward()
            optimizer.step()

            # print statistics
            running_loss += loss.item()

            if i % 2000 == 1999:  # print every 2000 mini-batches
                print('[%d, %5d] loss: %.3f' %
                      (epoch + 1, i + 1, running_loss / 2000))
                running_loss = 0.0

    print('Finished Training')


from bs4 import BeautifulSoup
from urllib.request import urlopen, Request
from datetime import date

def tool(url):
    repeat = True
    hdr = {
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
        'Accept-Encoding': 'none',
        'Accept-Language': 'en-US,en;q=0.8',
        'Connection': 'keep-alive'}

    req = Request(url=url, headers=hdr)

    # Fetching URL source code
    response = urlopen(req)
    webcontent = response.read()
    webcontent = str(webcontent)

    # Writing source code to text file
    f = open('novatech.html', 'w')
    f.write(webcontent)
    f.close()

    with open('novatech.html') as html_file:
        soup = BeautifulSoup(html_file, 'lxml')

    dic = {'type': None, 'brand': None, 'name': None, 'price': None, 'site': None,
           'pub_date': date.today().strftime("%d-%m-%Y")}

    names = []
    for a in soup.find_all(class_="search-box-liner search-box-results search-hover"):
        b = a.find('h2')
        c = b.find('a')
        names.append(c.text)

    prices = []
    for x in soup.find_all(class_="search-box-liner search-box-results search-hover"):
        for a in x.find_all('p', class_="newspec-price-listing"):
            a = a.text.strip('£')
            a = a.strip(' inc vat')
            prices.append(a)

    sites = []
    for a in soup.find_all(class_="search-box-liner search-box-results search-hover"):
        b = a.find('h2')
        c = b.find('a')
        d = c['href']
        d = 'https://novatech.co.uk' + d.strip('\ ').strip("'")[:-1]
        sites.append(d)

    brands = []
    for a in soup.find_all(class_="search-box-liner search-box-results search-hover"):
        b = a.find('h2').text
        c = b.split(' ')[0]
        if c == '*B-stock':
            if not b.split(' ')[3].isdigit():
                c = b.split(' ')[3].strip('warranty*')
            else:
                c = b.split(' ')[5].strip('warranty*')
        elif c[0].isdigit():
            c = 'intel'
        d = ""
        if '.' in c:
            for i in c:
                if i.isalpha():
                    d = "".join([d, i])
            c = d
        brands.append(c.lower().strip('.').title())

    typ = None
    try:
        name1 = names[0].lower()
        if 'graphics card' in name1:
            typ = 'graphics card'
        elif 'processor' in name1 or 'cpu' in name1:
            typ = 'processor'
        elif 'memory' in name1 or 'ram' in name1 or 'ddr' in name1:
            typ = 'memory'
    except IndexError:
        pass

    dic['name'] = names
    if not names:
        repeat = False

    dic['price'] = prices
    dic['site'] = sites
    dic['brand'] = brands
    dic['type'] = typ

    return dic, repeat

if __name__ == '__main__':
    # graphics
    pageno = 1
    repeat = True
    test = []
    while repeat:
        try:
            url = f'https://www.novatech.co.uk/products/components/memory-pc/ddr4memory/?pg={pageno}'
            dic, repeat = tool(url)
            if not dic['name'] == []:
                test.append(dic)
                print(dic)
            pageno += 1
        except:
            print("page not found")
            repeat = False
    print(test)


from django import forms
from django.utils.translation import gettext_lazy as _

from .models import Comment, Post

class PostForm(forms.ModelForm):

    class Meta:
        model = Post
        fields = ('text', 'group', 'image')
        labels = {
            'text': _('Текст поста:'),
            'group': _('Выберите группу:'),
            'image': _('Загрузить картинку:'),
        }

class CommentForm(forms.ModelForm):

    class Meta:
        model = Comment
        fields = ('text',)
        labels = {
            'text': _('Комментировать:'),
        }


import argparse
import requests
from bs4 import BeautifulSoup
import pandas as pd
from collections import Counter
import seaborn as sns
import matplotlib.pyplot as plt
import json
import os
import data

def parse_args():

    parser = argparse.ArgumentParser(description = "you should add those parameter")

    parser.add_argument("-source", choices=["local", "remote","test"], nargs=1, help="where data should be gotten from")

    # parser.add_argument('--remote', type=str, help='remote')
    # parser.add_argument('--local', type=str, help='local')
    # parser.add_argument('--test', type=str, help='test')`
    args = parser.parse_args()
    return args

'''get latitude and longitude'''
def lat_long():
    r = requests.get("https://www.latlong.net/category/cities-235-15.html")
    soup = BeautifulSoup(r.content, 'lxml')
    main_table = soup.findAll('div', {"class": "col-8"})[0]
    main_body = main_table.find('table')

    city_name = []
    latitude = []
    longitude = []

    for place in main_body.findAll('tr'):
        if (len(place.findAll('td')) > 0):
            place_name = place.findAll('td')[0].text
            place_latitude = place.findAll('td')[1].text
            place_longitude = place.findAll('td')[2].text
            #print(place_name, place_latitude, place_longitude)

            city_name.append(place_name)
            latitude.append(place_latitude)
            longitude.append(place_longitude)

    latitude_longitude = pd.DataFrame({'city name': city_name, 'latitude': latitude, 'longitude': longitude})
    latitude_longitude.to_csv('latitude_longitude.csv', index=False)
    return latitude_longitude

'''get GDHI of 9 regions'''
def get_GDHI():
    r = requests.get(
        "https://www.ons.gov.uk/economy/regionalaccounts/grossdisposablehouseholdincome/bulletins/regionalgrossdisposablehouseholdincomegdhi/1997to2016")
    soup = BeautifulSoup(r.content, 'lxml')
    main_table = soup.findAll('div', {"id": "the-fastest-growing-nuts1-region-per-head-is-the-east-of-england"})[0]

    # main_body = main_table.find('table',{'class':'t1'})
    # main_body = main_body.find('tbody')
    main_body = main_table

    name = []
    pop = []
    GDHI = []

    for place in main_body.findAll('tr', {'class': "r-6c019d9b-8d71-42da-8b23-fef011a0e2d7-3"}):
        if (len(place.findAll('td')) > 4) and len(place.findAll('td')[0].text) > 2:
            place_name = place.findAll('td')[0].text
            place_population = place.findAll('td')[2].text
            place_GDHI = place.findAll('td')[5].text
            place_GDHI = place_GDHI.replace(',', '')
            name.append(place_name)
            pop.append(place_population)
            GDHI.append(place_GDHI)
            # print(place_name,'  pop:',place_population,'  GDHI:',place_GDHI)

    for place in main_body.findAll('tr', {'class': "r-6c019d9b-8d71-42da-8b23-fef011a0e2d7-4"}):
        if (len(place.findAll('td')) > 4 and len(place.findAll('td')[0].text) > 2):
            place_name = place.findAll('td')[0].text
            place_name = place.findAll('td')[0].text
            place_population = place.findAll('td')[2].text
            place_GDHI = place.findAll('td')[5].text
            place_GDHI = place_GDHI.replace(',', '')
            name.append(place_name)
            pop.append(place_population)
            GDHI.append(place_GDHI)
            # print(place_name,'  pop:',place_population,'  GDHI:',place_GDHI)

    pop_GDHI_df = pd.DataFrame({'region name': name, 'GDHI': GDHI})
    pop_GDHI_df = pop_GDHI_df.drop([0, 9, 11, 12], axis=0)

    pop_GDHI_df.to_csv('pop_GDHI.csv', index=False)
    return pop_GDHI_df

'''get all crime numbers of every city in 2017 in UK.
   This step will take more than half an hour, it is not recommended to try during testing '''

def get_crime():
    latitude_longitude = lat_long()
    pop_GDHI_df = get_GDHI()
    first_city = []

    for i in range(0, latitude_longitude.shape[0]):
        all_city_name = latitude_longitude['city name'][i]
        single_name = all_city_name.split(',')[0]
        first_city.append(single_name)
    latitude_longitude.insert(0, 'single city name', first_city)

    city_dict = {'North East': 'Newcastle upon Tyne', 'North West': 'Manchester', 'Yorkshire and The Humber': 'Leeds', \
                 'East Midlands': 'Leicester', 'West Midlands': 'Birmingham', \
                 'East of England': 'Cambridge', 'South East': 'Winchester', 'South West': 'Bristol', \
                 'Northern Ireland': 'Belfast', 'London': 'London', 'Wales': 'St. Davids', 'Scotland': 'Glasgow'}

    data_list = ['2017-01', '2017-02', '2017-03', '2017-04', '2017-05', '2017-06', '2017-07', '2017-08', \
                 '2017-09', '2017-10', '2017-11', '2017-12']

    crime_type = []
    region = []
    data = []

    for i in range(0, pop_GDHI_df.shape[0]):

        print(f'load data of {i+1}th city')
        biggest_city = (pop_GDHI_df.iloc[i])['region name']
        target_city = city_dict[biggest_city]
        city_address = latitude_longitude.loc[latitude_longitude['single city name'] == target_city]
        latitude = float(city_address['latitude'].values)
        longitude = float(city_address['longitude'].values)

        #print(f'{i}th time')

        for b in data_list:
            print(f'load data of {b}')
            # print(latitude,longitude,b)
            url = f"https://data.police.uk/api/crimes-street/all-crime?lat={latitude}&lng={longitude}&date={b}"

            # r = requests.get(url)
            # j = json.loads(r.content)
            # for c in j:
            #     crime_type.append(c['category'])
            #     region.append(biggest_city)
            #     data.append(c['month'])

            try:
                r = requests.get(url)
                j = json.loads(r.content)
                for c in j:
                    crime_type.append(c['category'])
                    region.append(biggest_city)
                    data.append(c['month'])
            except:
                try:
                    r = requests.get(url)
                    j = json.loads(r.content)
                    for c in j:
                        crime_type.append(c['category'])
                        region.append(biggest_city)
                        data.append(c['month'])
                except:
                    try:
                        r = requests.get(url)
                        j = json.loads(r.content)
                        for c in j:
                            crime_type.append(c['category'])
                            region.append(biggest_city)
                            data.append(c['month'])
                    except:
                        print('connot find this content')
                        pass
                    continue

    instance_df = pd.DataFrame({'crime type': crime_type, 'region': region, 'data': data})
    instance_df.to_csv('instance.csv', index=False)
    return instance_df

def get_part_of_crime():
    latitude_longitude = lat_long()
    pop_GDHI_df = get_GDHI()
    first_city = []

    for i in range(0, latitude_longitude.shape[0]):
        all_city_name = latitude_longitude['city name'][i]
        single_name = all_city_name.split(',')[0]
        first_city.append(single_name)
    latitude_longitude.insert(0, 'single city name', first_city)

    city_dict = {'North East': 'Newcastle upon Tyne', 'North West': 'Manchester', 'Yorkshire and The Humber': 'Leeds', \
                 'East Midlands': 'Leicester', 'West Midlands': 'Birmingham', \
                 'East of England': 'Cambridge', 'South East': 'Winchester', 'South West': 'Bristol', \
                 'Northern Ireland': 'Belfast', 'London': 'London', 'Wales': 'St. Davids', 'Scotland': 'Glasgow'}

    data_list = ['2017-01', '2017-02']

    crime_type = []
    region = []
    data = []

    for i in range(0, pop_GDHI_df.shape[0]):

        # print(f'load data of {i+1}th city')
        biggest_city = (pop_GDHI_df.iloc[i])['region name']
        target_city = city_dict[biggest_city]
        city_address = latitude_longitude.loc[latitude_longitude['single city name'] == target_city]
        latitude = float(city_address['latitude'].values)
        longitude = float(city_address['longitude'].values)

        #print(f'{i}th time')

        for b in data_list:
            # print(f'load data of {b}')
            # print(latitude,longitude,b)
            url = f"https://data.police.uk/api/crimes-street/all-crime?lat={latitude}&lng={longitude}&date={b}"

            # r = requests.get(url)
            # j = json.loads(r.content)
            # for c in j:
            #     crime_type.append(c['category'])
            #     region.append(biggest_city)
            #     data.append(c['month'])

            try:
                r = requests.get(url)
                j = json.loads(r.content)
                for c in j:
                    crime_type.append(c['category'])
                    region.append(biggest_city)
                    data.append(c['month'])
            except:
                try:
                    r = requests.get(url)
                    j = json.loads(r.content)
                    for c in j:
                        crime_type.append(c['category'])
                        region.append(biggest_city)
                        data.append(c['month'])
                except:
                    try:
                        r = requests.get(url)
                        j = json.loads(r.content)
                        for c in j:
                            crime_type.append(c['category'])
                            region.append(biggest_city)
                            data.append(c['month'])
                    except:
                        print('connot find this content')
                        pass
                    continue

    instance_df = pd.DataFrame({'crime type': crime_type, 'region': region, 'data': data})
    instance_df.to_csv('instance.csv', index=False)
    return instance_df

def data_process(latitude_longitude,pop_GDHI_df,instance_df):
    city_pop = {}
    city_dict = {'North East': 'Newcastle upon Tyne', 'North West': 'Manchester', 'Yorkshire and The Humber': 'Leeds', \
                 'East Midlands': 'Leicester', 'West Midlands': 'Birmingham', \
                 'East of England': 'Cambridge', 'South East': 'Winchester', 'South West': 'Bristol', \
                 'Northern Ireland': 'Belfast', 'London': 'London', 'Wales': 'St. Davids', 'Scotland': 'Glasgow'}

    r = requests.get("https://en.wikipedia.org/wiki/List_of_English_districts_by_population")
    soup = BeautifulSoup(r.content,'lxml')
    main_body = soup.findAll('table', {"class": "wikitable"})[0]
    # main_body = main_body.find('table',{"class":"wikitable"})
    # main_body = main_body.find('tbody')

    for place in main_body.findAll('tr'):
        if (len(place.findAll('td')) > 2) and (isinstance(place.findAll('td')[2].text, int) != True):
            city = place.findAll('td')[1].text
            pop = place.findAll('td')[2].text
            pop = int(pop.replace(',', ''))
            city_pop[city] = pop

    city_pop['London'] = 8900000
    city_pop['Birmingham'] = 210700

    from collections import Counter
    total_crime_number = []
    population = []
    crime_rate = []
    all_crime_number = Counter(instance_df['region'])

    for i in all_crime_number:
        total_crime_number.append(all_crime_number[i])

    pop_GDHI_df.insert(1, 'total crime number', total_crime_number)

    for i in range(0, pop_GDHI_df.shape[0]):
        pop = city_pop[city_dict[pop_GDHI_df['region name'][i]]]
        population.append(pop)

    pop_GDHI_df.insert(2, 'population', population)

    for i in range(0, pop_GDHI_df.shape[0]):
        rate = int(pop_GDHI_df['total crime number'][i]) / int(pop_GDHI_df['population'][i])
        crime_rate.append(rate)

    pop_GDHI_df.insert(3, 'crime rate', crime_rate)

    all_crime = Counter(instance_df['crime type'])
    crime_list = []

    for i in all_crime:
        crime_list.append(i)

    crime_dict = {
        'anti-social-behaviour': [],
        'bicycle-theft': [],
        'burglary': [],
        'criminal-damage-arson': [],
        'drugs': [],
        'other-theft': [],
        'possession-of-weapons': [],
        'public-order': [],
        'robbery': [],
        'shoplifting': [],
        'theft-from-the-person': [],
        'vehicle-crime': [],
        'violent-crime': [],
        'other-crime': [],
    }

    for i in crime_dict:
        for b in range(0, pop_GDHI_df.shape[0]):
            consequence = Counter((instance_df.loc[instance_df['region'] == pop_GDHI_df['region name'][b]])['crime type'])
            if consequence[i] > 0:
                crime_dict[i].append(consequence[i] / pop_GDHI_df['population'][b])
            else:
                crime_dict[i].append('None')

    for i in crime_dict:
        pop_GDHI_df.insert(1, f'{i}', crime_dict[i])

    plt.plot(pop_GDHI_df['region name'], pop_GDHI_df['crime rate'], 'red', marker='*', label='crime rate')
    plt.xticks(rotation=45)
    plt.xlabel("region name")
    plt.ylabel("crime rate")
    plt.title("crime rate in 9 regions")
    plt.show()

    sns.regplot(data=pop_GDHI_df, x=pop_GDHI_df['GDHI'], y=pop_GDHI_df['crime rate'], line_kws={'color': 'red'})
    plt.xlabel("GDHI (£)")
    plt.ylabel(f"{i} rate (%)")
    plt.title("crime rate and GDHI")
    plt.show()

    for i in crime_list:
        sns.regplot(data=pop_GDHI_df, x=pop_GDHI_df['GDHI'], y=pop_GDHI_df[i], line_kws={'color': 'red'})
        plt.xlabel("GDHI (£)")
        plt.ylabel(f"{i} rate (%)")
        plt.title(f"{i} rate and GDHI ")
        plt.show()

module_path = os.path.dirname(__file__)
filename = module_path[:-3]

def get_local_latitude_longitude():
    latitude_longitude = pd.read_csv('data/latitude_longitude.csv')
    return latitude_longitude

def get_local_pop_GDHI_df():
    pop_GDHI_df = pd.read_csv('data/pop_GDHI.csv')
    return pop_GDHI_df

def get_local_instance_df():
    instance_df = pd.read_csv('data/instance.csv')
    return instance_df

if __name__ == '__main__':
    args = parse_args()
    location = args.source
    # if args.remote:
    if location==['remote']:
        print("Remote Processing")
        print('\n')
        print("get latitude and longitude of all cities")
        latitude_longitude = lat_long()

        print('\n')
        print(' get Gross disposable household income(GDHI) of every city ')
        pop_GDHI_df = get_GDHI()

        print('\n')
        print(' get total crime #')
        instance_df = get_crime()

        print('\n')
        print(' finished ')

    #if args.test:
    if location==['test']:
        print("Remote Processing")
        print('\n')
        print("get latitude and longitude of all cities")
        latitude_longitude = lat_long()

        print('\n')
        print(' get Gross disposable household income(GDHI) of every city ')
        pop_GDHI_df = get_GDHI()

        print('\n')
        print(' get total crime')
        instance_df = get_part_of_crime()

        print('\n')
        print(' finished ')

    # elif args.local:
    if location==['local']:
        print(" Remote Processing ")
        print('\n')
        print(" get latitude and longitude of all cities ")
        latitude_longitude = get_local_latitude_longitude()

        print('\n')
        print(' get Gross disposable household income(GDHI) of every city ')
        pop_GDHI_df = get_local_pop_GDHI_df()

        print('\n')
        print(' get total crime ')
        instance_df = get_local_instance_df()
        data_process(latitude_longitude, pop_GDHI_df, instance_df)

    else:
        print("unavailable Parameter")


# LC 621: Task Scheduler
# https://leetcode.com/problems/task-scheduler/
# Resources Used: https://www.geeksforgeeks.org/python-counter-objects-elements/#
# Yzer De Gula

from typing import List
from collections import deque, Counter
import heapq

"""
U-nderstand:
    # ? What is our input ?
        Our input is a characters array named tasks and an integer named n.
    # ? What is our output ?
        Our output is an integer representing the least amount of time needed
    to finish all the given tasks.
    # ? What does n represent ?
        n represents the cooldown period between two same tasks.
    # ? What is the cooldown period ?
        The cooldown period is the time needed to wait before executing the
    same task again.
    # ? Can our tasks be executed in any order ?
        Yes, our tasks can be executed in any order.
    # ? Can our input array be empty ?
        No, our input array cannot be empty.
    # ? Can n be negative ?
        No, n cannot be negative, it ranges from 0 to 100.
    # ? Can n be greater than the length of our input array ?
        Yes, n can be greater than the length of our input array.
    # ? Can our input array have anything other than uppercase letters ?
        No, our input array can only have uppercase letters.

M-atch:
        For this problem, we can utilize a Max Heap to keep track of the most
    frequent tasks. We can also use a dictionary to keep track of the number
    of times a task has been executed. We can do this simply with the Counter
    class from the collections module. We can also use a variable to keep
    track of the time needed to finish all the tasks. Additionally, we can
    use a double ended queue to keep track of the tasks that are ready to be
    executed. We can use a variable to keep track of the cooldown period.
    While there is still something in our max heap or q, we increment our
    time variable. Then if in the max heap there's a non-zero value, we pop
    it and increment it by 1. If the task count is not zero, we append it
    to the queue as a tuple of the task count and the time plus the cooldown
    period. If our queue is nonempty and the first value of our queue at
    index 0 is equal to the current time, we pop it and push it to the max
    heap. We return the time variable.

P-lan:
    1) Create a Hash Map of our tasks list using Counter()
    2) Using List Comprehension get the negative values of the Hash Maps values
    3) Heapify that list
    4) Declare our time variable and deque()
    5) While there is something in our Max Heap or queue
        i) Increment our time counter
        ii) Check if there is a non-zero value in our Max Heap
            a) If so, set the task count to 1 plus the popped value from the Max Heap
            b) If the task count is non-zero
                I) Append the tuple of the task count and the time counter plus the
                   cooldown time to the queue
        iii) If the queue is non-empty and the first value of our queue at index 0
             is equal to the current time
             a) We push onto the Max Heap the front of the queue
    6) Return time

I-mplement:
"""

class Solution:
    def leastInterval(self, tasks: List[str], n: int) -> int:
        # Create our Hash Map using Counter
        task_count = Counter(tasks)

        # Using List Comprehension negate the Hash Maps values
        max_heap = [-count for count in task_count.values()]
        #print(max_heap)

        # Heapify the list
        heapq.heapify(max_heap)

        # Declare our variables
        time = 0
        queue = deque()

        # While there is something in our Max Heap or queue
        while max_heap or queue:
            # Increment time
            time += 1
            # If there is a non-zero value in our Max Heap
            if max_heap:
                count = 1 + heapq.heappop(max_heap)
                # If the count is non-zero
                if count:
                    # Append the tuple [count, time + n] to the queue
                    queue.append([count, time + n])
            # If there is something in our queue and the first value at index 0 is equal to time
            if queue and queue[0][1] == time:
                # This means that the calculated cooldown time for this task is ready
                # So we pop the queue and add it back to the Max Heap
                heapq.heappush(max_heap, queue.popleft()[0])

        # Return time
        return time

    def display(self, tasks: List[str], n: int) -> int:
        print(f"Input: tasks = {tasks}, n = {n}")
        print(f"Output: {self.leastInterval(tasks, n)}\n")

# Testing
test = Solution()

# Expected: 8
tasks, n = ["A","A","A","B","B","B"], 2
test.display(tasks, n)

# Expects 6
tasks, n = ["A","A","A","B","B","B"], 0
test.display(tasks, n)

# Expects 16
tasks, n = ["A","A","A","A","A","A","B","C","D","E","F","G"], 2
test.display(tasks, n)


#!/usr/bin/python3
"""Python script that takes in a letter and sends a POST request to
http://0.0.0.0:5000/search_user with the letter as a parameter """
from sys import argv
import requests

if __name__ == "__main__":

    ur = "http://0.0.0.0:5000/search_user"
    if len(argv) < 2:
        value = ""
    else:
        value = argv[1]
    res = requests.post(ur, data={'q': value})
    try:
        data = res.json()
        if data:
            print(f"[{data['id']}] {data['name']}")
        else:
            print("No result")
    except ValueError:
        print("Not a valid JSON")


#!/usr/bin/python2

import zmq
import os

def server():
    '''
    Setup a VERY simple file server
    '''
    # Set up the zeromq context and REP socket
    context = zmq.Context(1)
    sock = context.socket(zmq.REP)
    sock.bind('tcp://*:4545')

    # Start the server loop
    while True:
        # Recieve the location of the file to serve
        msg = sock.recv()
        # Verify that the file is available
        if not os.path.isfile(msg):
            sock.send('')
            continue
        # Open the file for reading
        fn = open(msg, 'rb')
        stream = True
        # Start reading in the file
        while stream:
            # Read the file bit by bit
            stream = fn.read(128)
            if stream:
                # If the stream has more to send then send more
                sock.send(stream, zmq.SNDMORE)
            else:
                # Finish it off
                sock.send(stream)

if __name__ == '__main__':
    server()


import torch
import numpy as np
from os.path import isdir, join
from fnmatch import fnmatch, filter

def soft_update(source, target, tau):
    for target_param, param in zip(target.parameters(), source.parameters()):
        target_param.data.copy_(target_param.data * (1.0 - tau) + param.data * tau)

def clipped_gd(x, dx, eta, gamma):

    norm = torch.norm(dx, dim=-1, keepdim=True)
    eta = torch.clamp_max(gamma * eta / norm, eta)

    return x - eta * dx

# def soft_update(net, target, tau):
#
#     state = net.state_dict()
#
#     with torch.no_grad():
#         for n, p in target.named_parameters():
#             p.data.mul_(1 - tau).add_(tau, state[n].data)

# from https://github.com/songrotek/DDPG/blob/master/ou_noise.py
# and adapted to be synchronous with https://github.com/openai/baselines/blob/master/baselines/ddpg/noise.py
class OUNoise:
    def __init__(self, dt=0.01, mu=0, theta=0.15, sigma=0.2):
        self.dt = dt
        self.mu = mu
        self.theta = theta
        self.sigma = sigma
        self.reset()

    def reset(self):
        self.state = np.ones_like(self.action_dimension) * self.mu

    def noise(self):
        x = self.state
        dx = self.theta * (self.mu - x) * self.dt + self.sigma * np.random.randn(len(x)) * np.sqrt(self.dt)
        self.state = x + dx
        return self.state

    def __call__(self, *args, **kwargs):
        return self.noise(*args, **kwargs)

# From OpenAI Baselines:
# https://github.com/openai/baselines/blob/master/baselines/ddpg/noise.py
class OrnsteinUhlenbeckActionNoise:
    def __init__(self, mu, sigma, theta=.15, dt=1e-2, x0=None):
        self.theta = theta
        self.mu = mu
        self.sigma = sigma
        self.dt = torch.tensor(dt).float()
        self.x0 = x0
        self.reset()

    def noise(self):
        x = self.x_prev + self.theta * (self.mu - self.x_prev) * self.dt \
            + self.sigma * torch.sqrt(self.dt) * torch.zeros_like(self.mu).normal_()
        self.x_prev = x
        return x

    def __call__(self, *args, **kwargs):
        return self.noise(*args, **kwargs)

    def reset(self):
        self.x_prev = self.x0 if self.x0 is not None else torch.zeros_like(self.mu)

    def __repr__(self):
        return 'OrnsteinUhlenbeckActionNoise(mu={}, sigma={})'.format(self.mu, self.sigma)

class RandomNoise:
    def __init__(self, mu, sigma):
        self.mu = mu
        self.sigma = sigma
        self.reset()

    def noise(self):
        x = self.mu + self.sigma * torch.zeros_like(self.mu).normal_()
        return x

    def __call__(self, *args, **kwargs):
        return self.noise(*args, **kwargs)

    def reset(self):
        pass

# From OpenAI Baselines:
# https://github.com/openai/baselines/blob/master/baselines/ddpg/noise.py
class AdaptiveParamNoiseSpec(object):
    def __init__(self, initial_stddev=0.1, desired_action_stddev=0.2, adaptation_coefficient=1.01):
        """
        Note that initial_stddev and current_stddev refer to std of parameter noise,
        but desired_action_stddev refers to (as name notes) desired std in action space
        """
        self.initial_stddev = initial_stddev
        self.desired_action_stddev = desired_action_stddev
        self.adaptation_coefficient = adaptation_coefficient

        self.current_stddev = initial_stddev

    def adapt(self, distance):
        if distance > self.desired_action_stddev:
            # Decrease stddev.
            self.current_stddev /= self.adaptation_coefficient
        else:
            # Increase stddev.
            self.current_stddev *= self.adaptation_coefficient

    def get_stats(self):
        stats = {
            'param_noise_stddev': self.current_stddev,
        }
        return stats

    def __repr__(self):
        fmt = 'AdaptiveParamNoiseSpec(initial_stddev={}, desired_action_stddev={}, adaptation_coefficient={})'
        return fmt.format(self.initial_stddev, self.desired_action_stddev, self.adaptation_coefficient)

def gret_grad_norm(net, norm_type=2):

    return torch.norm(torch.stack([torch.norm(p.grad.detach(), norm_type) for p in net.parameters()]), norm_type)

class RunningStat(object):
    '''
    Keeps track of first and second moments (mean and variance)
    of a streaming time series.
     Taken from https://github.com/joschu/modular_rl
     Math in http://www.johndcook.com/blog/standard_deviation/
    '''
    def __init__(self, shape):
        self._n = 0
        self._M = np.zeros(shape)
        self._S = np.zeros(shape)

    # def push(self, x):
    #     x = np.asarray(x)
    #     assert x.shape == self._M.shape
    #     self._n += 1
    #     if self._n == 1:
    #         self._M[...] = x
    #     else:
    #         oldM = self._M.copy()
    #         self._M[...] = oldM + (x - oldM) / self._n
    #         self._S[...] = self._S + (x - oldM) * (x - self._M)

    def push(self, x):
        x = np.asarray(x)
        assert x.shape == self._M.shape
        self._n += 1
        if self._n == 1:
            self._M[...] = x
        else:
            oldM = self._M.copy()
            self._M[...] = oldM + (x - oldM) / self._n
            self._S[...] = self._S + (x - oldM) * (x - self._M)

    @property
    def n(self):
        return self._n
    @property
    def mean(self):
        return self._M
    @property
    def var(self):
        return self._S / (self._n - 1) if self._n > 1 else np.square(self._M)
    @property
    def std(self):
        return np.sqrt(self.var)
    @property
    def shape(self):
        return self._M.shape

class RewardFilter:
    """
    Incorrect reward normalization [copied from OAI code]
    update return
    divide reward by std(return) without subtracting and adding back mean
    """

    def __init__(self, prev_filter, shape, gamma, clip=None):
        assert shape is not None
        self.gamma = gamma
        self.prev_filter = prev_filter
        self.rs = RunningStat(shape)
        self.ret = np.zeros(shape)
        self.clip = clip

    def __call__(self, x, **kwargs):
        x = self.prev_filter(x, **kwargs)
        self.ret = self.ret * self.gamma + x
        self.rs.push(self.ret)
        # x = x / (self.rs.std + 1e-8)

        x = (x - self.rs.mean) / (self.rs.std + 1e-8)

        if self.clip > 0:
            x = np.clip(x, -self.clip, self.clip)
        return x

    def reset(self):
        self.ret = np.zeros_like(self.ret)
        self.prev_filter.reset()

class ZFilter:
    """
    y = (x-mean)/std
    using running estimates of mean,std
    """
    def __init__(self, prev_filter, shape, center=True, scale=True, clip=None):
        assert shape is not None
        self.center = center
        self.scale = scale
        self.clip = clip
        self.rs = RunningStat(shape)
        self.prev_filter = prev_filter

    def __call__(self, x, **kwargs):
        x = self.prev_filter(x, **kwargs)
        self.rs.push(x)
        if self.center:
            x = x - self.rs.mean
        if self.scale:
            if self.center:
                x = x / (self.rs.std + 1e-8)
            else:
                diff = x - self.rs.mean
                diff = diff/(self.rs.std + 1e-8)
                x = diff + self.rs.mean
        if self.clip > 0:
            x = np.clip(x, -self.clip, self.clip)
        return x

    def reset(self):
        self.prev_filter.reset()

class Identity:
    '''
    A convenience class which simply implements __call__
    as the identity function
    '''
    def __call__(self, x, *args, **kwargs):
        return x

    def reset(self):
        pass

# def generalized_advantage_estimation(r, t, e, v1, v2, gamma, lambda_gae, norm=False):
#
#     device = r.device
#
#     rewards = []
#     discounted_reward = 0
#     for reward, is_terminal in zip(r.flip(0), reversed(e.flip(0))):
#         if is_terminal:
#             discounted_reward = 0
#         discounted_reward = reward + (gamma * discounted_reward)
#         rewards.insert(0, discounted_reward)
#
#     # Normalizing the rewards:
#     rewards = torch.tensor(rewards).to(device)
#     rewards = (rewards - rewards.mean()) / (rewards.std() + 1e-5)
#
#     advantages = rewards - v1.detach()
#
#     a = advantages
#     v = rewards
#
#     return a, v

mu = 0
std = 1
lr = 0.1

def norm_r(r, e, gamma):

    global mu
    global std

    mu_target = r.mean()
    std_target = (r.std() + 1e-3) * max(1 - gamma, 1e-3, float(e.sum() / len(e))) * 10

    mu = (1 - lr) * mu + lr * mu_target
    std = (1 - lr) * std + lr * std_target

    return (r - mu) / std

def generalized_advantage_estimation(r, t, e, v1, v2, gamma, lambda_gae, norm=False):

    device = r.device
    b = torch.FloatTensor([1, 0]).to(device)
    aa = torch.FloatTensor([1, -gamma * lambda_gae]).to(device)
    av = torch.FloatTensor([1, -gamma]).to(device)

    if norm:
        r = norm_r(r, e, gamma)

    i = torch.nonzero(e).flatten()
    if len(i):
        if (i[-1] + 1) == len(e):
            i = torch.cat([torch.LongTensor([0]).to(device), i+1])
        else:
            i = torch.cat([torch.LongTensor([0]).to(device), i + 1, torch.LongTensor([len(e)]).to(device)])

        i = list(i[1:] - i[:-1])

        r = torch.split(r, i)
        t = torch.split(t, i)
        v1 = torch.split(v1, i)
        v2 = torch.split(v2, i)
    else:
        r = [r]
        t = [t]
        v1 = [v1]
        v2 = [v2]

    a = []
    v = []

    for ri, ti, v1i, v2i in zip(r, t, v1, v2):

        delta = -v1i + ri + gamma * (1 - ti) * v2i

        a.append(lfilter(delta.flip(0), aa, b).flip(0))
        v.append(lfilter(ri.flip(0), av, b).flip(0))

    a = torch.cat(a)
    v = torch.cat(v)

    return a, v

def lfilter(waveform, a_coeffs, b_coeffs):
    r"""Perform an IIR filter by evaluating difference equation.

    Args:
        waveform (Tensor): audio waveform of dimension of `(..., time)`.  Must be normalized to -1 to 1.
        a_coeffs (Tensor): denominator coefficients of difference equation of dimension of `(n_order + 1)`.
                                Lower delays coefficients are first, e.g. `[a0, a1, a2, ...]`.
                                Must be same size as b_coeffs (pad with 0's as necessary).
        b_coeffs (Tensor): numerator coefficients of difference equation of dimension of `(n_order + 1)`.
                                 Lower delays coefficients are first, e.g. `[b0, b1, b2, ...]`.
                                 Must be same size as a_coeffs (pad with 0's as necessary).

    Returns:
        Tensor: Waveform with dimension of `(..., time)`.  Output will be clipped to -1 to 1.
    """
    # pack batch
    shape = waveform.size()
    waveform = waveform.view(-1, shape[-1])

    assert (a_coeffs.size(0) == b_coeffs.size(0))
    assert (len(waveform.size()) == 2)
    assert (waveform.device == a_coeffs.device)
    assert (b_coeffs.device == a_coeffs.device)

    device = waveform.device
    dtype = waveform.dtype
    n_channel, n_sample = waveform.size()
    n_order = a_coeffs.size(0)
    n_sample_padded = n_sample + n_order - 1
    assert (n_order > 0)

    # Pad the input and create output
    padded_waveform = torch.zeros(n_channel, n_sample_padded, dtype=dtype, device=device)
    padded_waveform[:, (n_order - 1):] = waveform
    padded_output_waveform = torch.zeros(n_channel, n_sample_padded, dtype=dtype, device=device)

    # Set up the coefficients matrix
    # Flip coefficients' order
    a_coeffs_flipped = a_coeffs.flip(0)
    b_coeffs_flipped = b_coeffs.flip(0)

    # calculate windowed_input_signal in parallel
    # create indices of original with shape (n_channel, n_order, n_sample)
    window_idxs = torch.arange(n_sample, device=device).unsqueeze(0) + torch.arange(n_order, device=device).unsqueeze(1)
    window_idxs = window_idxs.repeat(n_channel, 1, 1)
    window_idxs += (torch.arange(n_channel, device=device).unsqueeze(-1).unsqueeze(-1) * n_sample_padded)
    window_idxs = window_idxs.long()
    # (n_order, ) matmul (n_channel, n_order, n_sample) -> (n_channel, n_sample)
    input_signal_windows = torch.matmul(b_coeffs_flipped, torch.take(padded_waveform, window_idxs))

    for i_sample, o0 in enumerate(input_signal_windows.t()):
        windowed_output_signal = padded_output_waveform[:, i_sample:(i_sample + n_order)]
        o0.sub_(torch.mv(windowed_output_signal, a_coeffs_flipped))
        o0.div_(a_coeffs[0])

        padded_output_waveform[:, i_sample + n_order - 1] = o0

    # output = torch.clamp(padded_output_waveform[:, (n_order - 1):], min=-1., max=1.)
    output = padded_output_waveform[:, (n_order - 1):]

    # unpack batch
    output = output.view(shape[:-1] + output.shape[-1:])

    return output

def iter_dict(d):
    for i in range(len(d[list(d.keys())[0]])):
        yield {k: v[i] for k, v in d.items()}

def include_patterns(*patterns):
    """Factory function that can be used with copytree() ignore parameter.

    Arguments define a sequence of glob-style patterns
    that are used to specify what files to NOT ignore.
    Creates and returns a function that determines this for each directory
    in the file hierarchy rooted at the source directory when used with
    shutil.copytree().
    """
    def _ignore_patterns(path, names):
        keep = set(name for pattern in patterns
                            for name in filter(names, pattern))
        ignore = set(name for name in names
                        if name not in keep and not isdir(join(path, name)))
        return ignore
    return _ignore_patterns


import sys
import os
from os.path import join as opj
import argparse

BASEPATH = "F://NuCLS"
sys.path.insert(0, BASEPATH)

parser = argparse.ArgumentParser(description='Train nucleus model.')
parser.add_argument('-f', type=int, default=[1], nargs='+', help='fold(s) to run')
parser.add_argument('-g', type=int, default=[0], nargs='+', help='gpu(s) to use')
parser.add_argument('--qcd', type=int, default=1, help='use QCd data for training?')
parser.add_argument('--train', type=int, default=1, help='train?')
parser.add_argument('--vistest', type=int, default=1, help='visualize results on testing?')
args = parser.parse_args()
args.qcd = bool(args.qcd)
args.train = bool(args.train)
args.vistest = bool(args.vistest)

# GPU allocation MUST happen before importing other modules
from GeneralUtils import save_configs, maybe_mkdir, AllocateGPU
AllocateGPU(GPUs_to_use=args.g)

from nucleus_model.MiscUtils import load_saved_otherwise_default_model_configs
# from configs.nucleus_model_configs import CoreSetQC, CoreSetNoQC
from nucleus_model.NucleusWorkflows import run_one_maskrcnn_fold

class CoreSetQC(object):
    """paths and configs related to the QCd core set."""

    # dname = 'v4_2020-04-05_FINAL_CORE'
    dname = 'FINAL_CORE'
    dataset_root = opj(BASEPATH, f'data/tcga-nucleus/{dname}/CORE_SET/QC/')
    dbpath = opj(BASEPATH, f'data/tcga-nucleus/{dname}/{dname}.sqlite')
    dataset_name = dname + '_QC'

    train_test_splits_path = opj(dataset_root, 'train_test_splits/')

    # # split to training and testing
    # save_cv_train_test_splits(
    #     root=dataset_root, savepath=train_test_splits_path,
    #     n_folds=5, by_hospital=True)

class CoreSetNoQC(object):
    """paths and configs related to the Non-QCd core set."""

    dname = CoreSetQC.dname
    dataset_root = opj(BASEPATH, f'data/tcga-nucleus/{dname}/CORE_SET/noQC/')
    dbpath = opj(BASEPATH, f'data/tcga-nucleus/{dname}/{dname}.sqlite')
    dataset_name = dname + '_noQC'

# %%===========================================================================
# Configs

model_name = '002_MaskRCNN_tmp'
dataset_name = CoreSetQC.dataset_name if args.qcd else CoreSetNoQC.dataset_name
all_models_root = opj(BASEPATH, f'results/tcga-nucleus/models/{dataset_name}/')
model_root = opj(all_models_root, model_name)
maybe_mkdir(model_root)

# load configs
configs_path = opj(model_root, 'nucleus_model_configs.py')
cfg = load_saved_otherwise_default_model_configs(configs_path=configs_path)

# for reproducibility, copy configs & most relevant code file to results
if not os.path.exists(configs_path):
    save_configs(
        configs_path=opj(BASEPATH, 'configs/nucleus_model_configs.py'),
        results_path=model_root)
save_configs(
    configs_path=os.path.abspath(__file__),
    results_path=model_root, warn=False)
save_configs(
    configs_path=opj(BASEPATH, 'nucleus_model/NucleusWorkflows.py'),
    results_path=model_root, warn=False)

# %%===========================================================================
# Now run

for fold in args.f:
    run_one_maskrcnn_fold(
        fold=fold, cfg=cfg, model_root=model_root, model_name=model_name,
        qcd_training=args.qcd, train=args.train, vis_test=args.vistest)

# %%===========================================================================


from django.urls import path
from .views import *

urlpatterns = [
    path('', index, name='index'),
    path('about/', about, name='about'),
    path('services/', services, name='services'),
    path('faq/', faq, name='faq'),
    path('404/', error, name='404'),
    path('pricing/', pricing, name='pricing'),
    path('portfolio-3-col/', portfolio_3, name='portfolio-3-col'),
    path('portfolio-4-col/', portfolio_4, name='portfolio-4-col'),
    path('portfolio-item/', portfolio_item, name='portfolio-item'),
    path('blog/', blog, name='blog'),
    path('blog-post/', blog_post, name='blog-post'),
    path('contact/', contact, name='contact'),
    path('create_service/', create_service, name='create_service'),
    path('login/', login_user, name='login'),
    path('registration', registration, name='registration'),

]


from aiogram.types import InlineKeyboardMarkup, InlineKeyboardButton
from aiogram.utils.callback_data import CallbackData
import ast

product_cb = CallbackData('product', 'id', 'action')
additeve_cb = CallbackData('additeve', 'id', 'title', 'isactive')

def product_markup(idx, count, additevs:str = "", active_additevs=[]):

    global product_cb
    global additeve_cb

    markup = InlineKeyboardMarkup()
#    additives = ast.literal_eval(additevs)
    additevs_items = [] #list(map(list,additives.items()))
    for i in range(0, len(additevs_items)-1, 2):
        first_name, first_price = additevs_items[i]
        second_name, second_price = additevs_items[i+1]
        # print("active_additevs -",active_additevs)
        if first_name in active_additevs:
            first = InlineKeyboardButton(f"✅ {first_name} - {first_price} грн.", callback_data=additeve_cb.new(id=idx, title=first_name, isactive=1))
        else:
            first = InlineKeyboardButton(f"❌ {first_name} - {first_price} грн.", callback_data=additeve_cb.new(id=idx, title=first_name, isactive=0))

        if second_name in active_additevs:
            second = InlineKeyboardButton(f"✅ {second_name} - {second_price} грн.", callback_data=additeve_cb.new(id=idx, title=second_name, isactive=1))
        else:
            second = InlineKeyboardButton(f"❌ {second_name} - {second_price} грн.", callback_data=additeve_cb.new(id=idx, title=second_name, isactive=0))
        markup.row(first, second)
    if(len(additevs_items)%2==1):
        first_name, first_price = additevs_items[-1]
        if first_name in active_additevs:
            markup.row(InlineKeyboardButton(f"✅ {first_name} - {first_price} грн.", callback_data=additeve_cb.new(id=idx, title=first_name, isactive=1)))
        else:
            markup.row(InlineKeyboardButton(f"❌ {first_name} - {first_price} грн.", callback_data=additeve_cb.new(id=idx, title=first_name, isactive=0)))
    # [InlineKeyboardButton(f"{name} - {price} гривен", callback_data=additeve_cb.new(title=name, isactive=0)) for name, price in additevs.items]

    back_btn = InlineKeyboardButton('➖', callback_data=product_cb.new(id=idx, action='decrease'))
    count_btn = InlineKeyboardButton(count, callback_data=product_cb.new(id=idx, action='count'))
    next_btn = InlineKeyboardButton('➕', callback_data=product_cb.new(id=idx, action='increase'))
    markup.row(back_btn, count_btn, next_btn)

    return markup


from ssm_ptc.models.hmm import HMM
from ssm_ptc.observations.ar_logit_normal_observation import ARLogitNormalObservation
from ssm_ptc.transformations.linear import LinearTransformation
from ssm_ptc.utils import find_permutation, random_rotation, k_step_prediction

import torch
import numpy as np
import matplotlib.pyplot as plt

from tqdm import tqdm_notebook as tqdm
import sys

torch.manual_seed(0)

K = 3
D = 2
T = 100

As = [random_rotation(D) for _ in range(K)]
true_tran = LinearTransformation(K=K, d_in=D, D=D, As=As)

bounds = np.array([[0, 20], [-5, 25]])
true_observation = ARLogitNormalObservation(K=K, D=D, M=0, transformation=true_tran, bounds=bounds)

true_model = HMM(K=K, D=D, M=0, observation=true_observation)

z, data = true_model.sample(T, return_np=False)

# Define a model to fit the data

tran = LinearTransformation(K=K, d_in=D, D=D)
observation = ARLogitNormalObservation(K=K, D=D, M=0, transformation=tran, bounds=bounds)
model = HMM(K=K, D=D, M=0, observation=observation)

# Model fitting

num_iters = 10 #9000

pbar = tqdm(total=num_iters, file=sys.stdout)

optimizer = torch.optim.Adam(model.params, lr=0.001)

losses = []
for i in np.arange(num_iters):

    optimizer.zero_grad()

    loss = model.loss(data)
    loss.backward(retain_graph=True)
    optimizer.step()

    loss = loss.detach().numpy()
    losses.append(loss)

    if i % 10 == 0:
        pbar.set_description('iter {} loss {:.2f}'.format(i, loss))
        pbar.update(10)

# check reconstruction
x_reconstruct = model.sample_condition_on_zs(z, data[0])

# infer the latent states
infer_z = model.most_likely_states(data)

perm = find_permutation(z.numpy(), infer_z, K1=K, K2=K)

model.permute(perm)
hmm_z = model.most_likely_states(data)

# check prediction
x_predict_cond_z = k_step_prediction(model, z, data)


# -*- coding: utf-8 -*-

import speech_recognition as sr
import urllib3
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
import webbrowser
import os
import pandas as pd
from bs4 import BeautifulSoup
import requests

def hava():
    url = requests.get("https://www.ntvhava.com/konum/eskisehir/3-gunluk-hava-tahmini")
    content = url.content
    soup = BeautifulSoup(content, "html.parser")

    günler = soup.find_all("div", class_="date")
    sıcaklıklar = soup.find_all("span", class_="range2 font25")
    sıcak = []
    for sıcaklık in sıcaklıklar:
        sıcak.append(sıcaklık.getText())
    del sıcak[1]
    del sıcak[2]
    del sıcak[3]

    tablo = {"-----Gün-----": [],
             "Sıcaklık": []}

    for gün, sıc in zip(günler, sıcak):
        tablo["-----Gün-----"].append(gün.getText())
        tablo["Sıcaklık"].append(sıc)

    data = pd.DataFrame(tablo)
    print(data)

urls = ["https://www.reddit.com/","https://www.youtube.com/","https://www.google.com/"]
print("***************SARVIS V1.0***************")
while True:
    r = sr.Recognizer()
    with sr.Microphone() as source:
        print("Dinliyorum... >> ")
        r.adjust_for_ambient_noise(source, duration=1)
        audio = r.listen(source, phrase_time_limit=5)

    try:
        command = r.recognize_google(audio,key=None, language="tr-TR")

        if "kapat" in command:
            print(command)
            print("durduruldu")
            sys.exit()
        elif command == "sayfalar":
            print(command)
            for sayfa in urls:
                webbrowser.open_new_tab(sayfa)
                print(sayfa+" açılıyor...")
        elif "hava durumu" in command:
            hava()
        elif command == "kaydet":
            try:
                    r = sr.Recognizer()
                    with sr.Microphone() as source:
                        print("Yazmak için dinliyorum... ")
                        r.adjust_for_ambient_noise(source, duration=1)
                        audio = r.listen(source, phrase_time_limit=10)
                    command_yaz = r.recognize_google(audio, key=None, language="tr-TR")
                    os.chdir("C:\\Users\\dogu2\\Desktop\\")
                    with open("yazıcı.txt", "w", encoding="utf-8") as yazdır:
                        yazdır.write(command_yaz)
                        yazdır.close()
            except sr.UnknownValueError:
                print("Anlaşılmadı..")
        elif command == "oku":
            metin = open("yazıcı.txt", "r", encoding="utf-8")
            okut = metin.read()
            print(okut)
        if "arama" in command:
            driver = webdriver.Chrome()
            driver.get("https://www.google.com/")
            ara = driver.find_element_by_class_name("gsfi")
            try:
                r = sr.Recognizer()
                with sr.Microphone() as source:
                    print("Aranacak şeyi söyle ")
                    r.adjust_for_ambient_noise(source, duration=1)
                    audio = r.listen(source, phrase_time_limit=5)

                    command_ara = r.recognize_google(audio, key=None, language="tr-TR")
                    ara.send_keys(command_ara)
                    ara.send_keys(Keys.ENTER)

            except sr.UnknownValueError:
                print("Tekrar söyle")

    except sr.UnknownValueError:
        print("Anlaşılmadı.Tekrar dene")


"""Defines URL patterns for learning_logs."""

from django.conf.urls import url

from .views import index, topics, topic, new_topic, new_entry, edit_entry, edit_topic

urlpatterns = [
    #Home page
    url(r'^$', index, name='index'),

    # Show all topics.
    url(r'^topics/$', topics, name='topics'),

    #show all topics
    url(r'^topics/(?P<topic_id>\d+)/$', topic, name='topic'),

    #page for adding a new topic
    url(r'^new_topics/$', new_topic, name='new_topic'),

    url(r'^new_entry/(?P<topic_id>\d+)/$', new_entry, name='new_entry'),

    #page for editing an entry
    url(r'^edit_entry/(?P<edit_id>\d+)/$', edit_entry, name='edit_entry'),

    #page for editing a topic
    url(r'^edit_topic/(?P<edit_id>\d+)/$', edit_topic, name='edit_topic')
]