# Licensed to the .NET Foundation under one or more agreements.
# The .NET Foundation licenses this file to you under the Apache 2.0 License.
# See the LICENSE file in the project root for more information.

import unittest

from iptest import IronPythonTestCase, is_cli, is_netcoreapp21, is_posix, run_test, skipUnlessIronPython

if is_cli:
    import clr
    import clrtype

    clr.AddReference("System.Xml")

    import System
    from System.Reflection import BindingFlags

    class IProduct(object, metaclass=clrtype.ClrInterface):
        __metaclass__ = clrtype.ClrInterface # https://github.com/IronLanguages/ironpython3/issues/836

        _clrnamespace = "IronPython.Samples.ClrType"

        @property
        @clrtype.accepts()
        @clrtype.returns(str)
        def Name(self): raise RuntimeError("this should not get called")

        @property
        @clrtype.accepts()
        @clrtype.returns(float)
        def Cost(self): raise RuntimeError("this should not get called")

        @clrtype.accepts()
        @clrtype.returns(bool)
        def IsAvailable(self): raise RuntimeError("this should not get called")

    class Product(IProduct, metaclass=clrtype.ClrClass):
        __metaclass__ = clrtype.ClrClass # https://github.com/IronLanguages/ironpython3/issues/836

        _clrnamespace = "IronPython.Samples.ClrType"

        _clrfields = {
            "name" : str,
            "cost" : float,
            "_quantity" : int
        }

        CLSCompliant = clrtype.attribute(System.CLSCompliantAttribute)
        clr.AddReference("System.Xml")
        XmlRoot = clrtype.attribute(System.Xml.Serialization.XmlRootAttribute)

        _clrclassattribs = [
            # Use System.Attribute subtype directly for custom attributes without arguments
            System.ObsoleteAttribute,
            # Use clrtype.attribute for custom attributes with arguments (either positional, named, or both)
            CLSCompliant(False),
            XmlRoot("product", Namespace="www.contoso.com")
        ]

        def __init__(self, name, cost, quantity):
            self.name = name
            self.cost = cost
            self._quantity = quantity

        # IProduct methods
        def Name(self): return self.name
        def Cost(self): return self.cost
        def IsAvailable(self): return self.quantity != 0

        @property
        @clrtype.accepts()
        @clrtype.returns(int)
        def quantity(self): return self._quantity

        @quantity.setter
        @clrtype.accepts(int)
        @clrtype.returns()
        def quantity(self, value): self._quantity = value

        @clrtype.accepts(float)
        @clrtype.returns(float)
        def calc_total(self, discount=0.0):
            return (self.cost - discount) * self.quantity

    if not is_netcoreapp21 and not is_posix:
        class NativeMethods(object, metaclass=clrtype.ClrClass):
            # Note that you could also the "ctypes" modules instead of pinvoke declarations
            __metaclass__ = clrtype.ClrClass # https://github.com/IronLanguages/ironpython3/issues/836

            from System.Runtime.InteropServices import DllImportAttribute, PreserveSigAttribute
            DllImport = clrtype.attribute(DllImportAttribute)
            PreserveSig = clrtype.attribute(PreserveSigAttribute)

            @staticmethod
            @DllImport("user32.dll")
            @PreserveSig()
            @clrtype.accepts(System.Char)
            @clrtype.returns(System.Boolean)
            def IsCharAlpha(c): raise RuntimeError("this should not get called")

            @staticmethod
            @DllImport("user32.dll")
            @PreserveSig()
            @clrtype.accepts(System.IntPtr, System.String, System.String, System.UInt32)
            @clrtype.returns(System.Int32)
            def MessageBox(hwnd, text, caption, type): raise RuntimeError("this should not get called")

@skipUnlessIronPython()
class ClrTypeTest(IronPythonTestCase):
    def setUp(self):
        super(ClrTypeTest, self).setUp()
        self.p = Product("Widget", 10.0, 42)

    @unittest.skipIf(is_netcoreapp21, 'No DefinePInvokeMethod')
    @unittest.skipIf(is_posix, 'Windows specific P/Invoke')
    def test_pinvoke_method(self):
        self.assertTrue(NativeMethods.IsCharAlpha('A'))
        # Call statically-typed method from another .NET language (simulated using Reflection)
        isCharAlpha = clr.GetClrType(NativeMethods).GetMethod('IsCharAlpha')
        args = System.Array[object](('1'.Chars[0],))
        self.assertFalse(isCharAlpha.Invoke(None, args))

    def test_classattributes(self):
        t = clr.GetClrType(self.p.GetType())
        oa = t.GetCustomAttributes(System.ObsoleteAttribute, True)[0]
        clsc = t.GetCustomAttributes(System.CLSCompliantAttribute, True)[0]
        xmlr = t.GetCustomAttributes(System.Xml.Serialization.XmlRootAttribute, True)[0]

        self.assertEqual(oa.Message, None)
        self.assertFalse(clsc.IsCompliant)
        self.assertEqual(xmlr.ElementName, 'product')
        self.assertEqual(xmlr.Namespace, 'www.contoso.com')

    def test_clrtypeinfo(self):
        t = self.p.GetType()
        self.assertEqual(t.FullName, 'IronPython.Samples.ClrType.Product')

        # fields and properties
        bf = BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.FlattenHierarchy
        f = [field.Name for field in t.GetFields(bf)]
        self.assertTrue('name' in f)
        self.assertTrue('cost' in f)
        self.assertTrue('_quantity' in f)

        p = [property.Name for property in t.GetProperties(bf)]
        self.assertTrue('quantity' in p)

        # methods
        m = [method.Name for method in t.GetMethods() if method.DeclaringType == method.ReflectedType]
        self.assertTrue('calc_total' in m)
        self.assertTrue('quantity' in m)

    def test_interface_members(self):
        # Calling IProduct.Name and IProduct.IsAvailable as a strongly-typed members
        name = clr.GetClrType(IProduct).GetProperty("Name").GetGetMethod()
        self.assertEqual(name.Invoke(self.p, None), 'Widget')
        isAvailable = clr.GetClrType(IProduct).GetMethod("IsAvailable")
        self.assertTrue(isAvailable.Invoke(self.p, None))

    def _call_typed_method(self):
        """Calling calc_total as a strongly-typed method using Reflection"""
        calc_total = self.p.GetType().GetMethod("calc_total")
        args = System.Array[object]( (1.0,) )
        res = calc_total.Invoke(self.p, args)
        return res

    # these methods are named with numbers because they need to run in a specific order

    def test_0typed_method(self):
        self.assertEqual(self._call_typed_method(), 378.0)

    def test_1python_dynamism(self):
        # The object is still a Python object, and so allows setting new attributes
        self.p.sale_discount = 2.0

        # The class is also a Python class, and can also be modified
        def new_calc_total(self, discount=0.0):
            return (self.cost - (discount + self.sale_discount)) * self.quantity
        Product.calc_total = new_calc_total
        self.assertTrue(self._call_typed_method(), 294.0)

run_test(__name__)


import subprocess
import tempfile

from mu.formats.html.writer import Writer as HtmlWriter

class Writer(HtmlWriter):
    """
    Same as HTML writer, except that we convert the output to Markdown.
    """

    def write_to(self, path: str) -> None:
        # Write html to temporary file
        with tempfile.NamedTemporaryFile("w") as of:
            super().write_to(of.name)
            of.flush()
            # Convert file with pandoc
            subprocess.check_call(
                ["pandoc", "--from=html", "--to=markdown", of.name, f"--output={path}"]
            )


from datetime import datetime
import json
import logging
import os

class HarFileHandler(logging.FileHandler):
    def __init__(self, filename, mode: str, encoding: str=None, delay: bool=False, errors=None,
        version="1.2", creator_version="537.36", pages_title="HAR Generator") -> None:
        super().__init__(filename, mode=mode, encoding=encoding, delay=delay)
        self._starter(version, creator_version, pages_title)

    def _starter(self, version, creator_version, pages_title):
        starter = f'''{{
  "log": {{
    "version": "{version}",
    "creator": {{
      "name": "WebInspector",
      "version": "{creator_version}"
    }},
    "pages": [
      {{
        "startedDateTime": "{datetime.now().strftime("%Y-%m-%dT%H:%M:%S.123Z")}",
        "id": "page_1",
        "title": "{pages_title}",
        "pageTimings": {{
          "onContentLoad": 908.7060000747442,
          "onLoad": 2029.8569998703897
        }}
      }}
    ],
    "entries": [
'''
        self.emit(logging.makeLogRecord({'msg': starter}))

    def _closer(self):
        closer = '''    ]
  }
}
'''
        self.emit(logging.makeLogRecord({'msg': closer}))

    def close(self) -> None:
        self._closer()
        return super().close()

class HarLogger:
    def __init__(self):
        self.logger = logging.getLogger()
        self.logger.setLevel(logging.DEBUG)
        file_handler = HarFileHandler(os.path.join('logs', 'test.har'), mode='w')
        file_handler.setLevel(logging.DEBUG)
        file_handler.setFormatter(logging.Formatter(fmt=f'%(message)s'))
        self.logger.addHandler(file_handler)
        self.entry_delimiter = ""

    def info(self, request, response):
        self.debug(request, response)

    def debug(self, request, response):
        entry = self._make_entry(request, response)
        self.logger.info(entry)

    def warning(self, request, response):
        self.debug(request, response)

    def critical(self, request, response):
        self.debug(request, response)

    def error(self, request, response):
        self.debug(request, response)

    def _make_entry(self, request, response):
        entry = self.entry_delimiter + f'''{{
        "_fromCache": "disk",
        "_initiator": {{
          "type": "script",
          "stack": {{}}
        }},
        "_priority": "High",
        "_resourceType": "fetch",
        "cache": {{}},
        "pageref": "page_1",
        "request": {{
          "method": "{response._method}",
          "url": "{request.full_url}",
          "httpVersion": "http/2.0",
          "headers": {self._make_headers(request.header_items())},
          "queryString": [],
          "cookies": [],
          "headersSize": -1,
          "bodySize": 0
        }},
        "response": {{
          "status": {response.status},
          "statusText": "{response.reason}",
          "httpVersion": "http/2.0",
          "headers": {self._make_headers(response.headers._headers)},
          "cookies": [],
          "content": {{
            "size": 0,
            "mimeType": "{response.headers._default_type}",
            "text": {json.dumps(response.read().decode(response.headers.get_content_charset()))}
          }},
          "redirectURL": "",
          "headersSize": -1,
          "bodySize": 0,
          "_transferSize": 0,
          "_error": null
        }},
        "serverIPAddress": "140.82.113.4",
        "startedDateTime": "{datetime.now().strftime("%Y-%m-%dT%H:%M:%S.123Z")}",
        "time": 4.252000013366342,
        "timings": {{
          "blocked": 1.195999818906188,
          "dns": -1,
          "ssl": -1,
          "connect": -1,
          "send": 0,
          "wait": 1.8329999623298645,
          "receive": 1.223000232130289,
          "_blocked_queueing": 1.076999818906188
        }}
      }}'''
        self.entry_delimiter = ",\n"
        return entry

    def _make_headers(self, r_headers, indent=2*6):
      list_dict = [{"name": header[0], "value": header[1]} for header in r_headers]
      return json.dumps(list_dict, indent=indent)

if __name__ == '__main__':
    a = HarLogger()
    from urllib import request
    headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0", "Accept":"application/json,text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Content-Type": "application/json", "DNT":"1","Connection":"close", "Upgrade-Insecure-Requests":"1"}
    req = request.Request('https://google.com', headers=headers)
    with request.urlopen(req, timeout=3) as response:
        print(dir(req))
        print(dir(response))
        a.info(req, response)
        b =1

    headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0", "Accept":"application/json,text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Content-Type": "application/json", "DNT":"1","Connection":"close", "Upgrade-Insecure-Requests":"1"}
    req = request.Request('https://reqres.in/api/users?page=2', headers=headers)
    with request.urlopen(req, timeout=6) as response:
        print(req.full_url)
        a.info(req, response)
        b =1


from django.urls import path, re_path
from . import views

urlpatterns = [
    path('login', views.login, name="admin_login"),
    path('logout', views.logout, name="admin_logout"),
    path('login/validate', views.validate_admin_login, name="validate_admin_login"),
    path('', views.dashboard, name="admin_dashboard"),
    path('message/<int:id>/send/<int:uid>', views.send_message_customers, name="admin_send_message_customers"),

    # ajax
    path('ajax/datatables/customers', views.admin_datatable_ajax_customers_list, name="admin_datatable_ajax_customers_list"),
    path('ajax/datatables/staffs', views.admin_datatable_ajax_staffs, name="admin_datatable_ajax_staffs"),
    path('ajax/datatables/managers', views.admin_datatable_ajax_managers, name="admin_datatable_ajax_managers"),
    path('ajax/datatables/customers/followup', views.admin_datatable_ajax_customers_followup, name="admin_datatable_ajax_customers_followup"),
    path('ajax/datatables/followup', views.admin_datatable_ajax_followup, name="admin_datatable_ajax_followup"),
    path('ajax/datatables/message', views.admin_datatable_ajax_message, name="admin_datatable_ajax_message"),
    path('ajax/datatables/purchases', views.admin_datatable_ajax_customers_purchases, name="admin_datatable_ajax_customers_purchases"),
    path('ajax/datatables/payments', views.admin_datatable_ajax_customers_payments, name="admin_datatable_ajax_customers_payments"),
    path('ajax/datatables/refunds', views.admin_datatable_ajax_customers_refunds, name="admin_datatable_ajax_customers_refunds"),
    path('ajax/datatables/extras', views.admin_datatable_ajax_customers_extras, name="admin_datatable_ajax_customers_extras"),
    path('ajax/datatables/subscriptions', views.admin_datatable_ajax_customers_subscriptions, name="admin_datatable_ajax_customers_subscriptions"),
    path('ajax/datatables/extraless', views.admin_datatable_ajax_customers_extraless, name="admin_datatable_ajax_customers_extraless"),
    path('ajax/datatables/vacations', views.admin_datatable_ajax_customers_vacations, name="admin_datatable_ajax_customers_vacations"),
    path('ajax/datatables/reports/payments', views.admin_datatable_ajax_payments_report, name="admin_datatable_ajax_payments_report"),
    path('ajax/delivery/info', views.get_delivery_info, name="admin_get_delivery_info"),
    path('ajax/purchase/info', views.get_purchase_info, name="admin_get_purchase_info"),
    path('ajax/routes/customers', views.get_customers_of_route, name="admin_get_customers_of_route"),
    path('ajax/message/customers/date', views.get_message_date_of_customer, name="admin_get_message_date_of_customer"),
    path('ajax/routes/route-order/save', views.save_route_order, name="admin_save_route_order"),
    path('ajax/routes/customers/change', views.change_customers_route, name="admin_change_customers_route"),
    path('ajax/routes/customers/assign', views.assign_customers_route, name="admin_assign_customers_route"),
    path('ajax/routes/customers/assignations/clear', views.clear_customers_route_assignation, name="admin_clear_customers_route_assignation"),
    path('ajax/routes/customers/assignations/clear/all', views.clear_all_route_assignation, name="admin_clear_all_route_assignation"),

    #edit
    path('customers/edit/<int:id>', views.customers_edit, name="admin_customers_edit"),
    path('staffs/edit/<int:id>', views.staffs_edit, name="admin_staffs_edit"),
    path('managers/edit/<int:id>', views.managers_edit, name="admin_managers_edit"),
    path('payments/edit/<int:id>', views.payments_edit, name="admin_payments_edit"),
    path('refunds/edit/<int:id>', views.refunds_edit, name="admin_refunds_edit"),
    path('extras/edit/<int:id>', views.extras_edit, name="admin_extras_edit"),
    path('followup/edit/<int:id>', views.followup_edit, name="admin_followup_edit"),
    path('subscriptions/<int:id>', views.subscriptions_edit, name="admin_subscriptions_edit"),
    path('vacations/<int:id>', views.vacations_edit, name="admin_vacations_edit"),
    path('extraless/<int:id>', views.extraless_edit, name="admin_extraless_edit"),

    # create
    path('customers/create', views.customers_create, name="admin_customers_create"),
    path('staffs/create', views.staffs_create, name="admin_staffs_create"),
    path('managers/create', views.managers_create, name="admin_managers_create"),
    path('payments/create', views.payments_create, name="admin_payments_create"),
    path('refunds/create', views.refunds_create, name="admin_refunds_create"),
    path('extras/create', views.extras_create, name="admin_extras_create"),
    path('subscriptions/create', views.subscriptions_create, name="admin_subscriptions_create"),
    path('extraless/create', views.extraless_create, name="admin_extraless_create"),
    path('vacations/create', views.vacations_create, name="admin_vacations_create"),
    path('message/create', views.message_create, name="admin_message_create"),
    path('followup/add', views.add_followup_customer, name="admin_add_followup_customer"),

    # views
    path('customers/transactions/purchases', views.customers_purchases, name="admin_customers_purchases"),
    path('customers/transactions/payments', views.customers_payments, name="admin_customers_payments"),
    path('customers/transactions/refunds', views.customers_refunds, name="admin_customers_refunds"),
    path('customers/transactions/extras', views.customers_extras, name="admin_customers_extras"),
    path('customers/transactions/<int:id>', views.customer_transactions, name="admin_customers_transactions"),
    path('customers/subscriptions', views.customer_subscriptions, name="admin_customers_subscriptions"),
    path('customers/extraless', views.customer_extraless, name="admin_customers_extraless"),
    path('customers/vacations', views.customer_vacations, name="admin_customers_vacations"),
    path('customers/list', views.customers_list, name="admin_customers_list"),
    path('staffs/list', views.staff_list, name="admin_staff_list"),
    path('managers/list', views.manager_list, name="admin_manager_list"),
    path('customers/followup', views.customers_followup, name="admin_customers_followup"),
    path('followup', views.followup, name="admin_followup"),
    path('message', views.message, name="admin_message"),
    path('message/<int:id>/customers', views.message_customers, name="admin_message_customers"),
    path('route/order', views.route_ordering, name="admin_route_ordering"),
    path('reports/payments', views.payments_report, name="admin_payments_report"),

    # delete
    path('customers/delete/<int:id>', views.customer_delete, name="admin_customer_delete"),
    path('staffs/delete/<int:id>', views.staff_delete, name="admin_staff_delete"),
    path('managers/delete/<int:id>', views.manager_delete, name="admin_manager_delete"),
    path('payments/delete/<int:id>', views.payment_delete, name="admin_payment_delete"),
    path('refunds/delete/<int:id>', views.refund_delete, name="admin_refund_delete"),
    path('extras/delete/<int:id>', views.extra_delete, name="admin_extra_delete"),
    path('subscriptions/delete/<int:id>', views.subscription_delete, name="admin_subscription_delete"),
    path('extraless/delete/<int:id>', views.extraless_delete, name="admin_extraless_delete"),
    path('vacation/delete/<int:id>', views.vacation_delete, name="admin_vacation_delete"),
    path('followup/remove/<int:id>', views.remove_customer_followup, name="admin_remove_customer_followup"),
    path('message/delete/<int:id>', views.delete_message, name="admin_delete_message"),
    path('followup/remove/all', views.remove_all_customer_followup, name="admin_remove_all_customer_followup"),

    # end
    path('subscriptions/end/<int:id>', views.subscription_end, name="admin_subscription_end"),
    path('vacations/end/<int:id>', views.vacation_end, name="admin_vacation_end"),
    path('extraless/end/<int:id>', views.extraless_end, name="admin_extraless_end"),
]


__author__ = "Suyash Soni"
__email__ = "suyash.soni248@gmail.com"

from flask import request
from util.response import intercept, Response
from controller.base import BaseController
from util.injector import inject
from service.plag_dao import PlagiarismDAO
from util.constants.error_codes import HttpErrorCode
from util.error_handlers.exceptions import ExceptionBuilder, BadRequest

class Document(BaseController):
    plag_dao: PlagiarismDAO = inject(PlagiarismDAO)

    @intercept()
    def post(self, *args, **kwargs):
        """Adds a new document to repo"""

        data = request.get_json(force=True)

        content = data.get('content', '')
        title = data.get('title', '')
        description = data.get('description', '')
        author = data.get('author', '')

        if content and title:
            doc = self.plag_dao.create_doc(content, title, description=description, author=author)
        else:
            ExceptionBuilder(BadRequest).error(HttpErrorCode.REQUIRED_FIELD, 'content', 'title').throw()

        return Response(status_code=201, message='Document added successfully!')

    @intercept()
    def get(self):
        """
        Fetches all the documents(paginated).
        :return:
        """
        res = self.plag_dao.get_docs(page=int(request.args.get("page", 1)),
                                    per_page=int(request.args.get("per_page", 10)), all='all' in request.args)
        docs_info = dict(data=[d.to_dict() for d in res['data']], count=res['count'])
        print(docs_info)
        return Response(data=docs_info)


"""boolean_default

Revision ID: 978129bf15fe
Revises: dfefdac3d90c
Create Date: 2021-03-04 15:29:30.427535

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '978129bf15fe'
down_revision = 'dfefdac3d90c'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.alter_column('samplers', 'private',
               existing_type=sa.BOOLEAN(),
               nullable=False)
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.alter_column('samplers', 'private',
               existing_type=sa.BOOLEAN(),
               nullable=True)
    # ### end Alembic commands ###


from absl.testing import parameterized

import numpy as np
import tensorflow.compat.v2 as tf
import tensorflow_probability as tfp

import tf_quant_finance as tff
from tensorflow.python.framework import test_util  # pylint: disable=g-direct-tensorflow-import

@test_util.run_all_in_graph_and_eager_modes
class SimulatedDataCalibrationTest(parameterized.TestCase, tf.test.TestCase):

  @parameterized.named_parameters(
      {
          'testcase_name': 'conjugate_gradient_optimizer',
          'optimizer_fn': tff.math.optimizer.conjugate_gradient_minimize,
      }, {
          'testcase_name': 'bfgs_optimizer',
          'optimizer_fn': tfp.optimizer.bfgs_minimize,
      })
  def test_calibration_correctness(self, optimizer_fn):
    true_parameters = np.array([[0.04, 0.15, 0.5, 0.3, 0.3],
                                [0.02, 0.25, 0.4, 0.25, 0.25]])

    forwards = np.array([5., 6.])
    expiries = np.array([1., 1.])

    strikes = np.array([
        [
            0.24893534, 0.35810619, 0.51515402, 0.74107533, 1.06607467,
            1.53360279, 2.20616584, 3.17368209, 4.56550358, 6.56770979,
            9.44798554, 13.59140914
        ],
        [
            0.8120117, 1.16812025, 1.68040056, 2.41734193, 3.47746967,
            5.00251751, 7.19637661, 10.35235484, 14.89239051, 21.4234634,
            30.8187449, 44.33433659
        ],
    ])

    volatilities = np.array([
        [
            0.53809037, 0.51236712, 0.48535488, 0.45686258, 0.42666483,
            0.39451975, 0.3602712, 0.32435877, 0.29085295, 0.28834027,
            0.3644639, 0.45468296
        ],
        [
            0.60080077, 0.55414627, 0.50349337, 0.44780265, 0.38588647,
            0.31868425, 0.27922126, 0.38249619, 0.51199324, 0.62087246,
            0.71471493, 0.7980863
        ],
    ])

    (model_parameters, converged, _) = self.evaluate(
        tff.experimental.svi.calibration(
            forwards=forwards,
            expiries=expiries,
            strikes=strikes,
            volatilities=volatilities,
            optimizer_fn=optimizer_fn))

    # Assert model convergence to expected parameters.
    self.assertTrue(converged.all())
    self.assertAllClose(model_parameters, true_parameters, atol=1e-4, rtol=1e-3)

  def test_calibration_correctness_bulk(self):
    np.random.seed(321)
    batch_size = 7
    num_strikes = 30
    forwards = 4. + 5. * np.random.random(size=batch_size)
    expiries = 0.5 + np.ones_like(forwards)
    log_moneyness = np.random.normal(size=(batch_size, num_strikes))
    strikes = forwards[:, np.newaxis] * np.exp(log_moneyness)

    svi_a = 0.1 + 0.3 * np.random.random(size=batch_size)
    svi_b = 0.3 * np.random.random(size=batch_size)
    svi_rho = np.random.random(size=batch_size) - 0.5
    svi_m = 0.5 * np.random.normal(size=batch_size)
    svi_sigma = 0.1 + 0.5 * np.random.random(size=batch_size)
    true_parameters = np.transpose([svi_a, svi_b, svi_rho, svi_m, svi_sigma])

    target_volatilities = tff.experimental.svi.implied_volatility_from_raw_svi_parameters(
        svi_parameters=true_parameters,
        log_moneyness=log_moneyness,
        expiries=expiries)

    (model_parameters, converged, _) = self.evaluate(
        tff.experimental.svi.calibration(
            forwards=forwards,
            expiries=expiries,
            strikes=strikes,
            volatilities=target_volatilities,
            optimizer_fn=tfp.optimizer.bfgs_minimize))

    # Assert model convergence to expected parameters.
    self.assertTrue(converged.all())
    self.assertAllClose(model_parameters, true_parameters, atol=1e-3, rtol=1e-2)

  def test_weights_to_handle_outliers(self):
    true_parameters = np.array([[0.04, 0.15, 0.5, 0.3, 0.3]])

    forwards = np.array([5.])
    expiries = np.array([1.])

    strikes = np.array([
        [
            0.24893534, 0.35810619, 0.51515402, 0.74107533, 1.06607467,
            1.53360279, 2.20616584, 3.17368209, 4.56550358, 6.56770979,
            9.44798554, 13.59140914
        ],
    ])

    volatilities = np.array([
        [
            0.53809037, 0.51236712, 0.48535488, 0.45686258, 0.42666483,
            0.39451975, 0.3602712, 0.32435877, 0.29085295, 0.28834027,
            0.3644639, 0.45468296
        ],
    ])
    weights = np.ones_like(volatilities)

    (model_parameters, converged, _) = self.evaluate(
        tff.experimental.svi.calibration(
            forwards=forwards,
            expiries=expiries,
            strikes=strikes,
            volatilities=volatilities,
            weights=weights,
            optimizer_fn=tfp.optimizer.bfgs_minimize))

    with self.subTest('Model without outliers converged'):
      self.assertTrue(converged.all())
    with self.subTest('Parameters correct for model without outliers'):
      self.assertAllClose(
          model_parameters, true_parameters, atol=1e-4, rtol=1e-3)

    # introduce outliers - changes tail volatilities to much higher values
    volatilities[0, 0] = 0.7
    volatilities[0, 0] = 0.8

    (model_parameters, converged, _) = self.evaluate(
        tff.experimental.svi.calibration(
            forwards=forwards,
            expiries=expiries,
            strikes=strikes,
            volatilities=volatilities,
            weights=weights,
            optimizer_fn=tfp.optimizer.bfgs_minimize))
    with self.subTest('Model with outliers did not converge'):
      self.assertFalse(converged.all())
    with self.subTest('Parameters incorrect for model with outliers'):
      self.assertNotAllClose(
          model_parameters, true_parameters, atol=1e-4, rtol=1e-3)

    # zero out optimization weights, corresponding to outliers
    weights[0, 0] = 0
    weights[0, -1] = 0

    (model_parameters, converged, _) = self.evaluate(
        tff.experimental.svi.calibration(
            forwards=forwards,
            expiries=expiries,
            strikes=strikes,
            volatilities=volatilities,
            weights=weights,
            optimizer_fn=tfp.optimizer.bfgs_minimize))

    with self.subTest('Model with weighted out outliers converged'):
      self.assertTrue(converged.all())
    with self.subTest(
        'Parameters correct for model with weighted out outliers'):
      self.assertAllClose(
          model_parameters, true_parameters, atol=1e-4, rtol=1e-3)

@test_util.run_all_in_graph_and_eager_modes
class RealMarketDataCalibrationTest(parameterized.TestCase, tf.test.TestCase):
  r"""Tests are loosely based on real market prices for GOOG call options.

  The quotes for GOOG210820C* were taken on 2021-05-28 from
  https://finance.yahoo.com/quote/GOOG/options?date=1629417600
  """

  def setUp(self):
    super().setUp()
    self.forwards = np.array([2402.])
    self.expiries = np.array([0.23])
    self.strikes = np.array([[
        1700., 1800., 1900., 2000., 2050., 2100., 2200., 2250., 2350., 2400.,
        2450., 2500., 2550., 2600., 2650., 2700., 2750., 2800., 2850., 2900.,
        2950., 3000.
    ]])
    self.volatilities = np.array([[
        0.5335, 0.4882, 0.4389, 0.3937, 0.3749, 0.3569, 0.3259, 0.3135, 0.29,
        0.283, 0.2717, 0.2667, 0.2592, 0.2566, 0.2564, 0.2574, 0.2595, 0.2621,
        0.2669, 0.2732, 0.2826, 0.2967
    ]])

  @parameterized.named_parameters(
      {
          'testcase_name':
              'conjugate_gradient_optimizer',
          'optimizer_fn':
              tff.math.optimizer.conjugate_gradient_minimize,
          'expected_parameters':
              np.array([[-0.1705, 0.3222, -0.0772, 0.0514, 0.5781]])
      }, {
          'testcase_name':
              'bfgs_optimizer',
          'optimizer_fn':
              tfp.optimizer.bfgs_minimize,
          'expected_parameters':
              np.array([[-0.1825, 0.3306, -0.0988, 0.0368, 0.6011]])
      })
  def test_real_market_data_calibration(self, optimizer_fn,
                                        expected_parameters):
    svi_parameters, converged, _ = self.evaluate(
        tff.experimental.svi.calibration(
            forwards=self.forwards,
            expiries=self.expiries,
            strikes=self.strikes,
            volatilities=self.volatilities,
            optimizer_fn=optimizer_fn,
            tolerance=1e-4))
    self.assertTrue(converged.all())
    self.assertAllClose(
        svi_parameters, expected_parameters, atol=1e-4, rtol=2e-2)

  @parameterized.named_parameters(
      {
          'testcase_name': 'conjugate_gradient_optimizer',
          'optimizer_fn': tff.math.optimizer.conjugate_gradient_minimize,
          'atol': 0.01,
          'rtol': 0.02
      }, {
          'testcase_name': 'bfgs_optimizer',
          'optimizer_fn': tfp.optimizer.bfgs_minimize,
          'atol': 0.01,
          'rtol': 0.02
      })
  def test_model_well_approximates_target_volatilities(self, optimizer_fn, atol,
                                                       rtol):
    svi_parameters, converged, _ = self.evaluate(
        tff.experimental.svi.calibration(
            forwards=self.forwards,
            expiries=self.expiries,
            strikes=self.strikes,
            volatilities=self.volatilities,
            optimizer_fn=optimizer_fn,
            tolerance=1e-4))
    self.assertTrue(converged.all())

    model_volatilities = self.evaluate(
        tff.experimental.svi.implied_volatility_from_raw_svi_parameters(
            svi_parameters=svi_parameters,
            forwards=self.forwards,
            strikes=self.strikes,
            expiries=self.expiries))
    self.assertAllClose(
        model_volatilities, self.volatilities, atol=atol, rtol=rtol)

  @parameterized.named_parameters(
      {
          'testcase_name': 'conjugate_gradient_optimizer',
          'optimizer_fn': tff.math.optimizer.conjugate_gradient_minimize,
          'rtol': 0.04
      }, {
          'testcase_name': 'bfgs_optimizer',
          'optimizer_fn': tfp.optimizer.bfgs_minimize,
          'rtol': 0.05
      })
  def test_model_well_approximates_target_variance(self, optimizer_fn, rtol):
    svi_parameters, converged, _ = self.evaluate(
        tff.experimental.svi.calibration(
            forwards=self.forwards,
            expiries=self.expiries,
            strikes=self.strikes,
            volatilities=self.volatilities,
            optimizer_fn=optimizer_fn,
            tolerance=1e-4))
    self.assertTrue(converged.all())

    total_variance = self.volatilities**2 * self.expiries
    model_variance = self.evaluate(
        tff.experimental.svi.total_variance_from_raw_svi_parameters(
            svi_parameters=svi_parameters,
            forwards=self.forwards,
            strikes=self.strikes))
    self.assertAllClose(model_variance, total_variance, rtol=rtol, atol=1e-4)

if __name__ == '__main__':
  tf.test.main()


import concurrent
from concurrent.futures import Future, ThreadPoolExecutor, ALL_COMPLETED
from dataclasses import dataclass
from typing import Callable, List, Tuple, Deque

from dragoneye.utils.app_logger import logger

@dataclass
class ThreadedFunctionData:
    callable: Callable
    args: Tuple
    error_msg: str
    timeout_msg: str = None

def execute_parallel_functions_in_threads(tasks_groups: Deque[List[ThreadedFunctionData]], max_workers, timeout=None)\
        -> List[Tuple[Future, str, str]]:
    """
    This function takes a Queue (tasks_groups: Deque[List[ThreadedFunctionData]])
    of lists, when each queued item represents a list of functions that should be run in parallel.
    """
    tasks_responses: List[Tuple[Future, str, str]] = []
    with ThreadPoolExecutor(max_workers=max_workers) as executor:

        for tasks_group in tasks_groups:
            for task in tasks_group:
                tasks_responses.append((
                    executor.submit(task.callable, *task.args),
                    task.error_msg, task.timeout_msg))

            concurrent.futures.wait([response[0] for response in tasks_responses], timeout=timeout, return_when=ALL_COMPLETED)

        timeout_tasks = [response for response in tasks_responses if response[0].running()]
        for timeout_task in timeout_tasks:
            logger.exception(timeout_task[2])
        failed_tasks = [response for response in tasks_responses if response[0].exception()]
        for failed_task in failed_tasks:
            logger.exception(failed_task[1], exc_info=failed_task[0].exception())

    return tasks_responses


import numpy as np
import matplotlib.pyplot as plt
import torch
import torch.nn.functional as F

from diayn_cont.post_epoch_funcs.df_env_eval import DfEnvEvaluationDIAYNCont

import rlkit.torch.pytorch_util as ptu
from rlkit.samplers.data_collector.path_collector import MdpPathCollector

import self_supervised.utils.my_pytorch_util as my_ptu

from seqwise_cont_skillspace.utils.get_colors import get_colors

class DfEnvEvaluationDIAYN(DfEnvEvaluationDIAYNCont):

    def __init__(self,
                 *args,
                 skill_dim,
                 **kwargs,
                 ):
        super().__init__(*args, **kwargs)
        self.skill_dim = skill_dim

    def collect_skill_influence_paths(self) -> dict:
        assert isinstance(self.seq_collector, MdpPathCollector)

        skill_array = [number for number in range(self.seq_collector._policy.skill_dim)]
        skill_ids = []
        skills = []
        for skill_id, skill in enumerate(skill_array):
            self.seq_collector._policy.skill = skill
            self.seq_collector.collect_new_paths(
                max_path_length=self.seq_len,
                num_steps=self.seq_len,
                discard_incomplete_paths=False
            )
            skills.append(skill)
            skill_ids.append(skill_id)
        skill_influence_paths = self.seq_collector.get_epoch_paths()
        skill_influence_paths = list(skill_influence_paths)
        assert isinstance(skill_influence_paths, list)

        for skill_id, skill, path in zip(skill_ids, skills, skill_influence_paths):
            path['skill_id'] = skill_id
            path['skill'] = skill
        self._check_skill_influence_paths(skill_influence_paths)

        skill_influence_paths = self._stack_paths(skill_influence_paths)
        return skill_influence_paths

    @torch.no_grad()
    def apply_df(
            self,
            *args,
            next_observations,
            **kwargs
    ) -> dict:
        next_observations = ptu.from_numpy(next_observations)
        skill_recon = my_ptu.eval(self.df_to_evaluate, next_observations)
        ret_dict = dict(skill_recon=skill_recon)

        return ret_dict

    def plot_posterior(
            self,
            *args,
            epoch,
            skill_recon,
            skill_id,
            skill,
            **kwargs
    ):
        pass

    def classifier_evaluation(
            self,
            *args,
            epoch,
            skill_recon,
            skill,
            **kwargs
    ):
        #skills_np = np.array([np.array([_skill] * self.seq_len) for _skill in skill])
        assert isinstance(skill, list)
        assert skill_recon.shape[:-1] == torch.Size((len(skill), self.seq_len))
        skill_recon_reshaped = skill_recon.reshape(len(skill) * self.seq_len, -1)
        assert my_ptu.tensor_equality(skill_recon_reshaped[:self.seq_len], skill_recon[0,])
        skills = torch.stack([torch.tensor(skill)] * self.seq_len, dim=-1).reshape(len(skill) * self.seq_len)

        df_accuracy_eval = F.cross_entropy(skill_recon_reshaped.cpu(), skills.cpu())

        self.diagno_writer.writer.writer.add_scalar(
            tag=self.get_log_string("Classifier Performance/Eval"),
            scalar_value=df_accuracy_eval,
            global_step=epoch,
        )


import re, os
import pandas as pd
import numpy as np
from nltk.tokenize import word_tokenize
from nltk import pos_tag
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from sklearn.preprocessing import LabelEncoder
from collections import defaultdict
from nltk.corpus import wordnet as wn
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import model_selection, naive_bayes, svm
from sklearn.metrics import accuracy_score
from textclassify import Prediction
from nltk.stem import PorterStemmer
from visual import Visual

class Process:
    def __init__(self, pth, sf, sw, stem):
        self.path = pth
        self.singleFile = sf
        self.sw = sw
        self.stem = stem

    def cleanSW(self, words):
        sw = []
        swords = open("sw.txt", 'r')
        swords = swords.read()
        for word in swords.split():
            word =  word.strip()
            sw.append(word)
        out = [word for word in words.split() if not word.lower() in sw]
        output = " ".join(out)
        return output

    def cleanPunct(self, words):
        words = re.sub(r'[^\x00-\x7f]',r'', words) # removes garbage chars
        return re.sub(r'[^\w\s]', '', words) # removes punctuation

    def cleanNums(self, words):
        return re.sub(r'\d+', 'num', words)

    def stemWords(self, words):
        porter = PorterStemmer()
        out = ""
        for word in words.split():
            out = out + porter.stem(word) + ' '
        return out

    def cleanContents(self, content):
        if self.sw:
            content = self.cleanSW(content)
        if self.stem:
            content = self.stemWords(content)

        content = self.cleanNums(content)
        content = self.cleanPunct(content.lower())

        return content

    def preProcess(self):
        outputStr = "text,date\n"
        if self.singleFile:
            content = open(self.path, "r", encoding = "latin-1")
            link = content.readline()
            date = content.readline()
            article = content.read().replace('\n', '')
            article = self.cleanContents(article, self.sw)

            outputStr += article + "," + date
        else:
            for f in os.listdir(self.path):
                content = open(self.path + f, "r", encoding = "latin-1")
                link= content.readline()
                date=content.readline()
                article = content.read().replace('\n', '')

                article = self.cleanContents(article)

                outputStr += article + "," + date

        tempFile = open("temp-data.csv", 'w')
        tempFile.write(outputStr)
        tempFile.close()

    def process(self):
        predictor = Prediction("temp-data.csv")
        predictor.predict()

    def visualize(self, o):
        if o == 1:
            os.system("bokeh serve visual.py")
        elif o == 2:
            os.system("bokeh serve visual2.py")

path = os.path.dirname(os.path.abspath(__file__)) + '/separated'
test = Process(r'/Users/ryanhebert/Desktop/Research/Data/CNN/TweetsHK/', False, True, True)
test.preProcess()
test.process()
test.visualize(1)


#Libs
import matplotlib.pyplot as plt
import cv2
import copy

#Variables
#Graphics
data = [-3,-2,-1,0,1,2,3]
print(data)

plt.stem(data)
plt.show()

data2 = [-20,-19,-18,-17,-16,-15,-14,-13,-12,-11,-10,-9,-8,-7,-6,-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20]
plt.stem(data2)
plt.show()

#Cv2
image_example = cv2.imread('images/bmw.jpg')
cv2.imshow("BMW",image_example)
cv2.waitKey(0)

print('Dados da imagem colorida:')
print(image_example.shape)

image_gray_example = cv2.imread("images/bmw.jpg", 0)
cv2.imshow("Gray BMW",image_gray_example)
cv2.waitKey(0)

print("Dados da imagem em tons de cinza: ")
print(image_gray_example.shape)

image_2_example = cv2.imread("images/renault_megane_st.jpeg")
image_show = cv2.imshow("Renault Megane", image_2_example)
cv2.waitKey(0)

image_2_example_blue = copy.copy(image_2_example)

image_2_example_blue[:,:,1] = 0
image_2_example_blue[:,:,2] = 0
cv2.imshow("Blue Megane",image_2_example_blue)
cv2.waitKey(0)

image_2_example_green = copy.copy(image_2_example)

image_2_example_green[:,:,0] = 0
image_2_example_green[:,:,2] = 0
cv2.imshow("Green Megane",image_2_example_green)
cv2.waitKey(0)

image_2_example_red = copy.copy(image_2_example)

image_2_example_red[:,:,0] = 0
image_2_example_red[:,:,1] = 0
cv2.imshow("Red Megane",image_2_example_red)
cv2.waitKey(0)

image_3_example_jimny = cv2.imread("images/suzuki_jimny.jpg")
cv2.imshow("Suzuki Jimny", image_3_example_jimny)
cv2.waitKey(0)

image_3_example_convertida_jimny = cv2.cvtColor(image_3_example_jimny, cv2.COLOR_BGR2GRAY)
cv2.imshow("Gray Suzuki Jimny",image_3_example_convertida_jimny)
cv2.waitKey(0)

image_example_jimny_convertida_HSV = cv2.cvtColor(image_3_example_jimny, cv2.COLOR_BGR2HSV)
cv2.imshow("HSV Suzuki Jimny", image_example_jimny_convertida_HSV)
cv2.waitKey(0)

image_example_jimny_convertida_RGB = cv2.cvtColor(image_3_example_jimny, cv2.COLOR_BGR2RGB)
cv2.imshow("RGB Suzuki Jimny",image_example_jimny_convertida_RGB)
cv2.waitKey(0)

image_example_megane_convertida_RGB = cv2.cvtColor(image_2_example, cv2.COLOR_BGR2RGB)
cv2.imshow("RGB Renault Megane",image_example_megane_convertida_RGB)
cv2.waitKey(0)

image_Sobel_example_Gray = cv2.cvtColor(image_2_example, cv2.COLOR_BGR2GRAY)
print('Imagem Original')
cv2.imshow("Renault Megane",image_Sobel_example_Gray)
print('\n\n')
cv2.waitKey(0)

gX = cv2.Sobel(image_Sobel_example_Gray, ddepth=cv2.CV_64F, dx=1, dy=0, ksize=3) #ksize: tamanho da matriz de vizinhaça dos bits
gY = cv2.Sobel(image_Sobel_example_Gray, ddepth=cv2.CV_64F, dx=0, dy=1, ksize=3) #o ddepth converte a imagem para outro formato

#Converter a imagem novamente para o formato 8 bits
gX = cv2.convertScaleAbs(gX)
gY = cv2.convertScaleAbs(gY)

#Juntando os gradientes em X e Y com o mesmo grau de importância
Sobel_grads_combined = cv2.addWeighted(gX, 0.5, gY, -.5, 0)

cv2.imshow("Gradiente de Sobel em X", gX)
cv2.waitKey(0)
cv2.imshow("Gradiente de Sobel em Y", gY)
cv2.waitKey(0)
cv2.imshow("Gradientes de Sobel em X e Y combinados", Sobel_grads_combined)
cv2.waitKey(0)

image_Canny_example_Gray = cv2.cvtColor(image_2_example, cv2.COLOR_BGR2GRAY)

Canny_1 = cv2.Canny(image_Canny_example_Gray, 10,50)
Canny_2 = cv2.Canny(image_Canny_example_Gray, 150,400)

cv2.imshow("Imagem Original", image_Canny_example_Gray)
cv2.waitKey(0)
cv2.imshow("Imagem 1 por Canny", Canny_1)
cv2.waitKey(0)
cv2.imshow("Imagem 2 por Canny", Canny_2)
cv2.waitKey(0)

image_Laplaciano_example_Gray = cv2.cvtColor(image_2_example, cv2.COLOR_BGR2GRAY)
cv2.imshow("Imagem original", image_Laplaciano_example_Gray)
cv2.waitKey(0)

laplaciano_image = cv2.Laplacian(image_Laplaciano_example_Gray, cv2.CV_64F, ksize=3)

laplaciano_image_abs = cv2.convertScaleAbs(laplaciano_image)

cv2.imshow("Imagem após a aplicação do laplaciano", laplaciano_image_abs)
cv2.waitKey(0)


#!/usr/bin/python3
import yaml
import json
import fileinput
import os

# ET CORE DOCKER COMPOSE FILES (ETM, EDM, EIM, ESM, EMP and EPM)
emp = '../emp/deploy/docker-compose.yml'
edm = '../edm/deploy/docker-compose.yml'
edm_lite = '../edm/deploy/docker-compose-lite.yml'
esm = '../esm/deploy/docker-compose.yml'
eim = '../eim/deploy/docker-compose.yml'
epm = '../epm/deploy/docker-compose.yml'

etm_complementary_singlenode = '../etm/deploy/docker-compose-complementary.yml'
etm_main_singlenode = '../etm/deploy/docker-compose-main.yml'
platform_services = '../platform-services/docker-compose.yml'

etm_complementary_lite = '../etm/docker/docker-compose-complementary.yml'
mysql_elasticsearch_lite = '../docker-compose-mysql-elasticsearch-lite.yml'
etm_main_lite = '../etm/docker/docker-compose-main.yml'
eim_in_etm = '../etm/elastest-torm/src/main/resources/test_engines/eim.yml'

etm_proxy = '../etm/docker/docker-compose-proxy.yml'

# ET TSS JSON FILES (EUS, EBS, ESS, EDS, EMS)
eus = '../eus/elastestservice.json'
ebs = '../ebs/elastestservice.json'
ess = '../ess/elastestservice.json'
eds = '../eds/elastestservice.json'
ems = '../ems/elastestservice.json'

# ET Test Engines DOCKER COMPOSE FILES (ECE and ERE)
ece = '../etm/elastest-torm/src/main/resources/test_engines/ece.yml'
ere = '../etm/elastest-torm/src/main/resources/test_engines/ere.yml'

# EUS Browsers file
eus_browsers_properties_file = '../eus/browsers/docker-browser.properties'
eus_properties_file = '../eus/application.properties'
etm_properties_file = '../etm/application.properties'

# Images to pull at the start
imagesFilesToPrePull = {'eusBrowsers': eus_browsers_properties_file,
                        'etmSocat': etm_properties_file,
                        'eus': eus}

tss_images_files = {'eus': eus, 'ebs': ebs, 'ess': ess, 'eds': eds, 'ems': ems}

def getCoreList():
    core_list = [emp, edm_lite, esm, eim, eim_in_etm, epm, etm_complementary_singlenode, platform_services,
                 etm_main_singlenode, etm_complementary_lite, etm_main_lite, etm_proxy]
    return core_list

def getCoreListByExecMode(mode):
    # Mini
    if (mode == 'mini'):
        core_list = [etm_complementary_lite,
                     etm_main_lite, etm_proxy, platform_services]
    # singlenode
    else:
        core_list = [emp, edm_lite, esm, eim, epm, etm_complementary_singlenode,
                     etm_main_singlenode, etm_proxy, platform_services]
    return core_list

def getTSSFile(tss):
    return tss_images_files[tss]

def getTSSList():
    tss_list = [eus, ebs, ess, eds, ems]
    return tss_list

def getEnginesList():
    engines_list = [ece, ere]
    return engines_list

def getFilePathByImage(imageToPull):
    return imagesFilesToPrePull[imageToPull]

# Yaml

def getYml(path):
    with open(path, 'r') as stream:
        try:
            return yaml.load(stream)
        except yaml.YAMLError as exc:
            return yaml.load('')

def saveYml(path, yml):
    with open(path, 'w') as outfile:
        yaml.dump(yml, outfile, default_flow_style=False)

# Json

def getJson(path):
    with open(path, 'r') as stream:
        data = json.load(stream)
        return data
    return json.load('')

def saveJson(path, json_file):
    with open(path, 'w') as outfile:
        json.dump(json_file, outfile, indent=4, separators=(',', ': '))

def getLineByContent(content, path):
    result = ''
    for line in fileinput.input(path):
        if(content in line):
            result = line
    return result

def checkIfDirExists(path):
    return os.path.exists(path)

def createDir(path):
    os.makedirs(path)

def checkIfFileExists(path):
    return os.path.isfile(path)

def readFileByLines(path, linesToRead):
    myfile = open(path)
    with myfile:
        head = [next(myfile) for x in xrange(linesToRead)]
    myfile.close()
    return head

def writeFile(path, content):
    file = open(path, 'a')
    file.write(content + '\n')
    file.close()


"""
Starlette Exception handlers for the application.
There is a default handler for 403, 404 and 500 errors.
"""

from starlette.exceptions import HTTPException
from starlette.requests import Request
from stadsarkiv_client.core.templates import templates
from stadsarkiv_client.core.context import get_context
from stadsarkiv_client.core.translate import translate
from stadsarkiv_client.core.logging import get_log

HTML_403_PAGE = "403"
HTML_404_PAGE = "404"
HTML_500_PAGE = "500"

log = get_log()

async def not_found(request: Request, exc: HTTPException):
    context_values = {"title": translate("404 Not Found")}

    # No need to log full exception. It's a 404
    log.error(f"404 Not Found: {request.url}")
    context = await get_context(request, context_values=context_values)
    return templates.TemplateResponse("errors/default.html", context, status_code=404)

async def server_error(request: Request, exc: HTTPException):
    context_values = {"title": translate("500 Server Error")}

    log.exception(f"500 Error: {request.url}", exc_info=exc)
    context = await get_context(request, context_values=context_values)
    return templates.TemplateResponse("errors/default.html", context, status_code=500)

async def forbidden_error(request: Request, exc: HTTPException):
    context_values = {"title": translate("403 Forbidden Error")}

    log.error(f"403 Forbidden: {request.url}", exc_info=exc)
    context = await get_context(request, context_values=context_values)
    return templates.TemplateResponse("errors/default.html", context, status_code=403)

exception_handlers = {403: forbidden_error, 404: not_found, 500: server_error}


import sys
from collections import deque
input = sys.stdin.readline

n, k = map(int, input().split())
color = [[2 for _ in range(n+2)]]
for _ in range(n):
    color.append([2] + list(map(int, input().split())) + [2])
color.append([2 for _ in range(n+2)])

board = [[deque() for _ in range(n+2)] for _ in range(n+2)]
marker = []
dir = []
for i in range(k):
    y, x, d = map(int, input().split())
    board[y][x].append(i)
    marker.append([y, x])
    dir.append(d - 1)

def opposite(k):
    if k == 0:
        return 1
    elif k == 1:
        return 0
    elif k == 2:
        return 3
    else:
        return 2

# idx 번째 marker를 이동시킴
def move(idx):
    tmp = deque()
    dy = [0, 0, -1, 1]
    dx = [1, -1, 0, 0]
    y, x = marker[idx]
    d = dir[idx]

    ny, nx = y + dy[d], x + dx[d]

    if color[ny][nx] == 0: # 흰색
        # 현재 board의 모든 말을 순서를 유지한 채(pop -> appendleft) tmp로 이동시킴
        while board[y][x]:
            popped = board[y][x].pop()
            tmp.appendleft(popped)
            # 각 marker의 현재 위치 갱신
            marker[popped] = [ny, nx]
        # tmp를 이동할 위치에 이어 붙임
        board[ny][nx] += tmp

        if len(board[ny][nx]) >= 4:
            return True

    elif color[ny][nx] == 1: # 빨간색
        # 현재 board의 모든 말을 순서를 반전시킨 채(pop -> append) tmp로 이동시킴
        while board[y][x]:
            popped = board[y][x].pop()
            tmp.append(popped)
            marker[popped] = [ny, nx]
        board[ny][nx] += tmp

        if len(board[ny][nx]) >= 4:
            return True

    else: # 파란색
        # 일단 방향을 바꿈
        dir[idx] = opposite(d)
        nny, nnx = y + dy[opposite(d)], x + dx[opposite(d)]
        # 바꾼 방향의 칸이 파란색이 아니면 이동
        if color[nny][nnx] != 2:
            return move(idx)

    return False

# move_marker: 모든 marker를 이동, 단 각 칸의 맨 아래에 있지 않은 marker는 이동하지 않음
def move_marker():
    for i in range(k):
        my, mx = marker[i]
        if board[my][mx].index(i) == 0:
            if move(i):
                return True

# 턴이 지날 때마다 turn 1씩 증가
turn = 1
while True:
    if turn > 1000:
        print(-1)
        break

    if move_marker():
        print(turn)
        break

    turn += 1


# http://www.pspad.com/cz/download.htm
# http://szj.cz/seznam-ceskych-podstatnych-jmen/ -> https://pastebin.com/7uPetKWh

import random
import re
import datetime
import firebase_admin
from firebase_admin import credentials, firestore

types = {}
guess_types = ['DEFAULT', 'ON', 'SN', 'BN', 'C',
               'YN', 'YR', 'YKR', 'PVZQ', 'other', 'LR']
word_types = ['H', 'ZQ', 'Z', 'YKRN', 'U', 'S', 'ATN', 'PI', 'MQ', 'JTN', 'Q',
              'PIV', 'YRN', 'AN', 'JN', 'K', 'IN', 'HR', 'MQR', 'YKRON', 'BTN', 'ACN', 'SN', 'BN', 'C']
guesses = []
words = []

def add_files(file_name, encoding='utf-8'):
    global guesses
    f = open(file_name, 'r', encoding=encoding)
    for line in f:
        word = line.strip()
        if len(word) == 5:
            if word not in guesses:
                guesses.append(word)
    f.close()

def get_previous_words(db):
    lastDate = datetime.datetime(2030, 5, 1)
    doc_ref = db.collection(u'word').where(
        u'exactTimeStamp', u'<=', lastDate).stream()
    return set([item.to_dict()['solution'].upper() for item in doc_ref])

if __name__ == '__main__':

    # Use a service account
    cred = credentials.Certificate(
        'data_calculate/wordle-cz-firebase-adminsdk-q7ldn-01d406365d.json')
    firebase_admin.initialize_app(cred)

    db = firestore.client()

    previous_words = get_previous_words(db)
    print(previous_words)

    f = open('data_prepare/input/cs_CZ.dic', 'r', encoding='utf-8')
    for line in f:
        word, word_type = line.strip(), 'DEFAULT'
        if '/' in word:
            (word, word_type) = word.split('/')
        if word_type in ['?']:
            word_type = 'other'
        if len(word) == 5 and re.fullmatch('[a-zěščřžýáíéů]+', word.lower()):
            types.setdefault(word_type, []).append(word)
            if word_type in word_types:
                words.append(word)
            if word_type in word_types or word_type in guess_types:
                guesses.append(word)
    f.close()

    add_files('data_prepare/input/Česká podstatná jména.txt')
    add_files('data_prepare/input/Czech.3-2-5.dic', 'windows-1250')

    TOTAL_EXPORT = 0

    for word_type in types:
        # print(word_type, len(types[word_type]))
        if (len(types[word_type]) > 20):
            fw = open('data_prepare/output/.exploration_{}'.format(word_type),
                      'w', encoding='utf-8')
            fw.write('\n'.join(types[word_type]))
            TOTAL_EXPORT += len(types[word_type])
            fw.close()

    words = list(set(word.upper()
                 for word in words if word.upper() not in previous_words))
    random.shuffle(words)
    random.shuffle(words)
    random.shuffle(words)
    random.shuffle(words)

    # fw = open('data_prepare/output/wordlist.ts', 'w', encoding='utf-8')
    # fw.write('export const WORDS = [')
    # fw.write(''.join('"{}",\n'.format(w.lower()) for w in words))
    # fw.write('];')
    # fw.close()

    fw = open('data_prepare/output/validGuesses.ts', 'w', encoding='utf-8')
    fw.write('export const VALIDGUESSES = [')
    fw.write(''.join('"{}",\n'.format(w.lower()) for w in guesses))
    fw.write('];')
    fw.close()

    fw = open('data_prepare/input/final_words_new2.txt', 'w', encoding='utf-8')
    fw.write('\n'.join(w.upper() for w in words))
    fw.close

    print("TOTAL_EXPORT ", TOTAL_EXPORT)


#euler 348

import math
import sys
from tqdm import tqdm

def is_square(integer):
    root = math.sqrt(integer)
    return integer == int(root + 0.5) ** 2

def is_cube(n):
    cube_root = n**(1./3.)
    return round(cube_root) ** 3 == n

def s_and_c(n_list):
	n1 = n_list[0]
	n2 = n_list[1]

	if is_square(n1):
		if is_cube(n2):
			return True
		else:
			return False
	elif is_cube(n1):
		if is_square(n2):
			return True
		else:
			return False
	else:
		return False

def pal(n):
	number = list(str(n))
	#number = n_list

	half = len(number) / 2
	i = 0
	last = len(number) - 1

	if len(number) % 2 == 0:
		end = half
	else:
		end = math.floor(half)

	while i < end:
		if number[i] == number[last]:
			i += 1
			last -= 1
		else:
			return False
	return True

def gen_halves(n):
	halfway = math.ceil(n/2) + 1
	for i in range(2, halfway):
		yield (i, (n-i))

all_nums = {}

s_min = 2
s_max = 50000
c_min = 2
c_max = 25000

print("Generating numbers")

squares = []
for i in range(s_min, s_max + 1):
	squares.append(i*i)

cubes = []
for j in range(c_min, c_max + 1):
	cubes.append(j*j*j)

print("Checking")

results = []

for s in tqdm(squares):

	for c in cubes:

		num = s + c
		if pal(num):

			if num not in all_nums:
				all_nums[num] = 1
			else:
				all_nums[num] += 1

				if all_nums[num] == 4:
					#print("Winner: " + str(num))
					results.append(num)

print(results)
print(sum(results))


import threading
import concurrent.futures
import multiprocessing

def f():
    pass

t1 = threading.Thread(target=f)
t1.start()
t1.join()

with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
    future = executor.submit(f)
if __name__ == '__main__':
    p1 = multiprocessing.Process(target=f)
    p1.start()
    p1.join()

    with concurrent.futures.ProcessPoolExecutor() as executor:
        executor.submit(f)

""" thread and process share almost same methods, high level object like Lock, RLock, Event, Semaphore, Condition, Barrier, Timer
Lock:
has two states as unlocked and locked.
acquire will blocked on locked until other thread release to unlocked, return immediately on unlocked and release.
Lock can't release on unlocked, will raise error, and one thread can release other thread's lock,
When release, only one of other thread will acquire the lock if multiple threads are acquiring.

RLock, can recursively lock acquire on locked state when owned by thread itself, will increase recursive count by one. release will reduce count by one.
And thread can only release the lock owned by itself, can't release another thread's lock.

Event: like the lock, it has internal flag true and false. set() to true, clear() to false. Once set(), it will change all the wait() thread to free, not like lock only one thread.

Semaphore: has a internal atomic counter as the limits of the number of thread can pass at the same time in acquire(),
which when acquire will reduce by one and when release will increase by one. It is very helpful to control a number of threads can do something first.

Condition: has a internal lock, but after acquire the internal lock, it can wait() for other threads to notify, when wait() it release the lock and wait for the notify.
The thread acquire the lock can notify others to wake up from wait() to acquire the lock.

Barrier: it is a gate which how many threads need to acquire this barrier before them can pass. After passing, barrier will reset to zero.

"""

barrier = threading.Barrier(3)

class thread(threading.Thread):
    def __init__(self, thread_ID):
        threading.Thread.__init__(self)
        self.thread_ID = thread_ID
    def run(self):
        print(str(self.thread_ID) + "\n")
        print("Parties = " + str(barrier.parties) + "\n")
        print("n_waiting = " + str(barrier.n_waiting) + "\n")
        barrier.wait()

thread1 = thread(100)
thread2 = thread(101)

thread1.start()
thread2.start()

thread5 = thread(104)
thread5.start()

#print(str(barrier.broken) + "\n")
print("n_waiting after free = " + str(barrier.n_waiting))
print("parties after free = " + str(barrier.parties))
#barrier.reset()
print("n_waiting after reset = " + str(barrier.n_waiting))
thread3 = thread(102)
thread3.start()
thread4 = thread(103)
thread4.start()
thread6 = thread(105)
thread6.start()

print("End")


import sys
import logging
import boto3

sys.path.insert(0, '.serverless/requirements')
from session.cognito import IdentityPool
from util.exceptions import NotAuthorizedException
from util.extract import callback_url, authorization_token
from config.environment_config import EnvironmentConfig
from data.s3_client import S3
from session.user import User
from config.logging_config import LoggingConfig
from web import render

log           = None  # type: logging.Logger
s3_client     = None  # type: S3
user          = None  # type: User
identity_pool = None  # type: IdentityPool
env           = EnvironmentConfig()

def setup():
    global log
    global s3_client
    global user
    global identity_pool

    print("setting up")
    LoggingConfig()
    log = logging.getLogger()
    log.info("logger set")
    log.setLevel(logging.INFO)
    log.info(f"Config: {repr(env)}")

    boto3.setup_default_session(aws_access_key_id=env.aws_key, aws_secret_access_key=env.aws_secret, region_name=env.region)
    log.info("boto session is set")

    s3_client = S3(env.bucket_name)
    log.info("s3_client is set")
    user = User(env.cognito_client_id, env.cognito_secret, env.cognito_region)
    log.info("user class set")
    identity_pool = IdentityPool(env.cognito_pool_id, env.cognito_client_id, env.cognito_region)
    log.info("identity_pool set")

def index(event, context):
    try:
        return dict(
            statusCode=200,
            body=render.index(),
            headers={'Content-Type': 'text/html; charset=utf-8'}
        )
    except Exception as e:
        return dict(
            statusCode=500,
            body=str(e)
        )

def edit(event, context):
    try:
        setup()
        print(event)
        query_params = event.get('queryStringParameters', {})
        access_token = authorization_token(event)

        if query_params and query_params.get('code') and query_params.get('state'):
            access_code  = query_params.get('code')
            access_state = query_params.get('state')
            redirect_uri = callback_url(event)
            user.start_session(identity_pool.token_url, access_code, redirect_uri, access_state)
            access_token = user.access_token
        elif access_token:
            user.verify(access_token)
        else:
            raise NotAuthorizedException

        return dict(
            statusCode=200,
            body="Fuck yeah",
            headers={"Authorization": f'Bearer {access_token}'}
        )
    except NotAuthorizedException as e:
        log.exception(e)
        return dict(
            statusCode=302,
            body="Not Authorized",
            headers={"Location": identity_pool.authorize_url}
        )
    except Exception as e:
        log.exception(e)

# event = {'body': None,
#  'headers': {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
#              'Authorization': 'Bearer ssFBMaebIVeFcumCUUvELF1wzaNVJRQ1XA5IalKHc1498FshEe1TWyLkdM16MM3w',
#              'Accept-Encoding': 'gzip, deflate, br',
#              'Accept-Language': 'en-US,en;q=0.5',
#              'CloudFront-Forwarded-Proto': 'https',
#              'CloudFront-Is-Desktop-Viewer': 'true',
#              'CloudFront-Is-Mobile-Viewer': 'false',
#              'CloudFront-Is-SmartTV-Viewer': 'false',
#              'CloudFront-Is-Tablet-Viewer': 'false',
#              'CloudFront-Viewer-Country': 'US',
#              'Host': '0pi.us-west-2.amazonaws.com',
#              'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; '
#                            'rv:57.0) Gecko/20100101 Firefox/57.0',
#              'Via': '2.0 05c830b46.cloudfront.net '
#                     '(CloudFront)',
#              'X-Amz-Cf-Id': 'kYcLP2KNDxNLVqw5ub4UdM',
#              'X-Amzn-Trace-Id': 'Root=1-e63464347ab1fe0c514ae0',
#              'X-Forwarded-For': '174.65.23.114, 216.137.44.11',
#              'X-Forwarded-Port': '443',
#              'X-Forwarded-Proto': 'https',
#              'cache-control': 'no-cache',
#              'pragma': 'no-cache',
#              'upgrade-insecure-requests': '1'},
#  'httpMethod': 'GET',
#  'isBase64Encoded': False,
#  'path': '/edit',
#  'pathParameters': None,
#  'queryStringParameters': {},
#  'requestContext': {'accountId': '428',
#                     'apiId': '0eiv1z65i4',
#                     'httpMethod': 'GET',
#                     'identity': {'accessKey': None,
#                                  'accountId': None,
#                                  'caller': None,
#                                  'cognitoAuthenticationProvider': None,
#                                  'cognitoAuthenticationType': None,
#                                  'cognitoIdentityId': None,
#                                  'cognitoIdentityPoolId': None,
#                                  'sourceIp': '174.65.23.114',
#                                  'user': None,
#                                  'userAgent': 'Mozilla/5.0 (Macintosh; Intel '
#                                               'Mac OS X 10.13; rv:57.0) '
#                                               'Gecko/20100101 Firefox/57.0',
#                                  'userArn': None},
#                     'path': '/dev/edit',
#                     'protocol': 'HTTP/1.1',
#                     'requestId': 'e14210b1-2720-11e8-9df4-0d105233d7ed',
#                     'requestTime': '14/Mar/2018:00:44:40 +0000',
#                     'requestTimeEpoch': 1520988280644,
#                     'resourceId': 'glglnt',
#                     'resourcePath': '/edit',
#                     'stage': 'dev'},
#  'resource': '/edit',
#  'stageVariables': None}

# print(pprint(event))
# edit(event, None)


import pandas as pd
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler

# QUESTION 1
df_full = pd.read_csv( "specs/SensorData_question1.csv" )

"""
1. Generate a new attribute called Original Input3 which is a copy of the
attribute Input3. Do the same with the attribute Input12 and copy it into
Original Input12
"""
df_full['Original Input3'] = df_full['Input3']
df_full['Original Input12'] = df_full['Input12']

"""
2. Normalise the attribute Input3 using the z-score transformation method
"""
#Z-Score= (X - Mean) / Standard Dev
Input3_Mean= df_full['Input3'].mean()
Input3_STD = df_full['Input3'].std()

df_full['Input3'] = (df_full['Input3']-Input3_Mean)/Input3_STD

"""
3. Normalise the attribute Input12 in the range [0.0, 1.0].
"""
#Normalized Data = (x-min(x))/(max(x)-min(x))
Input12_Min = df_full['Input12'].min()
Input12_Max = df_full['Input12'].max()

df_full['Input12'] = (df_full['Input12'] - Input12_Min)/(Input12_Max-Input12_Min)

"""
4. Generate a new attribute called Average Input, which is the average of all
the attributes from Input1 to Input12. This average should include the
normalised attributes values but not the copies that were made of these.
"""
df_full.loc[:,'Average Input'] = df_full.drop(['Original Input3','Original Input12'], axis=1).sum(axis=1)/len(df_full.drop(['Original Input3','Original Input12'], axis=1).columns)

"""
5. Save the newly generated dataset to ./output/question1 out.csv.
"""
df_full.to_csv('output/question1_out.csv', index=False)

# QUESTION 2
"""
1. Reduce the number of attributes using Principal Component Analysis
(PCA), making sure at least 95% of all the variance is explained.
"""
df_full2 = pd.read_csv( "specs/DNAData_question2.csv" )
# Get headings of columns
col_head= df_full2.columns.values
# Get values of each column
x = df_full2.loc[:, col_head].values
print(x.shape)
# normalize the columns
x = StandardScaler().fit_transform(x)
print(x.shape)
# Get the header of each columm that is equal to the shape of X on axis 1 (column header)
header_cols = [col_head[i] for i in range(x.shape[1])]
# Store the normalised data in a new df
normalised_data = pd.DataFrame(x,columns=header_cols)
print(normalised_data.shape)
# Allow PCA to determine the minimum number of components required to have a 95% explained variance
pca_data = PCA(.95)
# Z-Score the PCA data using x, the standardized data
PC_data = pca_data.fit_transform(x)
# Convert PC_data into a dataframe
PCA_DF = pd.DataFrame(data = PC_data)
# Print the resulting explained variance for each column
#---print('Explained Variance: {}'.format(pca_data.explained_variance_ratio_))

"""
2. Discretise the PCA-generated attribute subset into 10 bins, using bins of
equal width. For each component X that you discretise, generate a new
column in the original dataset named pcaX width. For example, the first
discretised principal component will correspond to a new column called
pca1 width.
"""

# For each 'new' attribute of PCA, do the following (skipping index column)
for i in range(1,len(PCA_DF.columns)):
    # store the values of each column in col_values
    col_values = PCA_DF[i].values
    # bin each column into equal widths of 10 depending on that columns values
    bins = pd.cut(col_values, 10)
    # Get the value of each row entry we are working on (i.e value by value of the column)
    labels = bins.codes
    # Get each bin range of the current value
    bin_range = bins.categories
    # create a blank series
    pca_width_values = pd.Series([])
    # for each value in the column get its bin label/range
    for j in range(len(col_values)):
        # get the label of the current bin range
        bin_label = labels[j]
        # create a new column and add the bin label for each value
        pca_width_values[j] = bin_range[bin_label]
    # Create a new column for the new bin width labels
    df_full2['pca' + str(i) + '_width'] = pca_width_values

"""
3. Discretise PCA-generated attribute subset into 10 bins, using bins of equal
frequency (they should all contain the same number of points). For each
component X that you discretise, generate a new column in the original
1
dataset named pcaX freq. For example, the first discretised principal
component will correspond to a new column called pca1 width.
"""
# does the same as above but replace cut with qcut for quartile cutting, bin set to 10.
for i in range(1,len(PCA_DF.columns)):
    col_values = PCA_DF[i].values
    bins2 = pd.qcut(col_values, 10)
    labels2 = bins2.codes
    bin_range2 = bins2.categories
    pca_width_values2 = pd.Series([])
    for j in range(len(col_values)):
        bin_label2 = labels2[j]
        pca_width_values2[j] = bin_range2[bin_label2]
    df_full2['pca' + str(i) + '_freq'] = pca_width_values2

# Export to a CSV
df_full2.to_csv('output/question2_out.csv', index=False)

#print(pd.qcut(col_values, 10).value_counts())


"""
Support for Blue Iris.
For more details about this platform, please refer to the documentation at
https://home-assistant.io/components/switch.blueiris/
"""
from datetime import timedelta
import voluptuous as vol

from homeassistant.const import (
    CONF_HOST, CONF_PORT, CONF_PASSWORD,
    CONF_USERNAME, CONF_SSL,
    CONF_ID, CONF_NAME)

from homeassistant.components.binary_sensor import DOMAIN as DOMAIN_BINARY_SENSOR
from homeassistant.components.camera import DOMAIN as DOMAIN_CAMERA
from homeassistant.components.switch import DOMAIN as DOMAIN_SWITCH

from homeassistant.components.mqtt import (
    CONF_PAYLOAD_AVAILABLE, DEFAULT_PAYLOAD_AVAILABLE,
    CONF_QOS, CONF_PAYLOAD_NOT_AVAILABLE, DEFAULT_PAYLOAD_NOT_AVAILABLE,
    DEFAULT_QOS)

VERSION = '2.0.0'

DOMAIN = 'blueiris'
DATA_BLUEIRIS = f'data_{DOMAIN}'
DATA_BLUEIRIS_API = f'{DATA_BLUEIRIS}_API'
DATA_BLUEIRIS_HA = f'{DATA_BLUEIRIS}_HA'
DATA_BLUEIRIS_HA_ENTITIES = f'{DATA_BLUEIRIS}_HA_Entities'
DEFAULT_NAME = "BlueIris"
DEFAULT_PORT = 80

DOMAIN_KEY_FILE = f"{DOMAIN}.key"

ATTR_ADMIN_PROFILE = 'Profile'
ATTR_SYSTEM_CAMERA_ALL_NAME = 'All'
ATTR_SYSTEM_CAMERA_ALL_ID = 'Index'
ATTR_SYSTEM_CAMERA_CYCLE_NAME = 'Cycle'
ATTR_SYSTEM_CAMERA_CYCLE_ID = '@Index'

AUDIO_EVENT_LENGTH = 2
RECONNECT_DELAY = 15

BLUEIRIS_AUTH_ERROR = "Authorization required"

SYSTEM_CAMERA_CONFIG = {
    ATTR_SYSTEM_CAMERA_ALL_NAME: ATTR_SYSTEM_CAMERA_ALL_ID,
    ATTR_SYSTEM_CAMERA_CYCLE_NAME: ATTR_SYSTEM_CAMERA_CYCLE_ID
}

SYSTEM_CAMERA_ID = [
    ATTR_SYSTEM_CAMERA_ALL_ID,
    ATTR_SYSTEM_CAMERA_CYCLE_ID
]

MQTT_AVAILABILITY_CONFIG = {
    CONF_PAYLOAD_AVAILABLE: DEFAULT_PAYLOAD_AVAILABLE,
    CONF_PAYLOAD_NOT_AVAILABLE: DEFAULT_PAYLOAD_NOT_AVAILABLE,
    CONF_QOS: DEFAULT_QOS
}

AUTHENTICATION_BASIC = 'basic'

NOTIFICATION_ID = f'{DOMAIN}_notification'
NOTIFICATION_TITLE = f'{DEFAULT_NAME} Setup'

DEFAULT_ICON = 'mdi:alarm-light'

CAMERA_ID_PLACEHOLDER = '[camera_id]'

ATTR_FRIENDLY_NAME = 'friendly_name'

PROTOCOLS = {
    True: 'https',
    False: 'http'
}

SCAN_INTERVAL = timedelta(seconds=30)

DEFAULT_FORCE_UPDATE = False

SENSOR_CONNECTIVITY_NAME = 'Connectivity'
SENSOR_MOTION_NAME = 'Motion'
SENSOR_AUDIO_NAME = 'Audio'
SENSOR_MAIN_NAME = 'Main'

SENSOR_DEVICE_CLASS = {
    SENSOR_AUDIO_NAME: 'sound'
}

MQTT_MESSAGE_TRIGGER = 'trigger'
MQTT_MESSAGE_TYPE = 'type'
MQTT_MESSAGE_VALUE_UNKNOWN = 'unknown'

MQTT_ALL_TOPIC = "BlueIris/+/Status"

CONFIG_OPTIONS = 'options'
CONFIG_CONDITIONS = 'conditions'
CONFIG_ITEMS = 'items'

ATTR_BLUE_IRIS_CAMERA = {
    "optionDisplay": CONF_NAME,
    "optionValue": CONF_ID,
    "FPS": "FPS",
    "audio": "Audio support",
    "width": "Width",
    "height": "Height",
    "isOnline": "Is Online",
    "isRecording": "Is Recording",
    "isYellow": "Issue",
    "nAlerts": "Alerts #",
    "nTriggers": "Triggers #",
    "nClips": "Clips #",
    "nNoSignal": "No Signal #",
    "error": "Error"
}
ATTR_BLUE_IRIS_STATUS = [
    "system name",
    "version",
    "license",
    "support",
    "user",
    "latitude",
    "longitude"
]

BI_DISCOVERY = f"{DOMAIN}_discovery"
BI_DISCOVERY_BINARY_SENSOR = f"{BI_DISCOVERY}_{DOMAIN_BINARY_SENSOR}"
BI_DISCOVERY_CAMERA = f"{BI_DISCOVERY}_{DOMAIN_CAMERA}"
BI_DISCOVERY_SWITCH = f"{BI_DISCOVERY}_{DOMAIN_SWITCH}"

BI_UPDATE_SIGNAL_CAMERA = f"{DOMAIN}_{DOMAIN_CAMERA}_UPDATE_SIGNAL"
BI_UPDATE_SIGNAL_BINARY_SENSOR = f"{DOMAIN}_{DOMAIN_BINARY_SENSOR}_UPDATE_SIGNAL"
BI_UPDATE_SIGNAL_SWITCH = f"{DOMAIN}_{DOMAIN_SWITCH}_UPDATE_SIGNAL"

CONFIG_FIELDS = {
    vol.Required(CONF_HOST): str,
    vol.Required(CONF_PORT): int,
    vol.Optional(CONF_SSL, default=False): bool,
    vol.Optional(CONF_USERNAME): str,
    vol.Optional(CONF_PASSWORD): str,
}

SUPPORTED_DOMAINS = [DOMAIN_SWITCH, DOMAIN_BINARY_SENSOR, DOMAIN_CAMERA]
SIGNALS = {
    DOMAIN_BINARY_SENSOR: BI_UPDATE_SIGNAL_BINARY_SENSOR,
    DOMAIN_CAMERA: BI_UPDATE_SIGNAL_CAMERA,
    DOMAIN_SWITCH: BI_UPDATE_SIGNAL_SWITCH
}

ENTITY_ID = "id"
ENTITY_NAME = "name"
ENTITY_STATE = "state"
ENTITY_ATTRIBUTES = "attributes"
ENTITY_ICON = "icon"
ENTITY_UNIQUE_ID = "unique-id"
ENTITY_EVENT = "event-type"
ENTITY_TOPIC = "topic"
ENTITY_DEVICE_CLASS = "device-class"
ENTITY_DEVICE_NAME = "device-name"
ENTITY_CAMERA_DETAILS = "camera-details"
ENTITY_BINARY_SENSOR_TYPE = "binary-sensor-type"

CONF_EXCLUDE_SYSTEM_CAMERA = "exclude-system-camera"
CONF_CLEAR_CREDENTIALS = "clear-credentials"

DOMAIN_LOAD = "load"
DOMAIN_UNLOAD = "unload"


import numpy as np
from direction import Direction,Orientation
from shape import Shape
import itertools
import parameters
from scipy.optimize import minimize

def get_agents_coordinates(center, num_agents, hexa_type ="paper"):
    """
    Computes the correct coordinates of the agents, according to an hexagonal lattice.
    :param center: center of the world
    :param num_agents: number of coordinates to return
    :param hexa_type: one of {"paper", "rectangle"} to define the shape of the lattice
    :return: a list of the coordinates for all agents (including seeds)
    """
    c_x = center[0]
    c_y = center[1]

    hexa = np.array([[0, 0, 0], [1, -1, 0], [0, -1, 1], [1, 0, -1]])
    if hexa_type == "paper":
        hexa = np.array(
            [[0, 0, 0], [1, -1, 0], [0, -1, 1],  [1, 0, -1], [-1, -1, 2], [0, -2, 2], [-2, -1, 3],
             [-1, -2, 3], [0, -3, 3],[1, -4, 3], [-2, -2, 4], [-1, -3, 4], [0, -4, 4], [1, -5, 4],
             [-3, -2, 5], [-2, -3, 5], [-1, -4, 5], [0, -5, 5], [1, -6, 5], [-4, -2, 6], [-3, -3, 6],
             [-2, -4, 6], [-1, -5, 6], [0, -6, 6], [1, -7, 6], [-4, -3, 7]])
    elif hexa_type == "rectangle":
        hexa = np.vstack((np.array([[0, 0, 0], [1, -1, 0], [0, -1, 1], [1, 0, -1]]),
                          np.array([coord for coord in get_hexa_coord(20)])))

    cubic_hexa = hexa[:, [0, 2]]
    hex2cart = np.array([[np.sqrt(3), np.sqrt(3) / 2], [0, -3 / 2]])*1/np.sqrt(3)
    cart = np.dot(hex2cart, cubic_hexa.T)

    cart_sliced = cart[:,0: 4+num_agents].copy()
    cart_sliced[0, :] = cart_sliced[0, :] + c_x-0.5
    cart_sliced[1, :] = cart_sliced[1, :] + c_y
    coord = [(x, y) for x, y in cart_sliced.T]
    return coord

def get_deltas(dir, orn):
    '''
    :param dir: desired movement direction {Direction.FWD, Direction.FWD_CW, Direction.FWD_CCW}
    :param orn: current robot orientation
    :return: dx, dy = {-1,0,1},{-1,0,1},... : displacement direction required
    '''
    dx = 0
    dy = 0

    if dir == Direction.FWD:
        if orn in [Orientation.N, Orientation.NE, Orientation.NW]:
            dy = 1
        if orn in [Orientation.S, Orientation.SE, Orientation.SW]:
            dy = -1
        if orn in [Orientation.W, Orientation.NW, Orientation.SW]:
            dx = -1
        if orn in [Orientation.E, Orientation.NE, Orientation.SE]:
            dx = 1
    elif dir == Direction.FWD_CW:
        if orn in [Orientation.N, Orientation.W, Orientation.NW]:
            dy = 1
        if orn in [Orientation.S, Orientation.SE, Orientation.E]:
            dy = -1
        if orn in [Orientation.W, Orientation.S, Orientation.SW]:
            dx = -1
        if orn in [Orientation.E, Orientation.NE, Orientation.N]:
            dx = 1
    elif dir == Direction.FWD_CCW:
        if orn in [Orientation.N, Orientation.NE, Orientation.E]:
            dy = 1
        if orn in [Orientation.S, Orientation.W, Orientation.SW]:
            dy = -1
        if orn in [Orientation.W, Orientation.NW, Orientation.N]:
            dx = -1
        if orn in [Orientation.E, Orientation.S, Orientation.SE]:
            dx = 1
    else:
        raise NotImplementedError("get_deltas not implemented for dir : " + str(dir))
    factor = 1
    if abs(dx*dy) == 1:
        # the robot is going on a diagonal axis => performing sqrt(2) instead of 1 in a step
        factor = 1/np.sqrt(2)
    dx *= factor
    dy *= factor

    return dx, dy

def at_least_three_non_colinear(list_of_points):
    """
    :param list_of_points:
    :return: True if at least three point of the list_of_points are non colinear. False otherwise
    """
    # flag = 0
    for p0, p1, p2 in itertools.combinations(list_of_points, 3):
        if not collinear(p0, p1, p2):
            return True
    return False

def collinear(p0, p1, p2):
    """
    :param p0, p1, p2: triplets of coordinates
    :return: True if the triplets p0, p1 and p2 are colinear. It uses the determinant method
    """
    x1, y1 = p1[0] - p0[0], p1[1] - p0[1]
    x2, y2 = p2[0] - p0[0], p2[1] - p0[1]
    return abs(x1 * y2 - x2 * y1) < parameters.EPSILON

def trilateration(s_pos, s_pos_n_list, distances_n_list):
    """
    :param s_pos:
    :param s_pos_n_list:
    :param distances_n_list:
    :return: the result of the minimization of the trilateration problem
    """
    # locations: [ (lat1, long1), ... ]
    # distances: [ distance1,     ... ]
    result = minimize(
        my_rmse,  # The error function
        s_pos,  # The initial guess
        args=(s_pos_n_list, distances_n_list),  # Additional parameters for mse
        method='L-BFGS-B',  # The optimisation algorithm
        options={
            'ftol': 1e-4,  # Tolerance
            'maxiter': 1e+8  # Maximum iterations
        }
    )
    return result.x

def my_rmse(s_pos, s_pos_n_list, distances_n_list):
    """
    :param s_pos: (s_x, s_y)
    :param s_pos_n_list: [ (s_x_1, s_y_1), (s_x_2, s_y_2), ... ]
    :param distances_n_list: [ d1, d2, ... ]
    :return:  mean squared error between distances from local coord system and global coord system
    """
    a = np.array(s_pos)
    distances_s_pos = [np.linalg.norm(a - np.array(b), ord=2) for b in s_pos_n_list]
    res = np.sqrt(np.mean(np.square(np.array(distances_s_pos)-np.array(distances_n_list))))
    return res

def get_hexa_coord(radius):
    """
    :param radius: indicates the size of coordinates to compute. generally, ~20 for 100 robots works fine
    :return: a list of the hexagonal coordinates as defined in [6]. Only acceptable coordinates are generated.
    """
    hexa_j = []
    hexa_k = []
    hexa_l = []
    for i in range(radius):
        for j in range(-i, i + 1):
            for k in range(-i, i + 1):
                if k <= -1:
                    for l in range(-i, i + 1):
                        if l >= 3:
                            if (np.abs(j) + np.abs(k) + np.abs(l) == 2 * i) and (j + k + l == 0):
                                hexa_j.append(j)
                                hexa_k.append(k)
                                hexa_l.append(l)
                        elif l >= 2 and j < 1:
                            if (np.abs(j) + np.abs(k) + np.abs(l) == 2 * i) and (j + k + l == 0):
                                hexa_j.append(j)
                                hexa_k.append(k)
                                hexa_l.append(l)
    return list(zip(hexa_j, hexa_k, hexa_l))


######################################
#
#             EXPERIMENT 2
#
#  Train 4 Networks on the WH+TW datasets
#
######################################

import pandas as pd
from tensorflow.keras.preprocessing.text import Tokenizer
import os
import time

from lib.utils import texts_preprocessing, get_embedding_matrix, fit_es, plot_history, plot_model_evaluation#plot_confusion_matrix#, predict_class
from lib.models import lstm_net, gru_net
from sklearn import metrics

def get_metrics(y_test, y_pred, Y_feat_names):
    res = {}
    res["accuracy"] = metrics.accuracy_score(y_test, y_pred)
    tmp = metrics.precision_score(y_test, y_pred, average=None)
    for index, cls in enumerate(Y_feat_names):
        res["precision_"+cls] = tmp[index]
    tmp = metrics.recall_score(y_test, y_pred, average=None)
    for index, cls in enumerate(Y_feat_names):
        res["recall_"+cls] = tmp[index]
    tmp = metrics.f1_score(y_test, y_pred, average=None)
    for index, cls in enumerate(Y_feat_names):
        res["f1_"+cls] = tmp[index]
    res["f1-micro"] = metrics.f1_score(y_test, y_pred, average="micro")
    res["f1-macro"] = metrics.f1_score(y_test, y_pred, average="macro")
    return res

def get_prediction_metrics(x_test, y_test, tokenizer, glove_dim):
    # preprocessing
    x_test_pre = texts_preprocessing(x_test, tokenizer, glove_dim)
    y_pred_loc = model.predict(x_test_pre)
    y_pred_loc = [Y_feat_names[0] if x[0]>=0.5 else Y_feat_names[1] for x in y_pred_loc]
    met_dict_loc = get_metrics(y_test, y_pred_loc, Y_feat_names)
    return met_dict_loc

#### Variables

# directories
main_path = "./"
dataset_filename = main_path+"data/annotation_results__ann"
dataset_wt_filename = main_path+"data/sample_ann2_"
glove_path = main_path+"glove/"
results_path = main_path+"experiments_02/"
models_path = results_path+"models/"
stats_path = results_path+"results/"
Y_feat_names = ["ns", "sens"]

# recurrent NN
VOCABULARY_SIZE = 10000
BATCH_SIZE = 50
DROPOUT = 0.5

# variables of NNs
ann_list = [2, 2, 3, 3]
glove_dim_list = [100, 100, 200, 200]
rnn_list = ["gru", "lstm", "gru", "lstm"]
bidirectional_list = [True, True, False, False]
num_layers_list = [2, 2, 1, 2]
num_nodes_list = [128, 256, 128, 128]
pyramid_list = [False, True, False, False]
glove = "twit"

experiment_time = time.perf_counter()

for index in range(10):
    # get data
    train_loc = pd.read_csv(dataset_wt_filename+(("0"+str(index+1))[-2:])+"_training.csv")
    val_loc = pd.read_csv(dataset_wt_filename+(("0"+str(index+1))[-2:])+"_validation.csv")
    test_loc = pd.read_csv(dataset_wt_filename+(("0"+str(index+1))[-2:])+"_test.csv")
    # transform data into sets for models
    x_train = train_loc["text"].tolist()
    y_train = pd.get_dummies(pd.DataFrame({"class": train_loc["class"].tolist()})["class"])[Y_feat_names].values
    x_val = val_loc["text"].tolist()
    y_val = pd.get_dummies(pd.DataFrame({"class": val_loc["class"].tolist()})["class"])[Y_feat_names].values
    x_test = test_loc["text"].tolist()
    y_test = pd.get_dummies(pd.DataFrame({"class": test_loc["class"].tolist()})["class"])[Y_feat_names].values
    # create tokenizer from training data
    tokenizer = Tokenizer(num_words=VOCABULARY_SIZE, split=' ')
    tokenizer.fit_on_texts(x_train)
    length = len(tokenizer.word_index)
    # loop on models
    for ann, glove_dim, rnn, bidirectional, num_layers, num_nodes, pyramid in zip(ann_list, glove_dim_list, rnn_list, bidirectional_list, num_layers_list, num_nodes_list, pyramid_list):
        # other
        if bidirectional:
            bid = "_bid"
        pyr = ""
        div_nodes = 0
        if pyramid:
            pyr = "_pyr"
            div_nodes = 2
        # load embedding
        emb_path = glove_path+'/glove.twitter.27B.' + str(glove_dim) + 'd.txt'
        if emb_path != "":
            emb_matrix = get_embedding_matrix(emb_path, tokenizer, glove_dim)
        else:
            emb_matrix = []
        # create and compile the model
        if rnn == "lstm":
            model = lstm_net(length, glove_dim, emb_matrix, glove_dim, num_layers, num_nodes, num_layers, DROPOUT, div_nodes, bidirectional)
        elif rnn == "gru":
            model = gru_net(length, glove_dim, emb_matrix, glove_dim, num_layers, num_nodes, num_layers, DROPOUT, div_nodes, bidirectional)
        # fit the model
        training_time = time.perf_counter()
        history = fit_es(model, x_train, y_train, x_val, y_val, tokenizer, glove_dim)
        training_time = time.perf_counter() - training_time
        print("Training time:", time.strftime("%H:%M:%S",time.gmtime(training_time)))
        # save model
        model_name = "sample"+(("0"+str(index+1))[-2:])+"_ann"+str(ann)+"_"+glove+"_gdim"+str(glove_dim)+"_"+rnn+bid+"_nlayers"+str(num_layers)+"_nnodes"+str(num_nodes)+pyr
        model.save(models_path+model_name)
        # model evaluation on test set
        met_dict = get_prediction_metrics(x_test, [Y_feat_names[0] if x[0]>=0.5 else Y_feat_names[1] for x in y_test], tokenizer, glove_dim)
        pd.DataFrame([met_dict]).to_csv(stats_path+model_name+".csv")

experiment_time = time.perf_counter() - experiment_time
print("Experiment time in seconds:", int(experiment_time))
print("Experiment time:", time.strftime("/%d, %H:%M:%S",time.gmtime(experiment_time)))


import torch
from torch import nn
import torch.nn.functional as F

class NNet(nn.Module):
    def __init__(self) -> None:
        super(NNet, self).__init__()
        self.conv1 = nn.Conv2d(1, 32, 3, 1)
        self.conv2 = nn.Conv2d(32, 64, 3, 1)
        self.dropout1 = nn.Dropout(0.25)
        self.dropout2 = nn.Dropout(0.5)
        self.fc1 = nn.Linear(9216, 128)
        self.fc2 = nn.Linear(128, 10)

    def forward(self, x):
        x = self.conv1(x)
        x = F.relu(x)
        x = self.conv2(x)
        x = F.relu(x)
        x = F.max_pool2d(x, 2)
        x = self.dropout1(x)
        x = torch.flatten(x, 1)
        x = self.fc1(x)
        x = F.relu(x)
        x = self.dropout2(x)
        x = self.fc2(x)
        return x

    def _init_params(self):
        pass

    def __str__(self) -> str:
        num_params = sum(p.numel() for p in self.parameters() if p.requires_grad == True)
        return f'\nModel\n\tnum params: {num_params}\n'


from flask import Flask
from flask_cors import CORS

app = Flask(__name__)
CORS(app)

@app.route("/")
def index():
    file = open("data.txt")
    return file.read()

@app.route("/help")
def help():
    return "This is a help message"

@app.route("/welp")
def welp():
    return "This is a welp message"

@app.route("/add")
def add():
    file = open("data.txt", "r+")
    count = file.read()
    if(count):
        count = int(count)
        count += 1
        file.truncate(0)
        file.seek(0)
    else:
        count = 1
    file.write(str(count))
    file.close()
    return "Added success"

@app.route("/clear")
def clear():
    file = open("data.txt", "w").close()
    return "File cleared"


import argparse
import pickle
import os

from .config import load_master_config
from .methods import concat_method_outputs

if __name__=="__main__":
    parser = argparse.ArgumentParser(description='parse key pairs into a dictionary')
    parser.add_argument("--config", dest="config", action='store',
                        metavar="Name of the configuration file in the config folder.", default='default')
    parser.add_argument("--n_instances", dest="insta", action='store', type=int,
                        metavar="Name of the configuration file in the config folder.", default='1')
    args = parser.parse_args()
    config_name = args.config
    n_instances = args.insta

    config = load_master_config(config_name)
    files_to_combine = list(config['save_to']['items'].keys())
    accumulator = {fn: [] for fn in files_to_combine}
    # load all dump files from the save_to directory
    for fref in files_to_combine:
        for i in range(n_instances):
            with open(os.path.join(config['save_to']['path'], str(i)+'_'+fref), 'rb') as f:
                accumulator[fref].append(pickle.load(f))

    accumulator_combined = {k: concat_method_outputs(preds) for k, preds in accumulator.items()}

    for fname, outs in accumulator_combined.items():
        # aggregate the needed outputs
        with open(os.path.join(config['save_to']['path'], fname), 'wb') as f:
            pickle.dump(outs, f)

    print(f'done, files combined in {config["save_to"]["path"]}')


import requests
from deprecated import deprecated
from ..target import Target
from ..real_estate import RealEstate
from .leboncoin_search_query import LeboncoinSearchQuery
from ..real_estate_service import RealEstateService
from ..settings import settings

class LeboncoinService(RealEstateService):
    @deprecated(version='1.1.1', reason="leboncoin is now using datadome which bans bots.")
    def __init__(self, api_key: str) -> None:
        self.url = settings.leboncoin_url
        self.headers = {
            "X-RapidAPI-Key": api_key,
            "X-RapidAPI-Host": settings.leboncoin_host
        }

    @deprecated(version='1.1.1', reason="leboncoin is now using datadome which bans bots.")
    def search(self, query: LeboncoinSearchQuery) -> list[RealEstate]:
        response = requests.post(
            self.url,
            headers=self.headers,
            timeout=settings.request_timeout_in_secs,
            data=query.model_dump()
        )

        return RealEstate.from_response(response, Target.LEBONCOIN)


import pytest
import vcr

from pyembed.core import PyEmbed
from pyembed.core.render import PyEmbedRenderer

class DummyRenderer(PyEmbedRenderer):
    def render(self, content_url, response):
        return "%s by %s from %s" % \
               (response.title, response.author_name, content_url)

@vcr.use_cassette('pyembed/core/test/fixtures/cassettes/correct_embedding.yml')
def test_should_get_correct_embedding():
    embedding = PyEmbed().embed(
        'https://twitter.com/BarackObama/status/266031293945503744')
    assert 'Four more years.' in embedding

@vcr.use_cassette('pyembed/core/test/fixtures/cassettes/another_correct_embedding.yml')
def test_should_get_another_correct_embedding():
    embedding = PyEmbed().embed(
        'http://www.flickr.com/photos/hansjuul/7899334594')
    assert '.jpg' in embedding

@vcr.use_cassette('pyembed/core/test/fixtures/cassettes/maximum_height.yml')
def test_should_embed_with_maximum_height():
    embedding = PyEmbed().embed(
        'http://www.youtube.com/watch?v=9bZkp7q19f0', max_height=200)
    assert 'height="200"' in embedding

@vcr.use_cassette('pyembed/core/test/fixtures/cassettes/custom_renderer.yml')
def test_should_embed_with_custom_renderer():
    embedding = PyEmbed(renderer=DummyRenderer()).embed(
        'http://www.youtube.com/watch?v=qrO4YZeyl0I')
    assert embedding == \
        'Lady Gaga - Bad Romance by LadyGagaVEVO from ' + \
        'http://www.youtube.com/watch?v=qrO4YZeyl0I'

@vcr.use_cassette('pyembed/core/test/fixtures/cassettes/no_discovery.yml')
def test_should_embed_when_no_discovery():
    embedding = PyEmbed().embed(
        'http://www.rdio.com/artist/Mike_Oldfield/album/Amarok/')
    assert 'rd.io' in embedding

@vcr.use_cassette('pyembed/core/test/fixtures/cassettes/text_xml.yml')
def test_should_embed_when_text_xml_returned():
    embedding = PyEmbed().embed('https://soundcloud.com/coltonprovias/rush')
    assert 'soundcloud.com' in embedding


# -*- coding: utf-8 -*-
"""
@author: Infaraway
@time: 2018/4/19 12:03
@Function:
"""
import collections

import gc
from gensim.models.word2vec import Word2Vec
import numpy as np
import pandas as pd

class Feature2vec:

    def __init__(self, data, size=15):
        self.size = size
        data = data.fillna('-1')

        # self.feature = ['interest1', 'interest2', 'interest5', 'kw1', 'kw2', 'topic1', 'topic2']
        self.feature = ['topic2']

        self.w2v_feature = pd.DataFrame()
        self.w2v_feature['uid'] = data['uid']
        for feat in self.feature:
            self.w2v_feature[feat] = data[feat]

    def get_feature(self):
        for feature in self.feature:
            print("this is feature:", feature)
            # self._select_topk(feature)
            self.w2v_feature[feature] = self.w2v_feature[feature].apply(lambda x: str(x).split(' '))
            model = Word2Vec(self.w2v_feature[feature], size=self.size, min_count=1, window=2, iter=5, workers=32)
            # model = Word2Vec.load('model/' + str(self.size) + feature +  '15_w2v.mod')
            model.save('data/w2v_model/w2v_' + str(self.size) + feature + '.mod')

            data_vec = []
            for row in self.w2v_feature[feature]:
                data_vec.append(self._base_word2vec(row, model))
            column_names = []
            for i in range(self.size):
                column_names.append(feature + str(i))

            data_vec = pd.DataFrame(data_vec, columns=column_names)
            data_vec['uid'] = self.w2v_feature['uid']
            data_vec = data_vec.round(4)
            data_vec.to_csv("data/w2v_feature/w2v_all_" + str(self.size) + feature + '.csv', index=False)
            gc.collect()

    def _base_word2vec(self, x, model):
        vec = np.zeros(self.size)
        # x = [word for word in x if model.__contains__(word)]
        for item in x:
            vec += model.wv[item]
        if len(x) == 0:
            return vec
        else:
            return vec / len(x)

    def _select_topk(self, feature):
        word_list = []
        for line in self.w2v_feature[feature]:
            words = str(line).split(' ')
            word_list += words
        result = collections.Counter(word_list)
        size = len(result)
        result = result.most_common(int(size))

        print(result[0], result[int(size * 0.01)], result[int(size * 0.05)], result[int(size * 0.1)],
              result[int(size * 0.2)], result[int(size * 0.3)], result[int(size * 0.4)], result[int(size * 0.5)])
        result = [res for res in result if 1000 < res[1] < 1000000]

        word_dict = {}
        for re in result:
            word_dict[re[0]] = 1

        self.w2v_feature[feature] = self.w2v_feature[feature].apply(
            lambda x: ' '.join([word for word in str(x).split(' ') if word_dict.__contains__(word)]))

    def get_topk(self):
        new_pd = pd.DataFrame()
        vector_feature = ['interest1', 'interest2', 'interest5', 'kw1', 'kw2', 'topic1', 'topic2']
        # vector_feature = ['kw2']
        for feature in vector_feature:
            print("this is feature:", feature)

            self.data[feature] = self.data[feature].apply(lambda x: str(x).split(' '))
            word_dict = self.select_topk(self.data[feature])
            new_pd[feature] = self.data[feature].apply(
                lambda x: ' '.join([word for word in x if word_dict.__contains__(word)]))
            temp = new_pd[feature].apply(lambda x: len(str(x).split(' ')))
            print(temp.describe())

        new_pd.to_csv('./data/data_top50%feature.csv', index=False)

if __name__ == '__main__':
    data = pd.read_csv('data/raw_data/userFeature.csv')
    # data = pd.read_csv('data/data_5%.csv')
    word_vec = Feature2vec(data, size=20)
    word_vec.get_feature()
    # word_vec.w2v_feature.to_csv('data/raw_data/user_w2vFeature.csv', index=False)
    data = data.fillna('-1')

    # features = ['uid', 'interest1', 'interest2', 'interest5', 'kw1', 'kw2', 'topic1', 'topic2']
    # data = data[features]
    # print(data.shape)
    # all_ = data.shape[0]
    # for feat in features:
    #     count = data[data[feat] == '-1'].shape[0]
    #     print(feat, count / all_)
    # data.to_csv('data/feature_data/user_w2vFeature_all.csv', index=False)