import functools
import fnmatch
import json
import itertools
import os

from botocore.paginate import Paginator

from c7n.query import QueryResourceManager, ChildResourceManager, TypeInfo, RetryPageIterator
from c7n.manager import resources
from c7n.utils import chunks, get_retry, generate_arn, local_session, type_schema
from c7n.actions import BaseAction
from c7n.filters import Filter

from c7n.resources.shield import IsShieldProtected, SetShieldProtection
from c7n.tags import RemoveTag, Tag

class Route53Base:

    permissions = ('route53:ListTagsForResources',)
    retry = staticmethod(get_retry(('Throttled',)))

    @property
    def generate_arn(self):
        if self._generate_arn is None:
            self._generate_arn = functools.partial(
                generate_arn,
                self.get_model().service,
                resource_type=self.get_model().arn_type)
        return self._generate_arn

    def get_arn(self, r):
        return self.generate_arn(r[self.get_model().id].split("/")[-1])

    def augment(self, resources):
        _describe_route53_tags(
            self.get_model(), resources, self.session_factory,
            self.executor_factory, self.retry)
        return resources

def _describe_route53_tags(
        model, resources, session_factory, executor_factory, retry):

    def process_tags(resources):
        client = local_session(session_factory).client('route53')
        resource_map = {}
        for r in resources:
            k = r[model.id]
            if "hostedzone" in k:
                k = k.split("/")[-1]
            resource_map[k] = r

        for resource_batch in chunks(list(resource_map.keys()), 10):
            results = retry(
                client.list_tags_for_resources,
                ResourceType=model.arn_type,
                ResourceIds=resource_batch)
            for resource_tag_set in results['ResourceTagSets']:
                if ('ResourceId' in resource_tag_set and
                        resource_tag_set['ResourceId'] in resource_map):
                    resource_map[resource_tag_set['ResourceId']]['Tags'] = resource_tag_set['Tags']

    with executor_factory(max_workers=2) as w:
        return list(w.map(process_tags, chunks(resources, 20)))

@resources.register('hostedzone')
class HostedZone(Route53Base, QueryResourceManager):

    class resource_type(TypeInfo):
        service = 'route53'
        arn_type = 'hostedzone'
        enum_spec = ('list_hosted_zones', 'HostedZones', None)
        # detail_spec = ('get_hosted_zone', 'Id', 'Id', None)
        id = 'Id'
        name = 'Name'
        universal_taggable = True
        # Denotes this resource type exists across regions
        global_resource = True
        cfn_type = 'AWS::Route53::HostedZone'

    def get_arns(self, resource_set):
        arns = []
        for r in resource_set:
            _id = r[self.get_model().id].split("/")[-1]
            arns.append(self.generate_arn(_id))
        return arns

HostedZone.filter_registry.register('shield-enabled', IsShieldProtected)
HostedZone.action_registry.register('set-shield', SetShieldProtection)

@resources.register('healthcheck')
class HealthCheck(Route53Base, QueryResourceManager):

    class resource_type(TypeInfo):
        service = 'route53'
        arn_type = 'healthcheck'
        enum_spec = ('list_health_checks', 'HealthChecks', None)
        name = id = 'Id'
        universal_taggable = True
        cfn_type = 'AWS::Route53::HealthCheck'

@resources.register('rrset')
class ResourceRecordSet(ChildResourceManager):

    class resource_type(TypeInfo):
        service = 'route53'
        arn_type = 'rrset'
        parent_spec = ('hostedzone', 'HostedZoneId', None)
        enum_spec = ('list_resource_record_sets', 'ResourceRecordSets', None)
        name = id = 'Name'
        cfn_type = 'AWS::Route53::RecordSet'

@resources.register('r53domain')
class Route53Domain(QueryResourceManager):

    class resource_type(TypeInfo):
        service = 'route53domains'
        arn_type = 'r53domain'
        enum_spec = ('list_domains', 'Domains', None)
        name = id = 'DomainName'

    permissions = ('route53domains:ListTagsForDomain',)

    def augment(self, domains):
        client = local_session(self.session_factory).client('route53domains')
        for d in domains:
            d['Tags'] = self.retry(
                client.list_tags_for_domain,
                DomainName=d['DomainName'])['TagList']
        return domains

@Route53Domain.action_registry.register('tag')
class Route53DomainAddTag(Tag):
    """Adds tags to a route53 domain

    :example:

    .. code-block:: yaml

        policies:
          - name: route53-tag
            resource: r53domain
            filters:
              - "tag:DesiredTag": absent
            actions:
              - type: tag
                key: DesiredTag
                value: DesiredValue
    """
    permissions = ('route53domains:UpdateTagsForDomain',)

    def process_resource_set(self, client, domains, tags):
        mid = self.manager.resource_type.id
        for d in domains:
            client.update_tags_for_domain(
                DomainName=d[mid],
                TagsToUpdate=tags)

@Route53Domain.action_registry.register('remove-tag')
class Route53DomainRemoveTag(RemoveTag):
    """Remove tags from a route53 domain

    :example:

    .. code-block:: yaml

        policies:
          - name: route53-expired-tag
            resource: r53domain
            filters:
              - "tag:ExpiredTag": present
            actions:
              - type: remove-tag
                tags: ['ExpiredTag']
    """
    permissions = ('route53domains:DeleteTagsForDomain',)

    def process_resource_set(self, client, domains, keys):
        for d in domains:
            client.delete_tags_for_domain(
                DomainName=d[self.id_key],
                TagsToDelete=keys)

@HostedZone.action_registry.register('delete')
class Delete(BaseAction):
    """Action to delete Route 53 hosted zones.

    It is recommended to use a filter to avoid unwanted deletion of R53 hosted zones.

    If set to force this action will wipe out all records in the hosted zone
    before deleting the zone.

    :example:

    .. code-block:: yaml

            policies:
              - name: route53-delete-testing-hosted-zones
                resource: aws.hostedzone
                filters:
                  - 'tag:TestTag': present
                actions:
                  - type: delete
                    force: true

    """

    schema = type_schema('delete', force={'type': 'boolean'})
    permissions = ('route53:DeleteHostedZone',)

    def process(self, hosted_zones):
        client = local_session(self.manager.session_factory).client('route53')
        error = None
        for hz in hosted_zones:
            if self.data.get('force'):
                self.delete_records(client, hz)
            try:
                self.manager.retry(
                    client.delete_hosted_zone,
                    Id=hz['Id'],
                    ignore_err_codes=('NoSuchHostedZone'))
            except client.exceptions.HostedZoneNotEmpty as e:
                self.log.warning(
                    "HostedZone: %s cannot be deleted, "
                    "set force to remove all records in zone",
                    hz['Name'])
                error = e
        if error:
            raise error

    def delete_records(self, client, hz):
        paginator = client.get_paginator('list_resource_record_sets')
        paginator.PAGE_ITERATOR_CLS = RetryPageIterator
        rrsets = paginator.paginate(HostedZoneId=hz['Id']).build_full_result()

        for rrset in rrsets['ResourceRecordSets']:
            # Trigger the deletion of all the resource record sets before deleting
            # the hosted zone

            # Exempt the two zone associated mandatory records
            if rrset['Name'] == hz['Name'] and rrset['Type'] in ('NS', 'SOA'):
                continue
            self.manager.retry(
                client.change_resource_record_sets,
                HostedZoneId=hz['Id'],
                ChangeBatch={
                    'Changes': [
                        {
                            'Action': 'DELETE',
                            'ResourceRecordSet': {
                                'Name': rrset['Name'],
                                'Type': rrset['Type'],
                                'TTL': rrset['TTL'],
                                'ResourceRecords': rrset['ResourceRecords']
                            },
                        }
                    ]
                },
                ignore_err_codes=('InvalidChangeBatch'))

@HostedZone.action_registry.register('set-query-logging')
class SetQueryLogging(BaseAction):
    """Enables query logging on a hosted zone.

    By default this enables a log group per route53 domain, alternatively
    a log group name can be specified for a unified log across domains.

    Note this only applicable to public route53 domains, and log groups
    must be created in us-east-1 region.

    This action can optionally setup the resource permissions needed for
    route53 to log to cloud watch logs via `set-permissions: true`, else
    the cloud watch logs resource policy would need to be set separately.

    Its recommended to use a separate custodian policy on the log
    groups to set the log retention period for the zone logs. See
    `custodian schema aws.log-group.actions.set-retention`

    :example:

    .. code-block:: yaml

        policies:
          - name: enablednsquerylogging
            resource: hostedzone
            region: us-east-1
            filters:
              - type: query-logging-enabled
                state: false
            actions:
              - type: set-query-logging
                state: true

    """

    permissions = (
        'route53:GetQueryLoggingConfig',
        'route53:CreateQueryLoggingConfig',
        'route53:DeleteQueryLoggingConfig',
        'logs:DescribeLogGroups',
        'logs:CreateLogGroup',
        'logs:GetResourcePolicy',
        'logs:PutResourcePolicy')

    schema = type_schema(
        'set-query-logging', **{
            'set-permissions': {'type': 'boolean'},
            'log-group-prefix': {'type': 'string', 'default': '/aws/route53'},
            'log-group': {'type': 'string', 'default': 'auto'},
            'state': {'type': 'boolean'}})

    statement = {
        "Sid": "Route53LogsToCloudWatchLogs",
        "Effect": "Allow",
        "Principal": {"Service": ["route53.amazonaws.com"]},
        "Action": ["logs:PutLogEvents", "logs:CreateLogStream"],
        "Resource": None}

    def validate(self):
        if not self.data.get('state', True):
            # By forcing use of a filter we ensure both getting to right set of
            # resources as well avoiding an extra api call here, as we'll reuse
            # the annotation from the filter for logging config.
            if not [f for f in self.manager.iter_filters() if isinstance(
                    f, IsQueryLoggingEnabled)]:
                raise ValueError(
                    "set-query-logging when deleting requires "
                    "use of query-logging-enabled filter in policy")
        return self

    def get_permissions(self):
        perms = []
        if self.data.get('set-permissions'):
            perms.extend(('logs:GetResourcePolicy', 'logs:PutResourcePolicy'))
        if self.data.get('state', True):
            perms.append('route53:CreateQueryLoggingConfig')
            perms.append('logs:CreateLogGroup')
            perms.append('logs:DescribeLogGroups')
            perms.append('tag:GetResources')
        else:
            perms.append('route53:DeleteQueryLoggingConfig')
        return perms

    def process(self, resources):
        if self.manager.config.region != 'us-east-1':
            self.log.warning("set-query-logging should be only be performed region: us-east-1")

        client = local_session(self.manager.session_factory).client('route53')
        state = self.data.get('state', True)

        zone_log_names = {z['Id']: self.get_zone_log_name(z) for z in resources}
        if state:
            self.ensure_log_groups(set(zone_log_names.values()))

        for r in resources:
            if not state:
                try:
                    client.delete_query_logging_config(Id=r['c7n:log-config']['Id'])
                except client.exceptions.NoSuchQueryLoggingConfig:
                    pass
                continue
            log_arn = "arn:aws:logs:us-east-1:{}:log-group:{}".format(
                self.manager.account_id, zone_log_names[r['Id']])
            client.create_query_logging_config(
                HostedZoneId=r['Id'],
                CloudWatchLogsLogGroupArn=log_arn)

    def get_zone_log_name(self, zone):
        if self.data.get('log-group', 'auto') == 'auto':
            log_group_name = "%s/%s" % (
                self.data.get('log-group-prefix', '/aws/route53').rstrip('/'),
                zone['Name'][:-1])
        else:
            log_group_name = self.data['log-group']
        return log_group_name

    def ensure_log_groups(self, group_names):
        log_manager = self.manager.get_resource_manager('log-group')
        log_manager.config = self.manager.config.copy(region='us-east-1')

        if len(group_names) == 1:
            groups = []
            if log_manager.get_resources(list(group_names), augment=False):
                groups = [{'logGroupName': g} for g in group_names]
        else:
            common_prefix = os.path.commonprefix(list(group_names))
            if common_prefix not in ('', '/'):
                groups = log_manager.get_resources(
                    [common_prefix], augment=False)
            else:
                groups = list(itertools.chain(*[
                    log_manager.get_resources([g]) for g in group_names]))

        missing = group_names.difference({g['logGroupName'] for g in groups})

        # Logs groups must be created in us-east-1 for route53.
        client = local_session(
            self.manager.session_factory).client('logs', region_name='us-east-1')

        for g in missing:
            client.create_log_group(logGroupName=g)

        if self.data.get('set-permissions', False):
            self.ensure_route53_permissions(client, group_names)

    def ensure_route53_permissions(self, client, group_names):
        if self.check_route53_permissions(client, group_names):
            return
        if self.data.get('log-group', 'auto') != 'auto':
            p_resource = "arn:aws:logs:us-east-1:{}:log-group:{}:*".format(
                self.manager.account_id, self.data['log-group'])
        else:
            p_resource = "arn:aws:logs:us-east-1:{}:log-group:{}/*".format(
                self.manager.account_id,
                self.data.get('log-group-prefix', '/aws/route53').rstrip('/'))

        statement = dict(self.statement)
        statement['Resource'] = p_resource

        client.put_resource_policy(
            policyName='Route53LogWrites',
            policyDocument=json.dumps(
                {"Version": "2012-10-17", "Statement": [statement]}))

    def check_route53_permissions(self, client, group_names):
        group_names = set(group_names)
        for p in client.describe_resource_policies().get('resourcePolicies', []):
            for s in json.loads(p['policyDocument']).get('Statement', []):
                if (s['Effect'] == 'Allow' and
                        s['Principal'].get('Service', ['']) == "route53.amazonaws.com"):
                    group_names.difference_update(
                        fnmatch.filter(group_names, s['Resource'].rsplit(':', 1)[-1]))
                    if not group_names:
                        return True
        return not bool(group_names)

def get_logging_config_paginator(client):
    return Paginator(
        client.list_query_logging_configs,
        {'input_token': 'NextToken', 'output_token': 'NextToken',
         'result_key': 'QueryLoggingConfigs'},
        client.meta.service_model.operation_model('ListQueryLoggingConfigs'))

@HostedZone.filter_registry.register('query-logging-enabled')
class IsQueryLoggingEnabled(Filter):

    permissions = ('route53:GetQueryLoggingConfig', 'route53:GetHostedZone')
    schema = type_schema('query-logging-enabled', state={'type': 'boolean'})

    def process(self, resources, event=None):
        client = local_session(self.manager.session_factory).client('route53')
        state = self.data.get('state', False)
        results = []

        enabled_zones = {
            c['HostedZoneId']: c for c in
            get_logging_config_paginator(
                client).paginate().build_full_result().get(
                    'QueryLoggingConfigs', ())}
        for r in resources:
            zid = r['Id'].split('/', 2)[-1]
            # query logging is only supported for Public Hosted Zones.
            if r['Config']['PrivateZone'] is True:
                continue
            logging = zid in enabled_zones
            if logging and state:
                r['c7n:log-config'] = enabled_zones[zid]
                results.append(r)
            elif not logging and not state:
                results.append(r)
        return results


import sys
from pyspark.sql.functions import *
from pyspark import SparkContext
from pyspark.streaming import StreamingContext
from pyspark.sql.types import *
from pyspark.context import SparkContext
from pyspark.sql.session import SparkSession
from pyspark.sql import SQLContext

inputPath = "C:\SparkCourse\Assignment4\merge" #directory to look for picking up the data by spark

def main():
    def process(time, rdd):
        if(rdd.count() > 0):
            df = sqlContext.createDataFrame(rdd)
            df.registerTempTable("myips")
            df.printSchema()
            print(sqlContext.sql("select _2, COUNT(*) from myips group by _2 having COUNT(*) > 1").collect()) #collect sourceip,destinationip having count > 1
        else:
            print("empty rdd")
    sc = SparkContext(appName="PysparkStreaming")
    ssc = StreamingContext(sc, 60)   #Streaming will execute in each 3 seconds
    sqlContext = SQLContext(sc)
    lines = ssc.textFileStream(inputPath)
    avgs = lines.flatMap(lambda line: line.split(" ")).map(lambda x: (1, x)) #pass sourceip,destination ip in the rdd
    avgs.foreachRDD(process) #process RDD

    ssc.start()
    ssc.awaitTermination()

if __name__ == "__main__":
    main()


OTEL_LOGS_EXPORTER = "OTEL_LOGS_EXPORTER"
"""
.. envvar:: OTEL_LOGS_EXPORTER

"""

OTEL_METRICS_EXPORTER = "OTEL_METRICS_EXPORTER"
"""
.. envvar:: OTEL_METRICS_EXPORTER

Specifies which exporter is used for metrics. See `General SDK Configuration
<https://opentelemetry.io/docs/concepts/sdk-configuration/general-sdk-configuration/#otel_metrics_exporter>`_.

**Default value:** ``"otlp"``

**Example:**

``export OTEL_METRICS_EXPORTER="prometheus"``

Accepted values for ``OTEL_METRICS_EXPORTER`` are:

- ``"otlp"``
- ``"prometheus"``
- ``"none"``: No automatically configured exporter for metrics.

.. note::

    Exporter packages may add entry points for group ``opentelemetry_metrics_exporter`` which
    can then be used with this environment variable by name. The entry point should point to
    either a `opentelemetry.sdk.metrics.export.MetricExporter` (push exporter) or
    `opentelemetry.sdk.metrics.export.MetricReader` (pull exporter) subclass; it must be
    constructable without any required arguments. This mechanism is considered experimental and
    may change in subsequent releases.
"""

OTEL_PROPAGATORS = "OTEL_PROPAGATORS"
"""
.. envvar:: OTEL_PROPAGATORS
"""

OTEL_PYTHON_CONTEXT = "OTEL_PYTHON_CONTEXT"
"""
.. envvar:: OTEL_PYTHON_CONTEXT
"""

OTEL_PYTHON_ID_GENERATOR = "OTEL_PYTHON_ID_GENERATOR"
"""
.. envvar:: OTEL_PYTHON_ID_GENERATOR
"""

OTEL_TRACES_EXPORTER = "OTEL_TRACES_EXPORTER"
"""
.. envvar:: OTEL_TRACES_EXPORTER
"""

OTEL_PYTHON_TRACER_PROVIDER = "OTEL_PYTHON_TRACER_PROVIDER"
"""
.. envvar:: OTEL_PYTHON_TRACER_PROVIDER
"""

OTEL_PYTHON_METER_PROVIDER = "OTEL_PYTHON_METER_PROVIDER"
"""
.. envvar:: OTEL_PYTHON_METER_PROVIDER
"""

_OTEL_PYTHON_LOGGER_PROVIDER = "OTEL_PYTHON_LOGGER_PROVIDER"
"""
.. envvar:: OTEL_PYTHON_LOGGER_PROVIDER
"""


from django import forms
from django.contrib import admin
from .models import Menu
from mptt.admin import MPTTModelAdmin

class MenuForm(forms.ModelForm):
    class Meta:
        model = Menu
        exclude = (
            'id',
            'slug',
            'title',
            'note',
            'created_by',
            'updated_by',
        )
        widgets={
            'parent': forms.Select(attrs={'class': 'bootstrap-select', 'data-width':"80%"})
            }

class MenuAdmin(MPTTModelAdmin):
    list_display = ['title_uz', 'index', ]
    search_fields = ['title_uz', 'title_uzb', 'title_ru', 'title_en', 'note']
    form = MenuForm
    exclude = (
        'id',
        'slug',
        'title',
        'note',
        'created_by',
        'updated_by',
    )

    def save_model(self, request, obj, form, change):
        if not obj.created_by:
            obj.created_by = request.user
        obj.updated_by = request.user
        return super().save_model(request, obj, form, change)

    current_object = None

    def get_form(self, request, obj=None, **kwargs):
        if obj:
            self.current_object = obj.id
        else:
            self.current_object = None
        return super(MenuAdmin, self).get_form(request, obj, **kwargs)

    def formfield_for_foreignkey(self, db_field, request, **kwargs):
        if db_field.name == "parent":
            if self.current_object:
                current = Menu.objects.get(pk=self.current_object)
                descendant = current.get_descendants(include_self=True)
                kwargs["queryset"] = Menu.objects.all().exclude(id__in=descendant)
            else:
                kwargs["queryset"] = Menu.objects.all()
        return super().formfield_for_foreignkey(db_field, request, **kwargs)

admin.site.register(Menu, MenuAdmin)


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import tensorflow as tf
from tensorflow.python.ops import control_flow_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops import state_ops
from tensorflow.python.framework import ops
from tensorflow.python.training import optimizer
from tensorflow.python.ops import control_flow_ops
from tensorflow.python.training import training_ops

__all__ = [
    'StochasticGradientLangevinDynamics',
]

class EasyStochasticGradientLangevinDynamics(tf.train.Optimizer):
    def __init__(self, learning_rate=0.001, data_size=1, use_locking=False, name="SGLD"):
        super(EasyStochasticGradientLangevinDynamics, self).__init__(use_locking, name)
        self._lr = learning_rate
        self._lr_t = None

        self._data_size = tf.convert_to_tensor(
          data_size, name='data_size')

    def _prepare(self):
        self._lr_t = ops.convert_to_tensor(self._lr, name="learning_rate")

    def _apply_dense(self, grad, var):
        lr_t = math_ops.cast(self._lr_t, var.dtype.base_dtype)
        num_pseudo_batchs = tf.cast(self._data_size, grad.dtype)
        noise = tf.random_normal(
            tf.shape(grad), stddev=tf.sqrt(lr_t), dtype=grad.dtype)
        new_grad = 0.5*lr_t*grad+noise
        var_update = state_ops.assign_sub(var, new_grad)
        return control_flow_ops.group(*[var_update])

    def _apply_sparse(self, grad, var):
        raise NotImplementedError("Sparse gradient updates are not supported.")

class StochasticGradientLangevinDynamics(tf.train.Optimizer):
  """An optimizer module for stochastic gradient Langevin dynamics.
  This implements the preconditioned Stochastic Gradient Langevin Dynamics
  optimizer [(Li et al., 2016)][1]. The optimization variable is regarded as a
  sample from the posterior under Stochastic Gradient Langevin Dynamics with
  noise rescaled in each dimension according to [RMSProp](
  http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf).
  Note: If a prior is included in the loss, it should be scaled by
  `1/data_size`, where data_size is the number of minibatches
  in the data.  I.e., it should be divided by the `data_size` term
  described below.
  Args:
    learning_rate: Scalar `float`-like `Tensor`. The base learning rate for the
      optimizer. Must be tuned to the specific function being minimized.
    preconditioner_decay_rate: Scalar `float`-like `Tensor`. The exponential
      decay rate of the rescaling of the preconditioner (RMSprop). (This is
      "alpha" in Li et al. (2016)). Should be smaller than but nearly `1` to
      approximate sampling from the posterior. (Default: `0.95`)
    data_size: Scalar `int`-like `Tensor`. The effective number of
      minibatches in the data set.  Trades off noise and prior with the SGD
      likelihood term. Note: Assumes the loss is taken as the mean over a
      minibatch. Otherwise if the sum was taken, divide this number by the
      batch size.  (Default: `1`)
    burnin: Scalar `int`-like `Tensor`. The number of iterations to collect
      gradient statistics to update the preconditioner before starting to draw
      noisy samples. (Default: `25`)
    diagonal_bias: Scalar `float`-like `Tensor`. Term added to the diagonal of
      the preconditioner to prevent the preconditioner from degenerating.
      (Default: `1e-8`)
    name: Python `str` describing ops managed by this function.
      (Default: `"StochasticGradientLangevinDynamics"`)
    variable_scope: Variable scope used for calls to `tf.get_variable`.
      If `None`, a new variable scope is created using name
      `tf.get_default_graph().unique_name(name or default_name)`.
  Raises:
    InvalidArgumentError: If preconditioner_decay_rate is a `Tensor` not in
      `(0,1]`.
  #### References
  [1]: Chunyuan Li, Changyou Chen, David Carlson, and Lawrence Carin.
       Preconditioned Stochastic Gradient Langevin Dynamics for Deep Neural
       Networks. In _Association for the Advancement of Artificial
       Intelligence_, 2016. https://arxiv.org/abs/1512.07666
  """

  def __init__(self,
               learning_rate,
               data_size,
               preconditioner_decay_rate=0.95,
               burnin=25,
               diagonal_bias=1e-8,
               name=None,
               variable_scope=None):
    default_name = 'StochasticGradientLangevinDynamics'
    with tf.name_scope(name, default_name, [
        learning_rate, preconditioner_decay_rate, data_size, burnin,
        diagonal_bias
    ]):
      if variable_scope is None:
        var_scope_name = tf.get_default_graph().unique_name(
            name or default_name)
        with tf.variable_scope(var_scope_name) as scope:
          self._variable_scope = scope
      else:
        self._variable_scope = variable_scope

      self._preconditioner_decay_rate = tf.convert_to_tensor(
          preconditioner_decay_rate, name='preconditioner_decay_rate')
      self._data_size = tf.convert_to_tensor(
          data_size, name='data_size')
      self._burnin = tf.convert_to_tensor(burnin, name='burnin')
      self._diagonal_bias = tf.convert_to_tensor(
          diagonal_bias, name='diagonal_bias')
      self._learning_rate = tf.convert_to_tensor(
          learning_rate, name='learning_rate')

      with tf.variable_scope(self._variable_scope):
        self._counter = tf.get_variable(
            'counter', initializer=0, trainable=False)

      self._preconditioner_decay_rate = control_flow_ops.with_dependencies([
          tf.assert_non_negative(
              self._preconditioner_decay_rate,
              message='`preconditioner_decay_rate` must be non-negative'),
          tf.assert_less_equal(
              self._preconditioner_decay_rate,
              1.,
              message='`preconditioner_decay_rate` must be at most 1.'),
      ], self._preconditioner_decay_rate)

      self._data_size = control_flow_ops.with_dependencies([
          tf.assert_greater(
              self._data_size,
              0,
              message='`data_size` must be greater than zero')
      ], self._data_size)

      self._burnin = control_flow_ops.with_dependencies([
          tf.assert_non_negative(
              self._burnin, message='`burnin` must be non-negative'),
          tf.assert_integer(
              self._burnin, message='`burnin` must be an integer')
      ], self._burnin)

      self._diagonal_bias = control_flow_ops.with_dependencies([
          tf.assert_non_negative(
              self._diagonal_bias,
              message='`diagonal_bias` must be non-negative')
      ], self._diagonal_bias)

      super(StochasticGradientLangevinDynamics, self).__init__(
          use_locking=False, name=name or default_name)

  def _create_slots(self, var_list):
    for v in var_list:
      init_rms = tf.ones_initializer(dtype=v.dtype)
      self._get_or_make_slot_with_initializer(v, init_rms, v.get_shape(),
                                              v.dtype, 'rms', self._name)

  def _prepare(self):
    # We need to put the conversion and check here because a user will likely
    # want to decay the learning rate dynamically.
    self._learning_rate_tensor = control_flow_ops.with_dependencies([
        tf.assert_non_negative(
            self._learning_rate, message='`learning_rate` must be non-negative')
    ], tf.convert_to_tensor(self._learning_rate, name='learning_rate_tensor'))
    self._decay_tensor = tf.convert_to_tensor(
        self._preconditioner_decay_rate, name='preconditioner_decay_rate')

    super(StochasticGradientLangevinDynamics, self)._prepare()

  def _apply_dense(self, grad, var):
    rms = self.get_slot(var, 'rms')

    with tf.control_dependencies([
        self._update_momentum(rms, grad, tf.cast(
            self._decay_tensor, var.dtype.base_dtype))]):
      new_grad = self._apply_noisy_update(rms, grad)

    return training_ops.apply_gradient_descent(
        var,
        tf.cast(self._learning_rate_tensor, var.dtype.base_dtype),
        new_grad,
        use_locking=self._use_locking).op

  def _apply_sparse(self, grad, var):
    rms = self.get_slot(var, 'rms')

    with tf.control_dependencies([
        self._update_momentum(rms, grad, tf.cast(
            self._decay_tensor, var.dtype.base_dtype))]):
      new_grad = self._apply_noisy_update(rms, grad)

    return training_ops.apply_gradient_descent(
        var,
        tf.cast(self._learning_rate_tensor, var.dtype.base_dtype),
        new_grad,
        use_locking=self._use_locking).op

  def _finish(self, update_ops, name_scope):
    update_ops.append([self._counter.assign_add(1)])
    return tf.group(*update_ops, name=name_scope)

  @property
  def variable_scope(self):
    """Variable scope of all calls to `tf.get_variable`."""
    return self._variable_scope

  def _apply_noisy_update(self, mom, grad):
    # Compute and apply the gradient update following
    # preconditioned Langevin dynamics
    stddev = tf.where(
        tf.squeeze(self._counter > self._burnin),
        tf.cast(tf.rsqrt(self._learning_rate), grad.dtype),
        tf.zeros([], grad.dtype))

    preconditioner = tf.rsqrt(
        mom + tf.cast(self._diagonal_bias, grad.dtype))

    '''
    return (
        0.5 * preconditioner*grad*tf.cast(
            self._data_size, grad.dtype)
        +tf.random_normal(tf.shape(grad), 1., dtype=grad.dtype)
        *stddev * tf.sqrt(preconditioner))
    '''

    return (
        preconditioner*grad
        +tf.random_normal(tf.shape(grad), mean=0., stddev=1.0, dtype=grad.dtype)
        *stddev*tf.sqrt(2*preconditioner/tf.cast(self._data_size, grad.dtype)))

  def _update_momentum(self, mom, grad, decay):
    # Keep an exponentially weighted moving average of squared gradients.
    # Not thread safe
    return mom.assign_add((1.0 - decay) * (tf.square(grad) - mom))


#Author: Miguel Guardado
#Date Created: 03/13/2021

import argparse
import pandas as pd
import numpy as np
import os
from scipy import stats
import shutil

parser = argparse.ArgumentParser()

def create_fine_map_region(eQTL_subset,kbregion):
    """
    This function will take in a list of eQTL regions for snps and input
    """
    kbregion=int(kbregion)*1000

    #Construct a new dataframe to break apart GTEx oriented summary stats where the variant_id
    #contains the chr,pos,alt,ref in one columns so I broke it apart to go by position in the summary stats

    #Will need to modify based on non-GTEx summary stats!
    eQTL_Region=[]
    for index, row in eQTL_subset.iterrows():
        rowtoappend = row['variant_id'].split("_")[0:4]
        rowtoappend[1] = int(rowtoappend[1])
        rowtoappend.append(row['pval'])
        rowtoappend.insert(0, row['variant_id'])
        Zscore = row['slope'] / row['slope_se']
        rowtoappend.append(Zscore)
        eQTL_Region.append(rowtoappend)

    eQTL_Region = pd.DataFrame(eQTL_Region,columns=['variant_id','chr','pos','ref','alt','pval','Zscore'])

    SiteofIntrest = eQTL_Region[eQTL_Region.pval == eQTL_Region.pval.min()]
    SiteofIntrest = SiteofIntrest.iloc[0]
    pos = SiteofIntrest['pos']

    # return a T/F logical variable for each row to determine if the snp is in inside the 200Kb Region.
    isin200KB = eQTL_Region['pos'].between((pos - kbregion), (pos + kbregion), inclusive=False)

    Final200KBRegion = eQTL_Region[isin200KB]

    eQTL_Zscore=Final200KBRegion[['variant_id', 'Zscore']]

    eQTL_sites=Final200KBRegion['variant_id']

    # Save raw data files for the zscore and ld matrix for caviar to read in.
    np.savetxt(f'{python_dir}/tmpfiles/Zscore_chr{args.job_id}.txt', eQTL_Zscore, fmt="%s")
    np.savetxt(f'{python_dir}/tmpfiles/sites_chr{args.job_id}.txt', eQTL_sites, fmt="%s")

    #return eQTL_Zscore,eQTL_sites

def create_ld_matrix(raw_genotypes_path):
    GenotypeMatrix = np.loadtxt(raw_genotypes_path, delimiter='\t', unpack=True)
    # Create Empty LD Correlation Matrix to add correlation values to.
    LDMatrix = np.zeros([len(GenotypeMatrix), len(GenotypeMatrix)])

    # I will only traverse the upper triangular matrix with the diagonal to append LD correlation values
    for i in range(0, len(GenotypeMatrix)):
        for j in range(i, len(GenotypeMatrix)):
            # Genoi = pd.read_csv(Filepath, sep="\t", usecols=[(i + 1)],names=['snp1'])
            # Genoj = pd.read_csv(Filepath, sep="\t", usecols=[(j + 1)],names=['snp2'])
            # Calculate Pearson Correlation Coefficient
            Corr = stats.pearsonr(GenotypeMatrix[i], GenotypeMatrix[j])
            # #Append r to the index and the symetric index on the lower triangle
            LDMatrix[i][j] = Corr[0]
            LDMatrix[j][i] = Corr[0]

    #print(LDMatrix)
    filename = f"{python_dir}/tmpfiles/LDCorrelationMatrix_chr{args.job_id}.txt"
    np.savetxt(filename, LDMatrix, fmt='%f')

def check_filepath(filepath,typeoffile):
    try:
        file=open(filepath)
        file.close()
        return True
    except IOError:
        print(f"{typeoffile} is not found, please respecify the directory")
        exit(0)

#Main Class of the project
if __name__ == '__main__':
    #First will read in the parser arguments for python
    parser.add_argument("-v", "--vcf_file", help=" ")
    parser.add_argument("-g", "--gene_id", help=" ")
    parser.add_argument("-e", "--eQTL_Sum", help=" ")
    parser.add_argument("-w", "--window_size", help=" ")
    parser.add_argument('-id',"--job_id", help=" ")
    parser.add_argument("-o","--output_dir",help='')

    python_dir=os.path.dirname(os.path.realpath(__file__))

    args = parser.parse_args()

    #Check filepaths is they are able to be opened
    check_filepath(args.vcf_file,f"{args.vcf_file}")
    check_filepath(args.eQTL_Sum,f"{args.eQTL_Sum}")
    check_filepath(args.gene_id,f"{args.gene_id}")

    if(not os.path.isdir(f'{python_dir}/tmpfiles')):
        os.mkdir(f'{python_dir}/tmpfiles')
    #Create output directory to put gene eQTL finemapping posteriors in
    if(not os.path.isdir(args.output_dir)):
        os.mkdir(args.output_dir)
    if(not os.path.isdir(f"{args.output_dir}/rawgeneposteriors{args.job_id}")):
        os.mkdir(f"{args.output_dir}/rawgeneposteriors{args.job_id}")

    #Load the eQTL summary stats file.
    eQTL_SummStats=pd.read_csv(args.eQTL_Sum,sep=" ")

    #Load the gene list for the eQTL in the file to preform finemapping on.
    gene_id_list=np.loadtxt(args.gene_id, dtype='str')

    Complete_Posterior_List=[]
    #This will now iterate per gene in gene_id list and run the fine mapping test for each gene
    for gene in gene_id_list:
        #Subset the specific eQTL summart stats for the trait of intrest
        eQTL_subset=eQTL_SummStats[eQTL_SummStats['gene_id']==gene]

        #This will create a defined kb region around the lead snps around
        create_fine_map_region(eQTL_subset,args.window_size)

        print("Loading vcf file....")
        cmd=f'vcftools --gzvcf {args.vcf_file} --snps {python_dir}/tmpfiles/sites_chr{args.job_id}.txt --012 --out {python_dir}/tmpfiles/variants_chr{args.job_id}'
        #Extract vcf based on the sites we need for the current eQtl genes
        os.system(cmd)

        #This function will take in the raw gentic data and create the LD matrix around that region.
        create_ld_matrix(f'{python_dir}/tmpfiles/variants_chr{args.job_id}.012')

        print("Running CAVIAR analysis.....")
        cmd=f"~/bin/caviar/CAVIAR-C++/CAVIAR -l {python_dir}/tmpfiles/LDCorrelationMatrix_chr{args.job_id}.txt -z {python_dir}/tmpfiles/Zscore_chr{args.job_id}.txt -c 1 -f 1 -o {args.output_dir}/rawgeneposteriors{args.job_id}/chr{args.job_id}_{gene}"
        os.system(cmd)

        # os.system("ls tmpfiles")
        #Once caviar is done, save the results and delete the other files for next iteration.
        #cmd=f'rm {python_dir}/tmpfiles/LDCorrelationMatrix_chr{args.job_id}.txt {python_dir}/tmpfiles/Zscore_chr{args.job_id}.txt {python_dir}/tmpfiles/sites_chr{args.job_id}.txt {python_dir}/tmpfiles/variants_chr{args.job_id}*'
        #os.system(cmd)

        #We will take the posteriors from the finemapping and all the results to a large dataframe
        eQTL_gene_finemapping=pd.read_csv(f"{args.output_dir}/rawgeneposteriors{args.job_id}/chr{args.job_id}_{gene}_post", sep='\t')
        eQTL_gene_finemapping['gene']=gene
        Complete_Posterior_List.append(eQTL_gene_finemapping)
        del eQTL_gene_finemapping

    Complete_Posterior_List = pd.concat(Complete_Posterior_List)
    Complete_Posterior_List.to_csv(f'{args.output_dir}/Whole_Blood_Posteriors{args.job_id}.txt', sep="\t", index=False)
    # shutil.rmtree('tmpfiles')


from sklearn.neural_network import MLPClassifier

class MLPModel:
    def __init__(self, feature_count, features, ground_truth):
        self.feature_count = feature_count
        self.model_params = {
            "hidden_layer_sizes": (feature_count, 100, 2),
            "activation": 'relu',
            'solver': 'adam',
            "max_iter": 10,
            "verbose": True
        }
        self.model = MLPClassifier(self.model_params)
        self.features = features
        self.ground_truth = ground_truth

    def run(self):
        self.model.fit(self.features, self.ground_truth)

    def predict(self, features):
        return self.model.predict(features)


import copy
import os
import pathlib
from enum import Enum
from typing import List, Mapping, Optional, Union

import torch
import yaml

from archai.common import utils
from archai.common.config import Config
from archai.common.ordered_dict_logger import get_global_logger

logger = get_global_logger()

# Each tensor shape is list
# A layer can output multiple tensors so its shapes are TensorShapes
# list of all layer outputs is TensorShapesList]
TensorShape=List[Union[int, float]]
TensorShapes=List[TensorShape]
TensorShapesList=List[TensorShapes]

class ConvMacroParams:
    """Holds parameters that may be altered by macro architecture"""

    def __init__(self, ch_in:int, ch_out:int) -> None:
        self.ch_in, self.ch_out = ch_in, ch_out

    def clone(self)->'ConvMacroParams':
        return copy.deepcopy(self)

class OpDesc:
    """Op description that is in each edge"""

    def __init__(self, name:str, params:dict, in_len:int,
                 trainables:Optional[Mapping],
                 children:Optional[List['OpDesc']]=None,
                 children_ins:Optional[List[int]]=None)->None:
        self.name = name
        self.in_len = in_len
        self.params = params # parameters specific to op needed to construct it

        self.trainables = trainables # TODO: make this private due to clear_trainable
        # If op is keeping any child op then it should save it in children.
        # This way we can control state_dict of children.
        self.children = children
        self.children_ins = children_ins

    def clone(self, clone_trainables=True)->'OpDesc':
        cloned = copy.deepcopy(self)
        if not clone_trainables:
            cloned.clear_trainables()
        return cloned

    def clear_trainables(self)->None:
        self.trainables = None
        if self.children is not None:
            for child in self.children:
                child.clear_trainables()

    def state_dict(self)->dict:
        return  {
                    'trainables': self.trainables,
                    'children': [child.state_dict() if child is not None else None
                                 for child in self.children] \
                                     if self.children is not None else None
                }

    def load_state_dict(self, state_dict)->None:
        self.trainables = state_dict['trainables']
        c, cs = self.children, state_dict['children']
        assert (c is None and cs is None) or \
                (c is not None and cs is not None and len(c) == len(cs))

        # TODO: when c and cs are both none, zip throws an error that the
        # first argument should be iterable
        if (c is None and cs is None):
            return
        for cx, csx in utils.zip_eq(c, cs):
            if cx is not None and csx is not None:
                cx.load_state_dict(csx)

class EdgeDesc:
    """Edge description between two nodes in the cell
    """
    def __init__(self, op_desc:OpDesc, input_ids:List[int])->None:
        assert op_desc.in_len == len(input_ids)
        self.op_desc = op_desc
        self.input_ids = input_ids

    def clone(self, conv_params:Optional[ConvMacroParams], clear_trainables:bool)\
            ->'EdgeDesc':
        # edge cloning is same as deep copy except that we do it through
        # constructor for future proofing any additional future rules and
        # that we allow overiding conv_params and clearing weights
        e = EdgeDesc(self.op_desc.clone(), self.input_ids)
        # op_desc should have params set from cloning. If no override supplied
        # then don't change it
        if conv_params is not None:
            e.op_desc.params['conv'] = conv_params
        if clear_trainables:
            e.op_desc.clear_trainables()
        return e

    def clear_trainables(self)->None:
        self.op_desc.clear_trainables()

    def state_dict(self)->dict:
        return  {'op_desc': self.op_desc.state_dict()}

    def load_state_dict(self, state_dict)->None:
        self.op_desc.load_state_dict(state_dict['op_desc'])

class NodeDesc:
    def __init__(self, edges:List[EdgeDesc], conv_params:ConvMacroParams) -> None:
        self.edges = edges
        self.conv_params = conv_params

    def clone(self):
        # don't override conv_params or reset learned weights
        # node cloning is currently equivalent to deep copy
        return NodeDesc(edges=[e.clone(conv_params=None, clear_trainables=False)
                               for e in self.edges], conv_params=self.conv_params)

    def clear_trainables(self)->None:
        for edge in self.edges:
            edge.clear_trainables()

    def state_dict(self)->dict:
        return  { 'edges': [e.state_dict() for e in self.edges] }

    def load_state_dict(self, state_dict)->None:
        for e, es in zip(self.edges, state_dict['edges']):
            e.load_state_dict(es)

class AuxTowerDesc:
    def __init__(self, ch_in:int, n_classes:int, stride:int) -> None:
        self.ch_in = ch_in
        self.n_classes = n_classes
        self.stride = stride

class CellType(Enum):
    Regular = 'regular'
    Reduction  = 'reduction'

class CellDesc:
    def __init__(self, id:int, cell_type:CellType, conf_cell:Config,
                 stems:List[OpDesc], stem_shapes:TensorShapes,
                 nodes:List[NodeDesc], node_shapes: TensorShapes,
            post_op:OpDesc, out_shape:TensorShape, trainables_from:int)->None:

        self.cell_type = cell_type
        self.id = id
        self.conf_cell = conf_cell
        self.stems = stems
        self.stem_shapes = stem_shapes
        self.out_shape = out_shape
        self.trainables_from = trainables_from
        self.reset_nodes(nodes, node_shapes, post_op, out_shape)

    def clone(self, id:int)->'CellDesc':
        c = copy.deepcopy(self) # note that trainables_from is also cloned
        c.id = id
        return c

    def clear_trainables(self)->None:
        for stem in self.stems:
            stem.clear_trainables()
        for node in self._nodes:
            node.clear_trainables()
        self.post_op.clear_trainables()

    def state_dict(self)->dict:
        return  {
                    'id': self.id,
                    'cell_type': self.cell_type,
                    'stems': [s.state_dict() for s in self.stems],
                    'stem_shapes': self.stem_shapes,
                    'nodes': [n.state_dict() for n in self.nodes()],
                    'node_shapes': self.node_shapes,
                    'post_op': self.post_op.state_dict(),
                    'out_shape': self.out_shape
                }

    def load_state_dict(self, state_dict)->None:
        assert self.id == state_dict['id']
        assert self.cell_type == state_dict['cell_type']

        for s, ss in utils.zip_eq(self.stems, state_dict['stems']):
            s.load_state_dict(ss)
        self.stem_shapes = state_dict['stem_shapes']

        for n, ns in utils.zip_eq(self.nodes(), state_dict['nodes']):
            n.load_state_dict(ns)
        self.node_shapes = state_dict['node_shapes']

        self.post_op.load_state_dict(state_dict['post_op'])
        self.out_shape = state_dict['out_shape']

    def reset_nodes(self, nodes:List[NodeDesc], node_shapes:TensorShapes,
                    post_op:OpDesc, out_shape:TensorShape)->None:
        self._nodes = nodes
        self.node_shapes = node_shapes
        self.post_op = post_op
        self.out_shape = out_shape

    def nodes(self)->List[NodeDesc]:
        return self._nodes

    def all_empty(self)->bool:
        return len(self._nodes)==0 or all((len(n.edges)==0 for n in self._nodes))
    def all_full(self)->bool:
        return len(self._nodes)>0 and all((len(n.edges)>0 for n in self._nodes))

class ModelDesc:
    def __init__(self, conf_model_desc:Config, model_stems:List[OpDesc], pool_op:OpDesc,
                 cell_descs:List[CellDesc], aux_tower_descs:List[Optional[AuxTowerDesc]],
                 logits_op:OpDesc)->None:

        self.conf_model_desc = conf_model_desc
        conf_dataset = conf_model_desc['dataset']
        self.ds_ch:int = conf_dataset['channels']
        self.n_classes:int = conf_dataset['n_classes']
        self.params = conf_model_desc['params'].to_dict()
        self.max_final_edges:int = conf_model_desc['max_final_edges']

        self.model_stems, self.pool_op = model_stems, pool_op
        self.logits_op = logits_op

        self.reset_cells(cell_descs, aux_tower_descs)

    def reset_cells(self, cell_descs:List[CellDesc],
                    aux_tower_descs:List[Optional[AuxTowerDesc]])->None:
        assert len(cell_descs) == len(aux_tower_descs)
        # every cell should have unique ID so we can tell where arch params are shared
        assert len(set(c.id for c in cell_descs)) == len(cell_descs)

        self._cell_descs = cell_descs
        self.aux_tower_descs = aux_tower_descs

    def clear_trainables(self)->None:
        for stem in self.model_stems:
            stem.clear_trainables()
        for attr in ['pool_op', 'logits_op']:
            op_desc:OpDesc = getattr(self, attr)
            op_desc.clear_trainables()
        for cell_desc in self._cell_descs:
            cell_desc.clear_trainables()

    def cell_descs(self)->List[CellDesc]:
        return self._cell_descs

    def cell_type_count(self, cell_type:CellType)->int:
        return sum(1 for c in self._cell_descs if c.cell_type==cell_type)

    def clone(self)->'ModelDesc':
        return copy.deepcopy(self)

    def has_aux_tower(self)->bool:
        return any(self.aux_tower_descs)

    def all_empty(self)->bool:
        return len(self._cell_descs)==0 or \
             all((c.all_empty() for c in self._cell_descs))

    def all_full(self)->bool:
        return len(self._cell_descs)>0 and \
            all((c.all_full() for c in self._cell_descs))

    def state_dict(self)->dict:
        return  {
                    'cell_descs': [c.state_dict() for c in self.cell_descs()],
                    'model_stems': [stem.state_dict() for stem in self.model_stems],
                    'pool_op': self.pool_op.state_dict(),
                    'logits_op': self.logits_op.state_dict()
                }

    def load_state_dict(self, state_dict)->None:
        for c, cs in utils.zip_eq(self.cell_descs(), state_dict['cell_descs']):
            c.load_state_dict(cs)
        for stem, state in utils.zip_eq(self.model_stems, state_dict['model_stems']):
            stem.load_state_dict(state)
        self.pool_op.load_state_dict(state_dict['pool_op'])
        self.logits_op.load_state_dict(state_dict['logits_op'])

    def save(self, filename:str, save_trainables=False)->Optional[str]:
        if filename:
            filename = utils.full_path(filename)

            if save_trainables:
                state_dict = self.state_dict()
                pt_filepath = ModelDesc._pt_filepath(filename)
                torch.save(state_dict, pt_filepath)

            # save yaml
            cloned = self.clone()
            cloned.clear_trainables()
            utils.write_string(filename, yaml.dump(cloned))

        return filename

    @staticmethod
    def _pt_filepath(desc_filepath:str)->str:
        # change file extension
        return str(pathlib.Path(desc_filepath).with_suffix('.pth'))

    @staticmethod
    def load(filename:str, load_trainables=False)->'ModelDesc':
        filename = utils.full_path(filename)
        if not filename or not os.path.exists(filename):
            raise RuntimeError("Model description file is not found."
                "Typically this file should be generated from the search."
                "Please copy this file to '{}'".format(filename))

        logger.info({'final_desc_filename': filename})
        with open(filename, 'r') as f:
            model_desc = yaml.load(f, Loader=yaml.Loader)

        if load_trainables:
            # look for pth file that should have pytorch parameters state_dict
            pt_filepath = ModelDesc._pt_filepath(filename)
            if os.path.exists(pt_filepath):
                state_dict = torch.load(pt_filepath, map_location=torch.device('cpu'))
                model_desc.load_state_dict(state_dict)
            # else no need to restore weights

        return model_desc


class Node(object):
    """Create Node"""
    def __init__(self, data, children = None):
        """Initializes instance"""
        self.data = data
        self.children = children or []

class Tree(object):
    """Creat Tree"""
    def __init__(self, root = None):
        if self.root == None:
            self.root = self

def createTreeNodes(phrase, vocab, idx=0):
    phrase_list = [char for char in phrase]
    phrase_set = set(phrase_list)

    # create a dictionary of the vocab words
    vocab_dictionary = {}

    for char in phrase_set:
        for word in vocab:
            if word.startswith(char):
                if  vocab_dictionary.get(char):
                    vocab_dictionary[char].append(word)
                else:
                    vocab_dictionary[char] = [word]

    # creat node
    node = Node(phrase[idx])

    key = node.data

    words = vocab_dictionary.get(key)

    if words:
        print('words------->',words)
        # check to see if current phrase starts with words from dictionary
        for word in words:
            if phrase[idx:].startswith(word):
                node.children.append(word)
                print('child------------>', word)

        # Use len of each child and add on to index to analyze next phrase
        for child in node.children:
            new_idx = idx + len(child)
            createTreeNodes(phrase[new_idx:], vocab, idx = new_idx)

def parse(phrase, vocab, index=0):
    """return a set of all possible phrases using vocabulary words"""

    sentences = Tree(createTreeNodes(phrase, vocab))

    # Traverse tree depth first to find all possible sentences

vocab = {'i', 'a', 'ten', 'oodles', 'ford', 'inner', 'to', 'night', 'ate', 'noodles', 'for', 'dinner', 'tonight'}
createTreeNodes('iatenoodlesfordinnertonight', vocab)

    # # Break the given phrase into vocab words

    #     # iterate through phrase
    #         # add char into a temp_word container
    #         # check to see if temp_word in vocabulary set
    #         # if temp_sentence in vocabulary set
    #             # Add a space to temp_sentence

    #         # if temp word NOT in vocabulary
    #             # continue adding char to temp_sentence

    #     # Add phrase into sentence set

    # # Run through phrase from right to left

    # sentences=set()
    # temp_sentence = ""
    # temp_word = ""

    # for char in phrase:
    #     temp_word+= char

    #     if temp_word in vocab:
    #         temp_sentence += temp_word + " "
    #         temp_word = ""

    # print('sentence ====>', temp_sentence)

    # sentence = temp_sentence
    # sentences.add(sentence)
    # # sentences.add(read_rightToLeft(sentence))

    # for sentence in sentences:

    #     print('set of sentences....',sentence)

# class TestFunction(unittest.TestCase):
#     def test(self):
#         vocab = {'i', 'a', 'ten', 'oodles', 'ford', 'inner', 'to', 'night', 'ate', 'noodles', 'for', 'dinner', 'tonight'}
#         self.assertEqual(parse('iatenoodlesfordinnertonight', vocab),
#             {
#                     "i a ten oodles for dinner to night",
#                     "i a ten oodles for dinner tonight",
#                     "i a ten oodles ford inner to night",
#                     "i a ten oodles ford inner tonight",
#                     "i ate noodles for dinner to night",
#                     "i ate noodles for dinner tonight",
#                     "i ate noodles ford inner to night",
#                     "i ate noodles ford inner tonight"
#                 }
#                 )

# if __name__=="__main__":
#     unittest.main()


from pandas.core.frame import DataFrame
from sklearn.preprocessing import LabelEncoder, OrdinalEncoder
import numpy as np
import pandas as pd

data = pd.read_csv(r'./a.csv', index_col=0)
data_ = data.copy()
y = data.iloc[:, -1]
le = LabelEncoder()
le = le.fit(y)
lable = le.transform(y)
# ['No' 'Unknown' 'Yes']
print(le.classes_)
# [0 2 2 1]
print(lable)

data.iloc[:, -1] = LabelEncoder().fit_transform(data.iloc[:, -1])

#     Age     Sex Embarked  Survived
# id
# 0    22    male        S         0
# 1    23  female        C         2
# 2    66  female        S         2
# 3    77    male        S         1
print(data)

# 把还是字母的第二列和第三列也换为数字： [array(['female', 'male'], dtype=object), array(['C', 'S'], dtype=object)]
# print(OrdinalEncoder().fit(data_.iloc[:, 1:-1]).categories_)

data_.iloc[:, 1:-1] = OrdinalEncoder().fit_transform(data_.iloc[:, 1:-1])

#     Age  Sex  Embarked  Survived
# id
# 0    22  1.0       1.0         0
# 1    23  0.0       0.0         2
# 2    66  0.0       1.0         2
# 3    77  1.0       1.0         1
print(data_)
# 但是还不够好，例如舱门，应该使用哑变量，这样的变化能让算法彻底领悟三个值是没有可计算性的，是有你没我的不等概念。
# 也就是独热编码。
from sklearn.preprocessing import OneHotEncoder
# a = OneHotEncoder(categories='auto').fit_transform(data.iloc[:, 1:-1])
enc = OneHotEncoder(categories='auto').fit(data.iloc[:, 1:-1])
# ['x0_female' 'x0_male' 'x1_C' 'x1_S']
print(enc.get_feature_names())
result = enc.transform(data.iloc[:, 1:-1]).toarray()
#  性别分 2 中， Embarked 也分 2 种，所以一共 4 列数据。
# [[0. 1. 0. 1.]
#  [1. 0. 1. 0.]
#  [1. 0. 0. 1.]
#  [0. 1. 0. 1.]]
print(result)
#         0  1
# 0    male  S
# 1  female  C
# 2  female  S
# 3    male  S
# inverse = pd.DataFrame(enc.inverse_transform(result))
# print(inverse)
newdata = pd.concat([data, pd.DataFrame(result)], axis=1)
newdata.drop(['Sex', 'Embarked'], axis=1,inplace=True)

print(newdata)
newdata.columns = ["Age", "Survived", "x0_female", "x0_male", "x1_C", "x1_S"]
print(newdata)


from collections import defaultdict
class Solution:
    def uniquePathsWithObstacles(self, obstacleGrid: list)-> int:
        cache = defaultdict(dict)
        n = len(obstacleGrid)
        m = len(obstacleGrid[0])
        def paths(i, j):
            if j >= m or i >= n:
                return 0
            if obstacleGrid[i][j] ==1:
                return 0

            if i in cache and j in cache[i]:
                return cache[i][j]

            if j==m-1 and i==n-1:
                return 1

            cache[i][j] =  paths(i+1, j) + paths(i, j+1)
            return cache[i][j]

        return paths(0, 0)

print(Solution().uniquePathsWithObstacles([
  [0,0,0],
  [0,1,0],
  [0,0,0]
]))


import functools
import time
import logging
import sys
from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor

from bonobo.constants import BEGIN, END
from bonobo.execution.strategies.base import Strategy
from bonobo.structs.graphs import Graph

logger = logging.getLogger(__name__)

class ExecutorStrategy(Strategy):
    """
    Strategy based on a concurrent.futures.Executor subclass (or similar interface).

    """

    executor_factory = Executor

    def create_executor(self, graph: Graph):
        return self.executor_factory()

    def execute(self, graph: Graph, **kwargs):
        context = self.create_graph_execution_context(graph, **kwargs)
        context.write(BEGIN, (), END)

        futures = []

        with self.create_executor(graph) as executor:
            try:
                context.start(self.get_starter(executor, futures))
            except Exception:
                logger.critical(
                    "Exception caught while starting execution context.",
                    exc_info=sys.exc_info(),
                )

            while not context.stopped:
                try:
                    context.tick()
                except KeyboardInterrupt:
                    logger.warning(
                        "KeyboardInterrupt received. Trying to terminate the nodes gracefully."
                    )
                    context.kill()
                    break

            context.stop()

        return context

    def get_starter(self, executor, futures):
        def starter(node):
            @functools.wraps(node)
            def _runner():
                logger.info("Running node %s", node)
                try:
                    with node:
                        node.loop()
                except Exception:
                    logger.critical(
                        "Critical error in threadpool node starter.",
                        exc_info=sys.exc_info(),
                    )

            try:
                futures.append(executor.submit(_runner))
            except Exception:
                logger.critical("futures.append", exc_info=sys.exc_info())

        return starter

class ThreadPoolExecutorStrategy(ExecutorStrategy):
    executor_factory = ThreadPoolExecutor

    def create_executor(self, graph: Graph):
        num_workers = len(graph)
        return self.executor_factory(
            max_workers=num_workers,
            thread_name_prefix=graph.name or str(graph),
            initializer=lambda: logger.info(
                "Starting threadpool for graph %s with %s workers", graph, num_workers
            ),
        )

class ProcessPoolExecutorStrategy(ExecutorStrategy):
    executor_factory = ProcessPoolExecutor

    def create_executor(self, graph: Graph):
        return self.executor_factory(max_workers=len(graph))


import numpy as np

def getSamples( f, n, xmin=0, dx=1.0, fddx=0.1 ):
    ddx=dx*fddx
    xs = np.arange( xmin-dx, xmin+dx*(n+2)+1e-8, dx )
    xs[0 ]=xs[ 1]+ddx
    xs[-1]=xs[-2]-ddx
    #print xs
    ys  = f(xs)
    dy0 = (ys[0 ] - ys[1 ])/ddx
    dy1 = (ys[-2] - ys[-1])/ddx
    ys[ 0] = ys[2 ] - dy0*2*dx
    ys[-1] = ys[-3] + dy1*2*dx
    xs[0 ]=xs[ 1]-dx
    xs[-1]=xs[-2]+dx
    #print "xs ", xs
    #print "ys ", ys
    return ys, xs

def getSamplesDeriv( f, n, xmin=0, dx=1.0, fddx=0.1 ):
    ddx=dx*fddx
    xs  = np.arange( xmin, xmin+dx*(n+1)+1e-8, dx )
    ys  = f(xs-ddx)
    ys_ = f(xs+ddx)
    dys = (ys_-ys)/(2*ddx)
    #ys  = 0.5*(ys+ys_)
    ys  = f(xs)
    return ys, dys, xs

def hermite(x, y0,y1, dy0,dy1 ):
    y01 = y0-y1;
    return (     y0
        +x*(           dy0
        +x*( -3*y01 -2*dy0 - dy1
        +x*(  2*y01 +  dy0 + dy1 ))));

def evalValTable( x, ys, xstep ):
    #x+=xstep*2
    invstep = 1/xstep
    s = x*invstep + 1
    i = s.astype(dtype=np.int)
    #print "i ", i
    y0  = ys[i  ]
    y1  = ys[i+1]
    dy0 = 0.5 * ( ys[i+1] - ys[i-1] )
    dy1 = 0.5 * ( ys[i+2] - ys[i  ] )
    return hermite( s-i, y0,y1, dy0,dy1 )

def evalValTableDeriv( x, ys, dys, xstep ):
    invstep = 1/xstep
    s = x*invstep
    i = s.astype(dtype=np.int)
    return hermite( s-i, ys[i],ys[i+1], dys[i]*xstep, dys[i+1]*xstep )


#   Translation information
#       Common translations
#       Idioms and phrases
#       Translation examples (context/category, time period, author, citation)

from categories import Category,TimePeriod

class TranslationInfo:
    def __init__(self):
        self.common_translations = []
        self.idioms_and_phrases = []
        self.translation_examples = []

class TranslationExample:
    def __init__(self,foreign,english,cat=Category.COMMON,time=TimePeriod.TWENTYFIRST,auth="",cite=""):
        self.category = cat
        self.foreign_language = foreign
        self.english_language = english
        self.time_period = time
        self.author = auth
        self.citation = cite


import requests
import logging
from bs4 import BeautifulSoup

def main():
    try:
        URL = 'https://www.iso-ne.com/'
        page = requests.get(URL)
        soup = BeautifulSoup(page.content, 'html.parser')

        td_element = soup.find('td', class_ ='homepage-price')
        lpm_text = td_element.text
        lpm_value = lpm_text[1:]
        print(lpm_value)

    except Exception as e:
        raise e

if __name__ == "__main__":
    main()


import tkinter as tk
from tkinter.filedialog import askopenfilename, askdirectory

#        self.files_found = 0

class Filemover:
    def __init__(self):
        pass

    def value_hub(self):
        pass

    def move_files(self):
        pass
        # print(f"{option} {extension} files from:\n{source}\n to:\n {destination}")s

class FrameGui:
    def __init__(self):
        self.root = tk.Tk()
        self.main_state = Buttons()
        # self.frame1 = tk.LabelFrame(self.root, text="test", padx=20,pady=20)

        self.frame1 = tk.Frame(self.root, padx=20,pady=20, bg = '#255869', bd = 0)
        self.frame1.grid(row = 0, column = 0)
        Options(root=self.root, state=self.main_state)
        Extensions(frame=self.frame1,state=self.main_state)
        Subfolders(state=self.main_state)
        Source(state=self.main_state)
        # Destination(state=self.main_state)
        self.config()
        self.root.mainloop()

    def config(self):
        self.root.title("Filemover")
        self.root.iconphoto(False, tk.PhotoImage(file='./assets/folder.png'))
        # self.root.pack(fill="both", expand=True)
        # self.root.geometry('900x450')
        # self.root.minsize(300, 300)
        # self.frame1.grid_rowconfigure(0, weight=1)
        # self.frame1.grid_columnconfigure(0, weight=1)

class Extensions:
    def __init__(self, frame=None, state=None):
        self.frame = frame
        self.state = state
        self.check_PNG = tk.BooleanVar()
        self.check_JPG = tk.BooleanVar()
        self.check_GIF = tk.BooleanVar()
        self.check_PSD = tk.BooleanVar()
        self.check_MP3 = tk.BooleanVar()
        self.check_WAV = tk.BooleanVar()
        self.check_ZIP = tk.BooleanVar()
        self.check_RAR = tk.BooleanVar()
        self.check_DOC = tk.BooleanVar()
        self.check_XLS = tk.BooleanVar()
        self.check_PDF = tk.BooleanVar()
        self.avaliable_extensions = [self.check_PNG,
                                     self.check_JPG,
                                     self.check_GIF,
                                     self.check_PSD,
                                     self.check_MP3,
                                     self.check_WAV,
                                     self.check_ZIP,
                                     self.check_RAR,
                                     self.check_DOC,
                                     self.check_XLS,
                                     self.check_PDF]
        self.extension_names = ["PNG", "JPG", "GIF", "PSD",
                                "MP3", "WAV", "ZIP", "RAR", "DOC", "XLS", "PDF"]
        self.create_checkboxes()

    def checkbox_value(self):
        checkboxes_selected = []
        for _ext in self.avaliable_extensions:
            checkboxes_selected.append(_ext.get())
        result = [x for x, y in zip(
            self.extension_names, checkboxes_selected) if y == True]
        self.state.extension = result

    def create_checkboxes(self):
        cn = 0
        for C in self.avaliable_extensions:
            C = tk.Checkbutton(self.frame, text=self.extension_names[cn], variable=self.avaliable_extensions[cn], anchor="w",
                               onvalue=1, offvalue=0, height=1, width=5, padx=14, command=self.checkbox_value)
            C.grid(row=cn, column=1, sticky=tk.E+tk.N)
            cn += 1

class Options:
    def __init__(self, root=None, state=None):
        self.root = root
        self.state = state
        self.option = tk.StringVar()
        self.dropdown()

    def send_option(self, selectx):
        self.state.file_option = selectx

    def dropdown(self):
        W = tk.OptionMenu(self.root, self.option,
                          "Copy to", "Move to", "Delete", command=self.send_option)
        self.option.set("Choose option")
        W.grid(row=14, column=7,sticky=tk.W+tk.S)

class Source:
    def __init__(self, state=None):
        self.state = state
        self.create_buttons()
    pass

    def get_source(self):
        directory = askdirectory()
        self.state.source=directory
        print(directory)

    def create_buttons(self):
        B1 = tk.Button(text="get src", anchor="nw", command=self.get_source)
        B1.grid(row=2, column=5, padx=14, sticky=tk.W+tk.S)

class Destination:
    def __init__(self, state=None):
        self.state = state
    pass

class Subfolders:
    def __init__(self, state=None):
        self.state = state
        self.variable = tk.BooleanVar()
        self.create_checkboxes()
    def checkbox_value(self):
        self.state.subfolders = self.variable.get()
    def create_checkboxes(self):
        C = tk.Checkbutton(text="check subfolders", variable=self.variable, anchor="nw",
                            onvalue=1, offvalue=0, height=1, width=5, padx=14, command=self.checkbox_value)
        C.grid(row=1, column=3)

class Buttons:
    def __init__(self):
        self.extension = []
        self.file_option = "Not chosen"
        self.source = False
        self.destination = False
        self.subfolders = False
        self.create_buttons()

    def check_state(self):
        print(self.extension)
        print(self.file_option)
        print(self.subfolders)

    def check_state_elaborate(self):
        if self.extension != [] and self.file_option == "Not chosen":
            print("choose file option")
        elif self.file_option == "Not chosen" and self.extension == []:
            print("choose file option and file extension")
        elif self.file_option != "Not chosen" and self.extension == []:
            print("choose file extension")
        elif self.file_option != "Not chosen" and self.extension != []:
            print(self.extension)
            print(self.file_option)

    def create_buttons(self):
        B1 = tk.Button(text="Start", command=self.check_state_elaborate)
        B1 = tk.Button(text="Start", command=self.check_state)
        B1.grid(row=12, column=1, padx=14, sticky=tk.W+tk.S)

if __name__ == '__main__':
    FrameGui()


from django.urls import path

from . import views

urlpatterns = [
    path('login/', views.login, name='index'),
    path('register/', views.RegisterView.as_view(), name='auth_register'),
    # path('otp/', views.otpgen, name='otp'),
    path('verifyotp/', views.OtpView.as_view(), name='verifyotp'),
    path('customers/', views.Customers.as_view(), name='customers'),
    # path('generateotp/', views.generateOtp, name='generateotp'),
    path('owners/', views.Owners.as_view(), name='owners'),
    path('logout/', views.Logout.as_view(), name='logout'),
]


import torch
import torch.nn as nn
import torch.nn.functional as F
from .log_squared_loss import LogSquaredLoss
from .median_MAE import MedianMAELoss

def tanh_scaled(x):
    x_tanh = torch.tanh(x)
    x_tanh_scaled = (1 + x_tanh) / 2
    return x_tanh_scaled

class LWMIntLayersL1Loss(nn.Module):
    def __init__(self, pixel_weight=2,activation=tanh_scaled):
        super(LWMIntLayersL1Loss, self).__init__()
        self.l1 = nn.L1Loss(reduction="none")
        self.pixel_weight = pixel_weight
        self.activation = activation

    def forward(self, target, output_mask, target_mask, lwm_op):
        op_label = output_mask.argmax(dim=1).unsqueeze(1)
        weight = torch.eq(op_label, target_mask.unsqueeze(1)).float()
        weight[weight == 0] = self.pixel_weight

        lwm_l1_loss = 0.0
        for lwm_layer_op in lwm_op:
            target_resized = F.interpolate(
                target, (lwm_layer_op.shape[2], lwm_layer_op.shape[3])
            )
            weight_resized = F.interpolate(
                weight, (lwm_layer_op.shape[2], lwm_layer_op.shape[3])
            )
            lwm_layer_l1 = (
                self.l1(lwm_layer_op, target_resized) * weight_resized
            ).mean()
            lwm_l1_loss += lwm_layer_l1
        return lwm_l1_loss, weight


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import math
import sympy as sym

x = sym.symbols('x')
f_expr = x - 1 / (3 + sym.sin(3.6 * x))
dfdx_expr = sym.diff(f_expr, x)

dfdx = sym.lambdify([x], dfdx_expr)

def tangents_method(f, dfdx, x, eps):
    f_value = f(x)
    while abs(f_value) > eps:
        try:
            x -= float(f_value) / dfdx(x)
        except ZeroDivisionError:
            raise SystemExit('Zero division')
        f_value = f(x)
    return x

def f(x):
    return x - 1 / (3 + math.sin(3.6 * x))

if __name__ == "__main__":
    solution = tangents_method(f, dfdx, x=1, eps=0.001)

    print("Result: {root}".format(root=solution))


"""Монстры."""
from doors_game.classes import Monster

# монстры 1 уровня
GOBLIN = Monster('Увечный гоблин', 1, 'Нельзя недооценивать калеку', image='monsters/1_goblin')
KOSTIN = Monster('Мистер Костин', 1.2, 'Скелет в чьём-то шкафу', image='monsters/1_kostin')
KRISOTKA = Monster('Молотая крысотка', 1.4, 'Опасная милашка', image='monsters/1_krisotka')
PLANT = Monster('Огорошенная трава', 1, 'Даже если умудришься проиграть - смывка автоматическая!', image='monsters/trava')
# монстры 2 уровня
HORSE = Monster('Конь андедный', 1.2, 'Жеребец апокалипсиса', image='monsters/2_horse')
LEPROKON = Monster('Лепрокон', 1.4, 'Сбежал из лепрозория', image='monsters/2_leprokon')
SLIZ = Monster('Сочащаяся слизь', 1.6, 'Мерзкая жижа', image='monsters/2_sliz')
# монстры 3 уровня
GARPISTKI = Monster('Гарпистки', 1.4, 'Песни на заказ', image='monsters/3_garpistki')
RIGACHU = Monster('Рыгачу', 1.6, 'Отвратительная атака', image='monsters/3_rigachu')
UTKONTIKORA = Monster('Утконтикора', 1.8, 'Родом из Австралии', image='monsters/3_utkontikora')
# монстры 4 уровня
GAZEBO = Monster('Газебо', 1.6, 'Обычная беседка...', image='monsters/4_gazebo')
LAGUSHKI = Monster('Улётные лягушки', 1.8, 'Летающие земноводные', image='monsters/4_lagushki')
PITBUL = Monster('Питбуль', 2, 'Осторожно - злая собака!', image='monsters/4_pitbul')
# монстры 5 уровня
AMAZONKA = Monster('Амазонка', 1.8, 'Подружка Конана-варвара', image='monsters/5_amazonka')
LICESOS = Monster('Лицесос', 2, 'Этот поцелуй ты не забудешь...', image='monsters/5_licesos')
MADEMONUAZELI = Monster('Мадемонуазели', 2.2, 'Светские львицы', image='monsters/5_mademonuazeli')
# монстры 6 уровня
NOS = Monster('Блуждающий нос', 2, 'С Гоголем на короткой ноге', image='monsters/6_nos')
ORKI = Monster('3872 орка', 2.2, 'Орки. 3872 шт. Ни больше, ни меньше', image='monsters/6_orki')
TROLL = Monster('Сетевой тролль', 2.4, 'Обитает на просторах Интернета', image='monsters/6_troll')
# монстры 7 уровня
BIGFUT = Monster('Бигфут', 2.2, 'Покупает обувь под заказ', image='monsters/7_bigfut')
VAMPIR = Monster('Закос под вампира', 2.4, 'Пересмотрел фильмов про Дракулу...', image='monsters/7_vampir')
YAZICHDEMON = Monster('Языческий демон', 2.6, 'Свободно владеет десятью языками', image='monsters/7_yazichdemon')
# монстры 8 уровня
BROILER = Monster('Сердитый бройлер', 2.4, 'Узнал, что не сможет летать', image='monsters/8_broiler')
GOLEM = Monster('Обдолбанный голем', 2.6, 'Смотрит мультики без телевизора', image='monsters/8_golem')
UZAS = Monster('Невыразимо жуткий неописуемый ужас', 2.8, 'Страх, как он есть', image='monsters/8_uzas')
# монстры 9 уровня
BRATIA = Monster('Бледные братья', 2.6, 'Джентельмены удачи', image='monsters/9_bratia')
GIPOGRIF = Monster('Гиппогриф', 2.8, 'Плод необычного союза', image='monsters/9_gipogrif')
TSAR = Monster('Царь Тут', 3, 'И Тут и Там', image='monsters/9_tsar')
# монстры 10 уровня
BULROG = Monster('Бульрог', 2.8, 'Из самой преисподней', image='monsters/10_bulrog')
DRAKON = Monster('Плутониевый дракон', 3, 'Бурей рожденный, с блондинкой обручённый', image='monsters/10_drakon')
KALMADZILA = Monster('Кальмадзилла', 3.2, 'Бывшая жена Паладина', image='monsters/10_kalmadzila')
# финальный босс
CTHULHU = Monster('Великий и Ужасный Ктулху', 3.5, 'Пощады не жди!', image='monsters/cthulhu')

MONSTERS = {
    1: (GOBLIN, KOSTIN, KRISOTKA),
    2: (HORSE, LEPROKON, SLIZ),
    3: (GARPISTKI, RIGACHU, UTKONTIKORA),
    4: (GAZEBO, LAGUSHKI, PITBUL),
    5: (AMAZONKA, LICESOS, MADEMONUAZELI),
    6: (NOS, ORKI, TROLL),
    7: (BIGFUT, VAMPIR, YAZICHDEMON),
    8: (BROILER, GOLEM, UZAS),
    9: (BRATIA, GIPOGRIF, TSAR),
    10: (BULROG, DRAKON, KALMADZILA)
}


import torch
import torch.distributed as dist
import torch.nn as nn
from torch.nn.parallel import DistributedDataParallel
import argparse
from tqdm import tqdm
parser = argparse.ArgumentParser()
parser.add_argument("--local_rank", type=int,default = 1)
args = parser.parse_args()
print(args)
model = nn.Linear(10,10)
x = torch.randn((1000,10))
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend='nccl',
                                         init_method='env://')
model = torch.nn.parallel.DistributedDataParallel(model,
                                                      device_ids=[args.local_rank],
                                                      output_device=args.local_rank)
model.cuda()
for i in tqdm(x):
    model.forward(i.unsqueeze(0).cuda())
print('end')


import csv
import tkinter
from tkinter import filedialog
import os
import datetime
import keras
import numpy as np
from numpy import genfromtxt
from keras.models import Sequential
from keras.layers import Dense, Dropout, Embedding, LSTM, Bidirectional

root = tkinter.Tk()
root.withdraw()
filez = filedialog.askopenfilenames(parent=root,title='SELECT ALL FEATURE files')
fileList = list(filez)

lens = []   		# list of swing lengths
for i in range(0, len(fileList)):		# go thru all swing files to get max length
	with open(fileList[i]) as csvDataFile:
		csvReader = csv.reader(csvDataFile)
		Hr = list( csv.reader(csvDataFile))
		lens.append(len(Hr))

ml = int(max(lens))
numFiles = len(fileList)
output_array = np.zeros((numFiles,ml,3))  # train data is an array of (swingFiles, maxLength, 3)
classes = np.zeros((numFiles,2))
netCount = 0

for i in range(0,len(fileList)):
	Hr= genfromtxt(fileList[i], delimiter=',')
	result = np.zeros((ml,3))
	data = Hr[:,0:3]
	ting = Hr[0,4:6]
	if 10 in ting:  # 10 means net
		#print(ting)
		netCount=netCount+1
		continue
	else:
		classes[i,:] = ting
		result[:data.shape[0],:data.shape[1]] = data
		output_array[i] = result

print("This is the netcount",netCount)
#output_array2 = np.zeros((numFiles-netCount,ml,3))
goodshots = numFiles -netCount
newArray = output_array[0:goodshots,:,:]
newY = classes[0:goodshots,:]
print("Good shots array is ", newArray.shape)
print("New Y train shape is", newY.shape)

y_train = np.random.randint(2, size=(numFiles, 2))
y_train2 = np.random.randint(10, size=(numFiles, 2))
y_train2 = classes

# Convert labels to categorical one-hot encoding
ylabels = keras.utils.to_categorical(y_train2, num_classes=10)

x_train = output_array
print("the shape of x_train array: ",x_train.shape)
print("the shape of y_train array: ",y_train.shape)
print("and the shape of new y train is:", classes.shape)
input0 = x_train[0]

# Generate dummy validation data
x_val = np.random.random((100, ml, 3))
y_val = np.random.random((100, 2))

max_features = 20000
# cut texts after this number of words
# (among top max_features most common words)
# maxlen = 100
batch_size = 2
batch_size=1

model = Sequential()
model.add(LSTM(32, return_sequences=True, stateful=True,
               batch_input_shape=(batch_size, ml, 3)))
model.add(LSTM(32, return_sequences=True, stateful=True))
model.add(LSTM(32, stateful=True))
#model.add(Dense(2, activation='sigmoid'))
model.add(Dense(2, activation='tanh'))

# try sgd optmizer maybe
model.compile(loss='categorical_crossentropy',
				optimizer='rmsprop',
				metrics=['accuracy'])

model.summary()

print('Train...')
#model.fit(x_train, y_train, batch_size=1, epochs=10)
model.fit(x_train, y_train2, batch_size=1, epochs=5)

# model.fit(x_train, y_train,
          # batch_size=batch_size,
          # epochs=4, validation_data=(x_val, y_val))


#!/usr/bin/env python
# coding:utf-8

import os

def ListFilesToTxt(dir, file, wildcard, recursion):
    exts = wildcard.split(" ")
    for root, subdirs, files in os.walk(dir):
        for name in files:
            for ext in exts:
                if (name.endswith(ext)):
                    name = os.path.join(rootdir, name)
                    file.write(name + "\n")
                    break
            if (not recursion):
                break

def Test():
    dir = os.getcwd()
    outfile = "binaries.txt"
    wildcard = ".jpg"
    file = open(outfile, "w")
    if not file:
        print("cannot open the file %s for writing" % outfile)
    ListFilesToTxt(dir, file, wildcard, 1)

    file.close()

rootdir = os.getcwd()
print(rootdir)
Test()

f = open("binaries.txt")  # 返回一个文件对象
line = f.readline()  # 调用文件的 readline()方法
print
line,

line = f.readline()  # 调用文件的 readline()方法
print
line


#!/usr/bin/env python3.5
#
# Simulate a simple market participant.
#
# Usage: client-participant.py PrivateChannelHostname PrivateChannelPort
#
# http://codingchallenge.wood.cz/

import asyncio
import datetime
import json
import sys
import time
from typing import Any, Dict

import utils

async def main():
    # Connect to the server
    assert len(sys.argv) == 3, 'Usage: client-participant.py PrivateChannelHostname PrivateChannelPort'
    host = sys.argv[1]
    port = int(sys.argv[2])
    reader, writer = await asyncio.open_connection(host, port)

    # Set up stdin
    userInput = asyncio.Queue()

    def onUserInput():
        userInput.put_nowait(sys.stdin.readline())

    loop = asyncio.get_event_loop()
    loop.add_reader(sys.stdin.fileno(), onUserInput)

    # Log incoming data
    bgLogger = asyncio.ensure_future(readIncomingData(reader))

    # Run Read-Eval-Print loop.
    try:
        await repl(userInput, writer)
    finally:
        bgLogger.cancel()
        writer.close()

async def readIncomingData(reader: asyncio.StreamReader) -> None:
    async for line in utils.LineReader(reader):
        print('\n<{!s} received {!r}>\n'.format(datetime.datetime.now(), line))

async def repl(userInput: asyncio.Queue, writer: asyncio.StreamWriter) -> None:
    while True:
        print('1: create order, 2: cancel order, q: quit')
        command = await prompt(userInput, 'Command?')

        if command == '1':
            side = await prompt(userInput, 'Side (BUY or SELL)?')
            assert side in {'BUY', 'SELL'}
            price = int(await prompt(userInput, 'Price?'))
            quantity = int(await prompt(userInput, 'Quantity?'))
            orderId = int(time.time() * 10) - 14617000000
            print('Assigning order ID', orderId)
            await sendMessage(writer, {
                'message': 'createOrder',
                'orderId': orderId,
                'side': side,
                'price': price,
                'quantity': quantity,
            })
        elif command == '2':
            orderId = await prompt(userInput, 'Order ID to cancel?')
            await sendMessage(writer, {
                'message': 'cancelOrder',
                'orderId': int(orderId),
            })
        elif command == 'q':
            break
        else:
            print('Unknown commad!')

async def prompt(userInput: asyncio.Queue, p: str) -> str:
    ''' Prompt the user to enter a line and wait for their response. '''
    sys.stdout.write(p + ' ')
    sys.stdout.flush()
    line = await userInput.get()
    return line.rstrip('\n')

async def sendMessage(writer: asyncio.StreamWriter, msg: Dict[str, Any]) -> None:
    ''' Encode and send a message to the server. '''
    data = json.dumps(msg)
    print('\n<{!s} sending {!r}>\n'.format(datetime.datetime.now(), data))
    writer.write(data.encode('utf-8') + b'\n')
    await writer.drain()

if __name__ == '__main__':
    loop = asyncio.get_event_loop()
    loop.run_until_complete(main())


import pickle
import os
import queue
import threading
import time
import numpy as np
from PyQt5 import QtCore
import serial
from MainWinDisplay import InerIoInfo, InerIoInfoParse, InerRunningPlanInfo, InerRunningPlanParse, InerSduInfo, InerSduInfoParse, InerValueStatsHandler, InerValueStatsRst
from MsgParse import Ato2tsrsParse, Atp2atoParse, Tsrs2atoParse
from TcmsParse import Ato2TcmsCtrl, Ato2TcmsState, MVBParse, MVBParseContentException, Tcms2AtoState
from ConfigInfo import ConfigFile

_sentinel = object()                                 # Object that signals shutdown
workQueue = queue.Queue(10000)
real_curve_buff = 50                                 # 每次绘图话添加的点数
real_curve_all_buff = 50000                          # 每次绘画总点数
newPaintList = np.zeros([5, real_curve_buff],dtype=int)        # 新添加的参数
paintList = np.zeros([5, real_curve_all_buff],dtype=int)       # 初始化1000个点用于绘图

class SerialRead(threading.Thread, QtCore.QObject):

    def __init__(self, name, serialHandle=serial.Serial):
        threading.Thread.__init__(self)
        self.name = name
        self.handle = serialHandle
        self.handle.set_buffer_size(4096)
        self.runningFlag = False
        super(QtCore.QObject, self).__init__()

    # 串口成功打开才启动此线程
    def run(self):
        while self.runningFlag:
            # 有数据就读取
            try:
                lineBytes = self.handle.read_until()  # 串口设置，测试时注释 默认终止符\n
            except Exception as err:
                self.setThreadEnabled(False)
            # 若队列未满，则继续加入:
            if not workQueue.full():
                workQueue.put(pickle.loads(pickle.dumps(lineBytes)), block=False, timeout=0.1)   # 必须读到数据
            else:
                time.sleep(0.1)
        # 发送停止信号
        workQueue.put(_sentinel)

    # 允许线程
    def setThreadEnabled(self, en=bool):
        self.runningFlag = en
        if en:
            pass
        elif self.handle and self.handle.is_open:
            self.handle.cancel_read()
            time.sleep(0.1)
            self.handle.reset_input_buffer()
            self.handle.close()
        else:
            pass

class RealPaintWrite(threading.Thread, QtCore.QObject):

    patShowSignal = QtCore.pyqtSignal(tuple)
    planShowSignal = QtCore.pyqtSignal(InerRunningPlanInfo, int)
    ioShowSignal = QtCore.pyqtSignal(str,str,InerIoInfo)
    sduShowSignal = QtCore.pyqtSignal(InerSduInfo)
    mvbShowSignal = QtCore.pyqtSignal(Ato2TcmsCtrl, Ato2TcmsState, Tcms2AtoState)

    def __init__(self, filepath=str, filefmt=str, portname=str):
        threading.Thread.__init__(self, )
        super(QtCore.QObject, self).__init__()
        self.portname = portname
        self.filepath = filepath  # 记录文件路径
        self.logFile = filepath + 'ATO记录中' + str(int(time.time())) + '.txt'  # 纪录写入文件
        self.logBuff = ''
        self.filename = self.getFileNameFromFmt(filefmt)  # 文件格式化命名
        self.cfg = ConfigFile()
        # 周期开始时间
        self.cycle_os_time_start = 0
        self.cycle_os_time_end = 0
        self.newPaintCnt = 0
        # 解析相关
        self.ioInfoParser = InerIoInfoParse()
        self.sduInfoParser = InerSduInfoParse()
        self.rpParser = InerRunningPlanParse()
        self.mvbParser = MVBParse()
        self.atp2atoParser = Atp2atoParse()
        self.ato2tsrsParser = Ato2tsrsParse()
        self.tsrs2atoParser = Tsrs2atoParse()
        # 解析结果
        self.a2tCtrlObj  = None
        self.a2tStatObj  = None
        self.t2aStatObj  = None
        self.atpatoMsgObj= None
        self.sduInfoObj  = None
        self.rpInfoObj   = None
        self.ioInfoObj   = None
        # 静态变量
        self.cycleNumStart = '0'
        self.count = 0
        self.last_mean = 0
        self.max_slot_cycle = 0
        self.max_slot = 0
        self.mean_slot = 0
        self.min_slot = 0
        self.timeContentStr = ''
        self.fsm = []
        self.sc_ctrl = []
        self.stoppoint = []
        self.timeStatictics = ()
        # 实时监控
        self.cycleStat = InerValueStatsRst()
        self.atpTimeStat = InerValueStatsRst()
        self.statisticHandle = InerValueStatsHandler()

    def newCyclePreProcess(self):
        # 创建周期后应重置ATP/ATO解析模块解析结果
        # 文件中可能有多个ATP/ATO消息时需要均解析，直到所有P->以及O->P解析完成后周期结束才能重置
        # 因为不同消息中可能带有差异项的包，需要在一个周期内均保留解析结果，在周期头或周期尾重置
        self.atp2atoParser.resetMsg()
        # MVB总是覆盖解析
        self.mvbParser.resetPacket()
        # 每周期重置更新标志
        self.rpParser.reset()
        # 重置测速测距
        self.sduInfoParser.reset()
        # 重置IO信息
        self.ioInfoParser.reset()
        # 重置ATO-TSRS消息
        self.ato2tsrsParser.resetMsg()
        self.tsrs2atoParser.resetMsg()

    # 串口成功打开才启动该线程
    def run(self):
        # 打开文件等待写入logFile
        with open(self.logFile, 'w', encoding='utf-8') as f:
            while True:
                if not workQueue.empty():
                    lineBytes = workQueue.get(block=False, timeout=0.1)
                    # 收到信号进行停止
                    if lineBytes is _sentinel:
                        break
                    else:
                        lineText = lineBytes.decode('gbk', errors='ignore').strip()
                        # 防止数据处理过程中被刷新
                        self.fileWrite(lineText, f) # 测试时关闭
                        self.lineProcessPaint(lineText)
                else:
                    time.sleep(0.1)
            # 清空缓存
            self.fileFlush(f)
        os.rename(self.logFile, self.filepath + self.filename)

    # 按行写入文件
    def fileWrite(self, line, f):
        try:
            f.write(line+'\n')
        except Exception as err:
            print('write info err' + str(time.time()))
            print(err)

    # 立即写入用于关断时候
    def fileFlush(self, f):
        # 目前预留不使用文件缓存
        for item in self.logBuff:
            f.write(item+'\n')
        f.flush()
        print("file exit flush!")

    # 每次测试完后重命名文件
    def getFileNameFromFmt(self, fmt=str):
        # 保存文件，立即更新文件名
        stuctLocTime = time.localtime(time.time())
        try:
            fmt = fmt.replace('%Y', str(stuctLocTime.tm_year), 1)
            fmt = fmt.replace('%M', str(stuctLocTime.tm_mon), 1)
            fmt = fmt.replace('%D', str(stuctLocTime.tm_mday), 1)
            fmt = fmt.replace('%h', str(stuctLocTime.tm_hour), 1)
            fmt = fmt.replace('%m', str(stuctLocTime.tm_min), 1)
            fmt = fmt.replace('%s', str(stuctLocTime.tm_sec), 1)
            fmt = fmt.replace('%N', self.portname, 1)
        except Exception as err:
            fmt = 'ATO' + str(stuctLocTime.tm_year) + str(stuctLocTime.tm_mon) + str(
                stuctLocTime.tm_mday) \
                          + str(stuctLocTime.tm_hour) + str(stuctLocTime.tm_min) + str(
                stuctLocTime.tm_sec) \
                          + self.portname
        return fmt

    # 处理和绘图
    def lineProcessPaint(self, line):
        # 更新绘制情况
        self.paintArrayUpdate()

        # 匹配速度曲线信息
        if self.patCycleCommonResearch(line):
            return

        # 匹配MVB信息
        if self.patMvbResearch(line):
            return

        # 匹配计划使用独立信号
        if self.patPlanResearch(line, self.cycle_os_time_start):
            return
        # 匹配IO独立信号
        if self.patIoInfoResearch(line):
            return
        # 测速测距识别
        if self.patSduInfoResearch(line, self.cycle_os_time_start):
            return

    # 模板识别
    def patCycleCommonResearch(self, line):
        update_flag = False
        # 所有匹配模板
        match = self.cfg.reg_config.pat_cycle_start.findall(line)
        if match:
            self.cycleNumStart = int(match[0][1])
            self.cycle_os_time_start = int(match[0][0])  # 当周期系统开始时间
            # 新的周期时重置之前的变量
            self.newCyclePreProcess()
        else:
            match = self.cfg.reg_config.pat_cycle_end.findall(line)
            if match:
                self.cycle_os_time_end = int(match[0][0])  # 当周期系统结束时间
                self.timeStatictics = self.staticticsCycleTime(int(match[0][1]))
                update_flag = True
            else:
                match = self.cfg.reg_config.pat_time.findall(line)
                if match:
                    self.timeContentStr = match[0]  # time 信息直接返回
                else:
                    match = self.cfg.reg_config.pat_fsm.findall(line)
                    if match:
                        self.fsm = match[0]
                    else:
                        match = self.cfg.reg_config.pat_ctrl.findall(line)
                        if match:
                            self.sc_ctrl = match[0]
                            # 允许速度处理
                            self.sc_ctrl = self.sc_ctrl[:4] + (str(self.atp2atoParser.msg_obj.sp2_obj.v_permitted),) + self.sc_ctrl[4:]  # tuple拼接
                        else:
                            match = self.cfg.reg_config.pat_p2o.findall(line)
                            if match:
                                self.atp2atoParser.msgParse(match[0])
                            else:
                                match = self.cfg.reg_config.pat_o2p.findall(line)
                                if match:
                                    self.atp2atoParser.msgParse(match[0])
                                else:
                                    match = self.cfg.reg_config.pat_a2t.findall(line)
                                    if match:
                                        self.ato2tsrsParser.msgParse(match[0])
                                    else:
                                        match = self.cfg.reg_config.pat_t2a.findall(line)
                                        if match:
                                            self.tsrs2atoParser.msgParse(match[0])
                                        else:
                                            match = self.cfg.reg_config.pat_stoppoint.findall(line)
                                            if match:
                                                self.stoppoint = match[0]
                                            else:
                                                pass
        # 发射信号
        if update_flag == True:
            # 生成信号结果
            result = (self.cycle_os_time_start, self.cycleNumStart, self.timeContentStr,
            self.fsm, self.sc_ctrl, self.stoppoint,
            self.atp2atoParser.msg_obj,self.mvbParser.tcms2ato_state_obj, self.timeStatictics,
            self.ato2tsrsParser.msg_obj, self.tsrs2atoParser.msg_obj)
            self.patShowSignal.emit(pickle.loads(pickle.dumps(result)))
            del result
        # 返回用于画图
        return update_flag

    # 实时图形绘制
    def paintArrayUpdate(self):
        if self.sc_ctrl:  # 如果匹配成功
            newPaintList[0, self.newPaintCnt] = self.sc_ctrl[1]   # 当前速度
            newPaintList[1, self.newPaintCnt] = self.sc_ctrl[2]   # ATO命令速度
            newPaintList[2, self.newPaintCnt] = self.sc_ctrl[3]   # ATP命令速度
            newPaintList[3, self.newPaintCnt] = self.sc_ctrl[4]   # ATP允许速度
            newPaintList[4, self.newPaintCnt] = self.sc_ctrl[6]   # 输出级位
            self.newPaintCnt += 1
        else:
            pass

        if self.newPaintCnt == real_curve_buff:
            paintList[:, real_curve_buff:real_curve_all_buff] = paintList[:, 0:real_curve_all_buff - real_curve_buff]
            paintList[:, 0:real_curve_buff] = newPaintList[:, 0:real_curve_buff]
            # 清除列表
            self.newPaintCnt = 0

    # 测速测距识别
    def patSduInfoResearch(self, line=str, cycleTime=int):
        updateflag = False
        if 'v&p' in line:
            self.sduInfoParser.sduInfoStringParse(line, cycleTime)
            updateflag = self.sduInfoParser.sduInfo.updateflag
        # 收集到sdu_ato和sdu_atp
        if updateflag:
            self.ioInfoObj = pickle.loads(pickle.dumps(self.sduInfoParser.sduInfo))
            self.sduShowSignal.emit(self.ioInfoObj)
        else:
            pass
        return updateflag

    # IO查询
    def patIoInfoResearch(self, line):
        updateflag = False
        self.ioInfoParser.ioStringParse(line)
        # 发送信号
        if self.ioInfoParser.ioInfo.updateflagIn == True or self.ioInfoParser.ioInfo.updateflagOut:
            self.ioInfoObj =  pickle.loads(pickle.dumps(self.ioInfoParser.ioInfo))
            self.ioShowSignal.emit(self.cycleNumStart, self.timeContentStr, self.ioInfoObj)
            updateflag = True
        else:
            pass
        return updateflag

    # 解析MVB内容
    def patMvbResearch(self, line):
        updateflag = False
        # 前提条件
        if 'MVB[' in line:
            match = self.cfg.reg_config.pat_mvb.findall(line)
            if match:
                try:
                    [a2tCtrl, a2tStat, t2aStat ] = self.mvbParser.parseProtocol(match[0])
                    updateflag = True
                except MVBParseContentException as err:
                    updateflag = False
                    print('mvb parse err[%d]:%s'%(err.port,err.mvbLine))
                # 当匹配成功时拷贝
                if self.mvbParser.ato2tcms_ctrl_obj.updateflag or \
                    self.mvbParser.ato2tcms_state_obj.updateflag or \
                    self.mvbParser.tcms2ato_state_obj.updateflag:

                    self.a2tCtrlObj = pickle.loads(pickle.dumps(a2tCtrl))
                    self.a2tStatObj = pickle.loads(pickle.dumps(a2tStat))
                    self.t2aStatObj = pickle.loads(pickle.dumps(t2aStat))
                    self.mvbShowSignal.emit(self.a2tCtrlObj,self.a2tStatObj, self.t2aStatObj)
        return updateflag

    # 提取计划内容
    def patPlanResearch(self, line=str, osTime=int):
        updateflag = False
        # 提高解析效率,当均更新时才发送信号
        if '[RP' in line:
            self.rpParser.rpStringParse(line, osTime)
            updateflag = self.rpParser.rpInfo.updateflag
        # 发送信号
        if updateflag:
            self.rpInfoObj = pickle.loads(pickle.dumps(self.rpParser.rpInfo))
            self.planShowSignal.emit(self.rpInfoObj, osTime)
        else:
            pass
        # 返回用于指示解析结果
        return updateflag

    # 统计ATO周期的运行时间, None不更新
    def staticticsCycleTime(self, cycleNum=int):
        if cycleNum != self.cycleNumStart:
            return None
        else:
            curSlot = (self.cycle_os_time_end - self.cycle_os_time_start)
            self.statisticHandle.statsProcess(curSlot, cycleNum, self.cycleStat)

        return self.cycleStat


"""
Repository layer for flower service
"""
import pickle
from typing import Optional
from sklearn.ensemble import RandomForestClassifier

class FlowerClassifier:
    """
    Flower classifier singleton
    """
    iris_classifier: Optional[RandomForestClassifier] = None

    def __init__(self):
        raise Exception('Please use get_instance() method')

    @classmethod
    def get_instance(cls) -> RandomForestClassifier:
        """
        Class method to obtain singleton flower classifier
        """
        if cls.iris_classifier is None:
            with open('modules/flower/model.pkl', 'rb') as iris_model:
                cls.iris_classifier = pickle.load(iris_model)

        return cls.iris_classifier

class FlowerRepository:
    """
    Flower repository that uses the random forrest classifier
    """
    iris_classifier: RandomForestClassifier

    def __init__(self) -> None:
        self.iris_classifier = FlowerClassifier.get_instance()

    def get_flower_predictions(self, flower_attributes: list[list[float]]) -> list[int]:
        """
        Uses the random forrest classifier to generate predictions on flowers
        """
        predicted_flowers = self.iris_classifier.predict(
            flower_attributes).tolist()

        if not isinstance(predicted_flowers, list):
            raise Exception('Error generating flower prediction')

        return predicted_flowers

    def get_flower_prediction(self, flower_attributes: list[float]) -> int:
        """
        Uses the random forrest classifier to generate a prediction on flower
        """
        predicted_flower, *_ = self.get_flower_predictions([flower_attributes])

        return predicted_flower