import torch
import torch.nn as nn
import numpy as np
import torch.nn.functional as F
from collections import namedtuple

from .set2set import Set2Vec

ReadoutConfig = namedtuple(
        'ReadoutConfig', [
            'hidden_dim',
            'readout_hidden_dim',
            'mode',
            'target_dim',
        ]
)

class Readout(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.config = config
        self.classify = (self.config.mode == 'clf')
        self.hidden_dim = config.hidden_dim
        self.target_dim = config.target_dim
        self.readout_hidden_dim = config.readout_hidden_dim
        self.activation = nn.LeakyReLU

    def forward(self, G):
        pass

class DTNNReadout(Readout):
    def __init__(self, config):
        super().__init__(config)
        net = nn.Sequential(
                nn.Linear(self.hidden_dim, self.readout_hidden_dim),
                self.activation(),
                nn.BatchNorm1d(self.readout_hidden_dim),
                nn.Linear(self.readout_hidden_dim, self.target_dim),
                )
        self.net = net

    def forward(self, h):
        bs, gd, dd = (s for s in h.size())
        x = h.view(-1, dd)
        x = self.net(x)
        x = x.view(bs, gd, -1)
        x = x.sum(1)
        return x

class FullyConnectedReadout(Readout):
    def __init__(self, config):
        super().__init__(config)
        net = nn.Sequential(
                nn.Linear(self.hidden_dim, self.readout_hidden_dim),
                self.activation(),
                nn.BatchNorm1d(self.readout_hidden_dim),
                nn.Linear(self.readout_hidden_dim, self.target_dim),
                )
        self.net = net

    def forward(self, h):
        x = torch.mean(h, 1)
        x = self.net(x)
        return x

class SetReadout(Readout):
    def __init__(self, config):
        super().__init__(config)
        self.set2vec = Set2Vec(self.hidden_dim, self.target_dim, config.readout_hidden_dim)

    def forward(self, h):
        x = self.set2vec(h)
        return x

class VCNReadout(Readout):
    def __init__(self, config):
        super().__init__(config)
        self.module_list = nn.ModuleList()
        for target in self.target_names:
            self.module_list.append(nn.Linear(self.hidden_dim, target.dim))

    def forward(self, G):
        h_dict = {v: G.node[v]['hidden'] for v in G.nodes()}
        out = {}
        for i, target in enumerate(self.target_names):
            out[target.name] = self.module_list[i](h_dict[target.name])
        return out

class VertexReadout(Readout):
    def __init__(self, config):
        super().__init__(config)
        net = nn.Sequential(
                nn.Linear(self.hidden_dim, self.readout_hidden_dim),
                self.activation(),
                nn.BatchNorm2d(self.readout_hidden_dim),
                nn.Linear(self.readout_hidden_dim, self.target_dim),
                )
        self.net = net

    def forward(self, h):
        bs, gd, dd = (s for s in h.size())
        x = h.view(-1, dd)
        x = self.net(x)
        x = x.view(bs, gd, -1)
        return x

def make_readout(readout_config):
    if readout_config.function == 'fully_connected':
        return FullyConnectedReadout(readout_config.config)
    elif readout_config.function == 'dtnn':
        return DTNNReadout(readout_config.config)
    elif readout_config.function == 'vcn':
        return VCNReadout(readout_config.config)
    elif readout_config.function == 'vertex':
        return VertexReadout(readout_config.config)
    elif readout_config.function == 'set':
        return SetReadout(readout_config.config)
    else:
        raise ValueError("Unsupported readout function! ({})".format(readout_config.function))


from collections import namedtuple
import time
from .utils import (
    client_array_operation,
    make_valid_data,
    create_host_urn,
    create_resource_arn,
    create_hash,
    set_required_access_v2,
    transformation,
    ipaddress_to_urn
)
from .registry import RegisteredResourceCollector
from schematics import Model
from schematics.types import StringType, ModelType, ListType, BooleanType

InstanceData = namedtuple("InstanceData", ["instance", "instance_type"])

class Tag(Model):
    Key = StringType(required=True)
    Value = StringType(required=True)

class Subnet(Model):
    SubnetId = StringType(required=True)
    Tags = ListType(ModelType(Tag), default=[])
    AvailabilityZone = StringType(required=True)
    VpcId = StringType(required=True)

class Vpc(Model):
    VpcId = StringType(required=True)
    IsDefault = BooleanType(default=False)
    Tags = ListType(ModelType(Tag), default=[])

class SecurityGroup(Model):
    GroupName = StringType(default="UKNOWN")
    GroupId = StringType(required=True)
    VpcId = StringType()

class VpnGateway(Model):
    class VpnGatewayVpcAttachment(Model):
        VpcId = StringType(required=True)
        State = StringType(default="UNKNOWN")

    VpnGatewayId = StringType(required=True)
    VpcAttachments = ListType(ModelType(VpnGatewayVpcAttachment), default=[])

class InstanceType(Model):
    InstanceType = StringType(required=True)
    Hypervisor = StringType(default="")

class Instance(Model):
    class InstanceState(Model):
        Name = StringType(required=True)

    class SecurityGroup(Model):
        GroupId = StringType(required=True)

    InstanceId = StringType(required=True)
    InstanceType = StringType(required=True)
    State = ModelType(InstanceState)
    Tags = ListType(ModelType(Tag), default=[])
    PrivateIpAddress = StringType()
    PublicDnsName = StringType()
    PublicIpAddress = StringType()
    SubnetId = StringType()
    VpcId = StringType()
    SecurityGroups = ListType(ModelType(SecurityGroup), default=[])

class RunInstances(Model):
    class ResponseElements(Model):
        class InstancesSet(Model):
            class RunInstance(Model):
                instanceId = StringType(required=True)

            items = ListType(ModelType(RunInstance), required=True)

        instancesSet = ModelType(InstancesSet, required=True)

    responseElements = ModelType(ResponseElements, required=True)

class Ec2InstanceCollector(RegisteredResourceCollector):
    API = "ec2"
    API_TYPE = "regional"
    COMPONENT_TYPE = "aws.ec2"

    def __init__(self, location_info, client, agent):
        RegisteredResourceCollector.__init__(self, location_info, client, agent)
        self.instance_types = {}

    def process_all(self, filter=None):
        if not filter or "instances" in filter:
            self.process_instances()
        if not filter or "security_groups" in filter:
            self.process_security_groups()
        if not filter or "vpcs" in filter:
            self.process_vpcs()
        if not filter or "subnets" in filter:
            self.process_subnets()
        if not filter or "vpn_gateways" in filter:
            self.process_vpn_gateways()

    @set_required_access_v2("ec2:DescribeInstanceTypes")
    def collect_instance_type(self, instance_type):
        # Items never change, only added, safe to hold in memory
        if instance_type not in self.instance_types:
            instance_type_data = self.client.describe_instance_types(InstanceTypes=[instance_type]).get(
                "InstanceTypes", []
            )
            if instance_type_data:
                self.instance_types[instance_type] = instance_type_data[0]
        return self.instance_types.get(instance_type, {})

    def collect_instance(self, instance_data):
        instance_type = instance_data.get("InstanceType", "")
        instance_type_data = self.collect_instance_type(instance_type) or {}
        return InstanceData(instance=instance_data, instance_type=instance_type_data)

    def collect_instances(self, **kwargs):
        for reservation in client_array_operation(
                self.client,
                "describe_instances",
                "Reservations",
                Filters=[
                    {
                        "Name": "instance-state-code",  # Don't return terminated instances
                        "Values": [
                            "0",  # pending
                            "16",  # running
                            "32",  # shutting-down
                            "64",  # stopping
                            "80",  # stopped
                        ],
                    }
                ],
                **kwargs
        ):
            for instance_data in reservation.get("Instances", []):
                yield self.collect_instance(instance_data)

    @set_required_access_v2("ec2:DescribeInstances")
    def process_instances(self, **kwargs):
        for data in self.collect_instances(**kwargs):
            self.process_instance(data)

    def process_some_instances(self, ids):
        self.process_instances(InstanceIds=ids)

    @transformation()
    def process_instance_type(self, data):
        instance_type = InstanceType(data, strict=False)
        instance_type.validate()
        return instance_type

    @transformation()
    def process_instance(self, data):
        instance = Instance(data.instance, strict=False)
        instance.validate()

        self.agent.event(
            {
                "timestamp": int(time.time()),
                "event_type": "ec2_state",
                "msg_title": "EC2 instance state",
                "msg_text": instance.State.Name,
                "host": instance.InstanceId,
                "tags": ["state:" + instance.State.Name],
            }
        )

        output = make_valid_data(data.instance)
        urns = [
            create_host_urn(instance.InstanceId),
            create_resource_arn(
                "ec2",
                self.location_info.Location.AwsRegion,
                self.location_info.Location.AwsAccount,
                "instance",
                instance.InstanceId,
            ),
        ]
        if not instance.Tags:
            output["Tags"] = []
        output["Tags"].append({"Key": "host", "Value": instance.InstanceId})
        output["Tags"].append({"Key": "instance-id", "Value": instance.InstanceId})
        if instance.PrivateIpAddress:
            urns.append(ipaddress_to_urn(instance.PrivateIpAddress, instance.VpcId))
            output["Tags"].append({"Key": "private-ip", "Value": instance.PrivateIpAddress})
        if instance.PublicDnsName:
            urns.append(create_host_urn(instance.PublicDnsName))
            output["Tags"].append({"Key": "fqdn", "Value": instance.PublicDnsName})
        if instance.PublicIpAddress:
            urns.append(create_host_urn(instance.PublicIpAddress))
            output["Tags"].append({"Key": "public-ip", "Value": instance.PublicIpAddress})

        output["URN"] = urns

        if data.instance_type:  # Don't run if instance type not found
            instance_type = self.process_instance_type(data.instance_type)
            output["isNitro"] = instance_type.Hypervisor == "nitro"

        # Map the subnet and if not available then map the VPC
        if instance.SubnetId:
            self.emit_relation(instance.InstanceId, instance.SubnetId, "uses-service", {})
        elif instance.VpcId:  # pragma: no cover
            self.emit_relation(instance.InstanceId, instance.VpcId, "uses-service", {})

        for security_group in instance.SecurityGroups:
            self.emit_relation(instance.InstanceId, security_group.GroupId, "uses-service", {})

        self.emit_component(instance.InstanceId, "instance", output)

    def collect_security_groups(self, **kwargs):
        for security_group in client_array_operation(self.client,
                                                     "describe_security_groups",
                                                     "SecurityGroups",
                                                     **kwargs):
            yield security_group

    @set_required_access_v2("ec2:DescribeSecurityGroups")
    def process_security_groups(self, **kwargs):
        for security_group_data in self.collect_security_groups(**kwargs):
            self.process_security_group(security_group_data)

    @transformation()
    def process_security_group(self, data):
        security_group = SecurityGroup(data, strict=False)
        security_group.validate()
        output = make_valid_data(data)
        output["Version"] = create_hash(output)
        output["Name"] = security_group.GroupName
        output["URN"] = [
            create_resource_arn(
                "ec2",
                self.location_info.Location.AwsRegion,
                self.location_info.Location.AwsAccount,
                "security-group",
                security_group.GroupId,
            )
        ]
        if security_group.VpcId:  # pragma: no cover
            self.emit_relation(security_group.VpcId, security_group.GroupId, "has-resource", {})

        self.emit_component(security_group.GroupId, "security-group", output)

    def collect_vpcs(self):
        for vpc in client_array_operation(self.client, "describe_vpcs", "Vpcs"):
            yield vpc

    @set_required_access_v2("ec2:DescribeVpcs")
    def process_vpcs(self):
        for vpc_data in self.collect_vpcs():
            self.process_vpc(vpc_data)

    @transformation()
    def process_vpc(self, data):
        vpc = Vpc(data, strict=False)
        vpc.validate()
        output = make_valid_data(data)
        # construct a name
        vpc_name = vpc.VpcId
        name_tag = [tag for tag in vpc.Tags if tag.Key == "Name"]
        if vpc.IsDefault:
            vpc_name = "default"
        elif len(name_tag) > 0:
            vpc_name = name_tag[0].Value
        output["Name"] = vpc_name
        # add a URN
        output["URN"] = [
            create_resource_arn(
                "ec2", self.location_info.Location.AwsRegion, self.location_info.Location.AwsAccount, "vpc", vpc.VpcId
            )
        ]
        self.emit_component(vpc.VpcId, "vpc", output)

    def collect_subnets(self):
        for subnet in client_array_operation(self.client, "describe_subnets", "Subnets"):
            yield subnet

    @set_required_access_v2("ec2:DescribeSubnets")
    def process_subnets(self):
        for subnet_data in self.collect_subnets():
            self.process_subnet(subnet_data)

    @transformation()
    def process_subnet(self, data):
        subnet = Subnet(data, strict=False)
        subnet.validate()
        output = make_valid_data(data)
        # construct a name
        subnet_name = subnet.SubnetId
        name_tag = [tag for tag in subnet.Tags if tag.Key == "Name"]
        if len(name_tag) > 0:
            subnet_name = name_tag[0].Value
        if subnet.AvailabilityZone:  # pragma: no cover
            subnet_name = "{}-{}".format(subnet_name, subnet.AvailabilityZone)
        output["Name"] = subnet_name
        # add a URN
        output["URN"] = [
            create_resource_arn(
                "ec2",
                self.location_info.Location.AwsRegion,
                self.location_info.Location.AwsAccount,
                "subnet",
                subnet.SubnetId,
            )
        ]
        self.emit_component(subnet.SubnetId, "subnet", output)
        self.emit_relation(subnet.SubnetId, subnet.VpcId, "uses-service", {})

    def collect_vpn_gateways(self):
        for vpn_gateway in client_array_operation(self.client,
                                                  "describe_vpn_gateways",
                                                  "VpnGateways",
                                                  Filters=[{"Name": "state", "Values": ["pending", "available"]}]
                                                  ):
            yield vpn_gateway

    @set_required_access_v2("ec2:DescribeVpnGateways")
    def process_vpn_gateways(self):
        for vpn_gateway_data in self.collect_vpn_gateways():
            self.process_vpn_gateway(vpn_gateway_data)

    @transformation()
    def process_vpn_gateway(self, data):
        vpn_gateway = VpnGateway(data, strict=False)
        vpn_gateway.validate()
        output = make_valid_data(data)
        output["Name"] = vpn_gateway.VpnGatewayId
        self.emit_component(vpn_gateway.VpnGatewayId, "vpn-gateway", output)
        for vpn_attachment in vpn_gateway.VpcAttachments:
            if vpn_attachment.State == "attached":
                self.emit_relation(vpn_gateway.VpnGatewayId, vpn_attachment.VpcId, "uses-service", {})

    @transformation()
    def process_batch_instances(self, event, seen):
        data = RunInstances(event, strict=False)
        data.validate()
        instance_ids = [
            instance.instanceId
            for instance in data.responseElements.instancesSet.items
            if instance.instanceId not in seen
        ]
        self.process_instances(InstanceIds=instance_ids)
        seen.update(set(instance_ids))

    def process_state_notification(self, event, seen):
        instance_id = event.get("instance-id", "")
        if instance_id not in seen:
            seen.add(instance_id)
            if event.get("state") == "terminated":
                self.agent.delete(instance_id)
            else:
                self.process_instances(InstanceIds=[instance_id])

    def process_one_instance(self, instance_id):
        self.process_instances(InstanceIds=[instance_id])

    def process_one_security_group(self, security_group_id):
        self.process_security_groups(GroupIds=[security_group_id])

    EVENT_SOURCE = "ec2.amazonaws.com"
    CLOUDTRAIL_EVENTS = [
        {"event_name": "RunInstances", "processor": process_batch_instances},
        {"event_name": "StartInstances", "processor": process_batch_instances},
        {"event_name": "StopInstances", "processor": process_batch_instances},
        {"event_name": "TerminateInstances", "processor": process_batch_instances},
        {"event_name": "InstanceStateChangeNotification", "processor": process_state_notification},
        {"event_name": "AttachVolume", "path": "responseElements.instanceId", "processor": process_one_instance},
        {"event_name": "DetachVolume", "path": "responseElements.instanceId", "processor": process_one_instance},
        {
            "event_name": "ModifyInstanceAttribute",
            "path": "requestParameters.instanceId",
            "processor": process_one_instance,
        },
        {
            "event_name": "RevokeSecurityGroupIngress",
            "path": "requestParameters.groupId",
            "processor": process_one_security_group,
        },
        {
            "event_name": "AuthorizeSecurityGroupIngress",
            "path": "requestParameters.groupId",
            "processor": process_one_security_group,
        },
    ]


from __future__ import print_function, division
import os
import time
import random
import numpy as np
from base import BaseModel
from replay_memory import ReplayMemory
from utils import save_pkl, load_pkl
import tensorflow as tf
import matplotlib.pyplot as plt

class Agent(BaseModel):
    def __init__(self, config, environment, sess):
        self.sess = sess
        self.weight_dir = 'weight'
        self.record_dir = 'record'
        self.recordfile_name = ''
        self.now_time = time.strftime("%m-%d-%H-%M",time.localtime(time.time()))
        self.env = environment
        model_dir = './Model/a.model'
        self.memory = ReplayMemory(model_dir)
        self.max_step = 100000
        # The number of RB, The number of vehicle
        self.RB_number = 20
        self.num_vehicle = len(self.env.vehicles)
        # The following two variables are used to store the transmission power
        # and channel selection of each V2V link
        # The one is used for testing, and the other is used for training
        self.action_all_with_power = np.zeros([self.num_vehicle, 3, 2],
                                              dtype='int32')  # this is actions that taken by V2V links with power
        self.action_all_with_power_training = np.zeros([self.num_vehicle, 3, 2],
                                                       dtype='int32')
        self.reward = []
        # Settings related to learning rate
        self.learning_rate = 0.01    # 0.01
        self.learning_rate_minimum = 0.0001
        self.learning_rate_decay = 0.96
        self.learning_rate_decay_step = 500000
        # each 100 steps update the target_q network
        self.target_q_update_step = 100  # 100
        #Discount factor
        self.discount = 0.5
        self.double_q = True
        self.build_dqn()
        # The number of V2V links.
        self.V2V_number = 3 * len(self.env.vehicles)  # every vehicle need to communicate with 3 neighbors
        self.training = True

    # This function is used to store the transmit power and channel selected by each V2V link
    # Store in an <"action"> matrix
    def merge_action(self, idx, action):
        self.action_all_with_power[idx[0], idx[1], 0] = action % self.RB_number
        self.action_all_with_power[idx[0], idx[1], 1] = int(np.floor(action / self.RB_number))

    def get_state(self, idx):
        # ===============================
        #  Get State from the environment
        # ===============================
        vehicle_number = len(self.env.vehicles)
        V2V_channel = (self.env.V2V_channels_with_fastfading[idx[0], self.env.vehicles[idx[0]].destinations[idx[1]],
                       :] - 80) / 60
        V2I_channel = (self.env.V2I_channels_with_fastfading[idx[0], :] - 80) / 60
        Eve_channel_I = (self.env.Eve_channels_with_fastfading_I[idx[0], :] - 80) / 60
        Eve_channel_V = (self.env.Eve_channels_with_fastfading_V[idx[0], self.env.vehicles[idx[0]].destinations[idx[1]],
                       :] - 80) / 60
        V2V_interference = (-self.env.V2V_Interference_all[idx[0], idx[1], :] - 60) / 60
        # The <"NeiSelection"> representative RB occupation
        NeiSelection = np.zeros(self.RB_number)
        for i in range(3):
            for j in range(3):
                if self.training:
                    NeiSelection[self.action_all_with_power_training[self.env.vehicles[idx[0]].neighbors[i], j, 0]] = 1
                else:
                    NeiSelection[self.action_all_with_power[self.env.vehicles[idx[0]].neighbors[i], j, 0]] = 1
        for i in range(3):
            if i == idx[1]:
                continue
            if self.training:
                if self.action_all_with_power_training[idx[0], i, 0] >= 0:
                    NeiSelection[self.action_all_with_power_training[idx[0], i, 0]] = 1
            else:
                if self.action_all_with_power[idx[0], i, 0] >= 0:
                    NeiSelection[self.action_all_with_power[idx[0], i, 0]] = 1
        # Status include V2I_channel, V2V_interference, V2V_channel, Eve_channel_I, Eve_channel_V, NeiSelection
        return np.concatenate((V2I_channel, V2V_interference, V2V_channel, Eve_channel_I, Eve_channel_V, NeiSelection))

    def predict(self, s_t, step, test_ep=False):
        # ==========================
        #  Select actions
        # ==========================
        ep = 1 / (step / 1000000 + 1)
        # Random selection or training selection
        if random.random() < ep and test_ep is False:  # epsion to balance the exporation and exploition
            # Each number from 0 ~ 60 represents a choice
            action = np.random.randint(60)  # 20RBs X 3 power level
        else:
            action = self.q_action.eval({self.s_t: [s_t]})[0]
        return action

    # This function used for collcet data for training, and training a mini batch
    def observe(self, prestate, state, reward, action):
        # -----------
        # Collect Data for Training and Experience replay
        # ---------
        self.memory.add(prestate, state, reward, action)  # add the state and the action and the reward to the memory
        # print(self.step)
        if self.step > 0:
            if self.step % 50 == 0:
                # print('Training')
                self.q_learning_mini_batch()  # training a mini batch
                # self.save_weight_to_pkl()
            if self.step % self.target_q_update_step == self.target_q_update_step - 1:
                # print("Update Target Q network:")
                self.update_target_q_network()  # update the Target-Q network parameter

    def save_record(self, record_content):
        if not os.path.exists(self.record_dir):
            os.makedirs(self.record_dir)
        if(self.recordfile_name == ''):
            if(self.double_q == True and self.dueling_q == True):
                self.recordfile_name = "double_q&dueling_q"
            else:
                if(self.double_q == True):
                    self.recordfile_name = "double_q"
                else:
                    if(self.dueling_q == True):
                        self.recordfile_name = "dueling_q"
                    else:
                        self.recordfile_name = "normal_q"
        with open(os.path.join(self.record_dir, "V-num-%d_%s-%s.txt" % \
            (self.num_vehicle, self.now_time, self.recordfile_name)), 'a') as f:
            f.write(record_content)

    # The network training and testing funtion
    def train(self):
        num_game, self.update_count, ep_reward = 0, 0, 0.
        total_reward, self.total_loss, self.total_q = 0., 0., 0.
        max_avg_ep_reward = 0
        ep_reward, actions = [], []
        mean_big = 0
        number_big = 0
        mean_not_big = 0
        number_not_big = 0
        print(self.num_vehicle)
        #!Step1: Start a new simulation environment
        self.env.new_random_game(self.num_vehicle)   # episode
        for self.step in (range(0, 40000)):  # need more configuration
            #!Step2: Begin training, the tutal steps is 40000
            # initialize set some varibles
            if self.step == 0:
                num_game, self.update_count, ep_reward = 0, 0, 0.
                total_reward, self.total_loss, self.total_q = 0., 0., 0.
                ep_reward, actions = [], []
            # Restart a new simulation environment
            if (self.step % 2000 == 1):
                self.env.new_random_game(self.num_vehicle)
            print(self.step)
            state_old = self.get_state([0, 0])
            # print("state", state_old)
            self.training = True
            for k in range(1):
                for i in range(len(self.env.vehicles)):
                    for j in range(3):
                        #!Step3: Get training data for each pair of V2V links and training
                        # Include <"state_old, state_new, reward_train, action">
                        # Besides: The training a batch in <"observe"> function
                        state_old = self.get_state([i, j])
                        action = self.predict(state_old, self.step)
                        # self.merge_action([i,j], action)
                        self.action_all_with_power_training[i, j, 0] = action % self.RB_number
                        self.action_all_with_power_training[i, j, 1] = int(np.floor(action / self.RB_number))
                        reward_train = self.env.act_for_training(self.action_all_with_power_training, [i, j])
                        state_new = self.get_state([i, j])
                        self.observe(state_old, state_new, reward_train, action)
            if (self.step % 2000 == 0) and (self.step > 0):
                #!Step4: Testing
                self.training = False
                number_of_game = 10
                if (self.step % 10000 == 0) and (self.step > 0):
                    number_of_game = 50
                if (self.step == 38000):
                    number_of_game = 100
                V2V_Eifficency_list = np.zeros(number_of_game)
                V2I_Eifficency_list = np.zeros(number_of_game)
                V2V_security_rate_list = np.zeros(number_of_game)
                for game_idx in range(number_of_game):
                    self.env.new_random_game(self.num_vehicle)
                    test_sample = 200
                    Eifficency_V2V = []
                    Eifficency_V2I = []
                    Security_rate = []
                    print('test game idx:', game_idx)
                    for k in range(test_sample):
                        action_temp = self.action_all_with_power.copy()
                        for i in range(len(self.env.vehicles)):
                            self.action_all_with_power[i, :, 0] = -1
                            sorted_idx = np.argsort(self.env.individual_time_limit[i, :])
                            for j in sorted_idx:
                                state_old = self.get_state([i, j])
                                action = self.predict(state_old, self.step, True)
                                self.merge_action([i, j], action)
                            if i % (len(self.env.vehicles) / 10) == 1:  # add 10
                                action_temp = self.action_all_with_power.copy()
                                V2V_reward, V2I_reward, V2V_security_rate = self.env.act_asyn(action_temp)
                                Eifficency_V2V.append(np.sum(V2V_reward))
                                Eifficency_V2I.append(np.sum(V2I_reward))
                                Security_rate.append(np.sum(V2V_security_rate))
                        # print("actions", self.action_all_with_power)
                    V2V_Eifficency_list[game_idx] = np.mean(np.asarray(Eifficency_V2V))
                    V2I_Eifficency_list[game_idx] = np.mean(np.asarray(Eifficency_V2I))
                    V2V_security_rate_list[game_idx] = np.mean(np.asarray(Security_rate))
                    # print("action is", self.action_all_with_power)
                    # print('failure probability is, ', percent)
                    # print('action is that', action_temp[0,:])
                #!Step5: Save weight parameters
                self.save_weight_to_pkl()
                print('The number of vehicle is ', len(self.env.vehicles))
                print('Mean of the V2V Eifficency is that ', np.mean(V2V_Eifficency_list))
                print('Mean of the V2I Eifficency is that ', np.mean(V2I_Eifficency_list))
                print('Mean of V2V Security Rate is that ', np.mean(V2V_security_rate_list))
                self.save_record("V2V Efficiency: %f \tV2I Efficiency: %f\tSecurity Rate: %f\tCompound Efficiency: %f\tStep : %d\n" % \
                    (np.mean(V2V_Eifficency_list),np.mean(V2I_Eifficency_list),\
                        np.mean(V2V_security_rate_list)/self.num_vehicle,\
                        0.1 * np.mean(V2I_Eifficency_list) + 0.9 * np.mean(V2V_Eifficency_list), self.step))

                # print('Test Reward is ', np.mean(test_result))

    def q_learning_mini_batch(self):

        # Training the DQN model
        s_t, s_t_plus_1, action, reward = self.memory.sample()
        t = time.time()
        if self.double_q:  # double Q learning
            pred_action = self.q_action.eval({self.s_t: s_t_plus_1})
            q_t_plus_1_with_pred_action = self.target_q_with_idx.eval({self.target_s_t: s_t_plus_1,
                                                                       self.target_q_idx: [[idx, pred_a] for idx, pred_a
                                                                                in enumerate(pred_action)]})
            target_q_t = self.discount * q_t_plus_1_with_pred_action + reward
        else:
            q_t_plus_1 = self.target_q.eval({self.target_s_t: s_t_plus_1})
            max_q_t_plus_1 = np.max(q_t_plus_1, axis=1)
            target_q_t = self.discount * max_q_t_plus_1 + reward
        _, q_t, loss, w = self.sess.run([self.optim, self.q, self.loss, self.w],
                                        {self.target_q_t: target_q_t, self.action: action, self.s_t: s_t,
                                         self.learning_rate_step: self.step})  # training the network

        print('loss is ', loss)
        self.total_loss += loss
        self.total_q += q_t.mean()
        self.update_count += 1

    def build_dqn(self):
        # --- Building the DQN -------
        self.w = {}
        self.t_w = {}

        initializer = tf.truncated_normal_initializer(0, 0.02)
        activation_fn = tf.nn.relu
        n_hidden_1 = 500
        n_hidden_2 = 250
        n_hidden_3 = 120
        n_input = 120
        n_output = 60
        # The DQN network weights and biases
        def encoder(x):
            weights = {
                'encoder_h1': tf.Variable(tf.truncated_normal([n_input, n_hidden_1], stddev=0.1)),
                'encoder_h2': tf.Variable(tf.truncated_normal([n_hidden_1, n_hidden_2], stddev=0.1)),
                'encoder_h3': tf.Variable(tf.truncated_normal([n_hidden_2, n_hidden_3], stddev=0.1)),
                'encoder_h4': tf.Variable(tf.truncated_normal([n_hidden_3, n_output], stddev=0.1)),
                'encoder_b1': tf.Variable(tf.truncated_normal([n_hidden_1], stddev=0.1)),
                'encoder_b2': tf.Variable(tf.truncated_normal([n_hidden_2], stddev=0.1)),
                'encoder_b3': tf.Variable(tf.truncated_normal([n_hidden_3], stddev=0.1)),
                'encoder_b4': tf.Variable(tf.truncated_normal([n_output], stddev=0.1)),

            }
            layer_1 = tf.nn.relu(tf.add(tf.matmul(x, weights['encoder_h1']), weights['encoder_b1']))
            layer_2 = tf.nn.relu(tf.add(tf.matmul(layer_1, weights['encoder_h2']), weights['encoder_b2']))
            layer_3 = tf.nn.relu(tf.add(tf.matmul(layer_2, weights['encoder_h3']), weights['encoder_b3']))
            layer_4 = tf.nn.relu(tf.add(tf.matmul(layer_3, weights['encoder_h4']), weights['encoder_b4']))
            return layer_4, weights
        # Used for prediction
        with tf.variable_scope('prediction'):
            self.s_t = tf.placeholder('float32', [None, n_input])
            self.q, self.w = encoder(self.s_t)
            self.q_action = tf.argmax(self.q, dimension=1)
        # Used for get target-Q
        with tf.variable_scope('target'):
            self.target_s_t = tf.placeholder('float32', [None, n_input])
            self.target_q, self.target_w = encoder(self.target_s_t)
            self.target_q_idx = tf.placeholder('int32', [None, None], 'output_idx')
            self.target_q_with_idx = tf.gather_nd(self.target_q, self.target_q_idx)
        # Used for update the target-Q network parameters
        with tf.variable_scope('pred_to_target'):
            self.t_w_input = {}
            self.t_w_assign_op = {}
            for name in self.w.keys():
                print('name in self w keys', name)
                self.t_w_input[name] = tf.placeholder('float32', self.target_w[name].get_shape().as_list(), name=name)
                self.t_w_assign_op[name] = self.target_w[name].assign(self.t_w_input[name])

        def clipped_error(x):
            try:
                return tf.select(tf.abs(x) < 1.0, 0.5 * tf.square(x), tf.abs(x) - 0.5)
            except:
                return tf.where(tf.abs(x) < 1.0, 0.5 * tf.square(x), tf.abs(x) - 0.5)
        # Used for Optimizer
        with tf.variable_scope('optimizer'):
            self.target_q_t = tf.placeholder('float32', None, name='target_q_t')
            self.action = tf.placeholder('int32', None, name='action')
            action_one_hot = tf.one_hot(self.action, n_output, 1.0, 0.0, name='action_one_hot')
            q_acted = tf.reduce_sum(self.q * action_one_hot, reduction_indices=1, name='q_acted')
            self.delta = self.target_q_t - q_acted
            self.global_step = tf.Variable(0, trainable=False)
            self.loss = tf.reduce_mean(tf.square(self.delta), name='loss')
            self.learning_rate_step = tf.placeholder('int64', None, name='learning_rate_step')
            self.learning_rate_op = tf.maximum(self.learning_rate_minimum,
                                               tf.train.exponential_decay(self.learning_rate, self.learning_rate_step,
                                                                          self.learning_rate_decay_step,
                                                                          self.learning_rate_decay, staircase=True))
            self.optim = tf.train.RMSPropOptimizer(self.learning_rate_op, momentum=0.95, epsilon=0.01).minimize(
                self.loss)

        tf.initialize_all_variables().run()
        self.update_target_q_network()

    def update_target_q_network(self):
        for name in self.w.keys():
            self.t_w_assign_op[name].eval({self.t_w_input[name]: self.w[name].eval()})

    # These two functions are used to save and load weight parameters
    def save_weight_to_pkl(self):
        if not os.path.exists(self.weight_dir):
            os.makedirs(self.weight_dir)
        for name in self.w.keys():
            save_pkl(self.w[name].eval(), os.path.join(self.weight_dir, "%s.pkl" % name))

    def load_weight_from_pkl(self):
        with tf.variable_scope('load_pred_from_pkl'):
            self.w_input = {}
            self.w_assign_op = {}
            for name in self.w.keys():
                self.w_input[name] = tf.placeholder('float32')
                self.w_assign_op[name] = self.w[name].assign(self.w_input[name])
        for name in self.w.keys():
            self.w_assign_op[name].eval({self.w_input[name]: load_pkl(os.path.join(self.weight_dir, "%s.pkl" % name))})
        self.update_target_q_network()


import gradio as gr
import pytesseract
from langchain import PromptTemplate
from langchain.chains import RetrievalQA
from langchain.chat_models import ChatOpenAI
from langchain.embeddings import OpenAIEmbeddings
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.vectorstores import Chroma
from pdf2image import convert_from_path

# 質問テンプレート
template = """
あなたは親切なアシスタントです。下記の質問に日本語で回答してください。
質問：{question}
回答：
"""

prompt = PromptTemplate(
    input_variables=["question"],
    template=template,
)

def pdf_to_text_ocr(pdf_file):
    images = convert_from_path(pdf_file)
    text = ""

    for image in images:
        text += pytesseract.image_to_string(image, lang="jpn+eng")

    return text

def process_input(pdf_file, input_text):
    # PDFファイルの読み込み
    pdf_text = pdf_to_text_ocr(pdf_file.name)

    # テキストの分割
    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
    texts = text_splitter.create_documents([pdf_text])

    # 埋め込みの作成
    embeddings = OpenAIEmbeddings()
    vectordb = Chroma.from_documents(texts, embeddings)

    # RetrievalQAの作成
    qa = RetrievalQA.from_chain_type(llm=ChatOpenAI(model_name="gpt-3.5-turbo"), chain_type="stuff",
                                     retriever=vectordb.as_retriever())

    # 質問の送信と回答の取得
    question = input_text
    query = prompt.format(question=question)
    response = qa.run(query)

    return response

# UIコンポーネントの作成
pdf_upload = gr.inputs.File(type="file", label="PDFファイルをアップロード")
textarea = gr.inputs.Textbox(lines=15, placeholder="GPTの応答がここに表示されます...", label="GPT")
input_box = gr.inputs.Textbox(lines=1, placeholder="ここに質問を入力してください", label="")

iface = gr.Interface(
    fn=process_input,
    inputs=[pdf_upload, input_box],
    outputs=textarea,
    layout="vertical",
    css=".gr-input {width: 80%;}",
    allow_flagging='never'
)

iface.launch()


from selenium.webdriver import Chrome,ChromeOptions
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
import xlsxwriter

opts = ChromeOptions()
opts.add_experimental_option("detach", True)
driver = Chrome(chrome_options=opts)

driver.get("https://google.com")
driver.maximize_window()

searchBox = driver.find_element(By.CLASS_NAME,"gLFyf")

searchBox.send_keys("IBTECH")
searchBox.send_keys(Keys.ENTER)

def purifyExtensions(i, j):
    if '.net' in newLinks[i][j]:
        newRefs.append(newLinks[i][j].split('.net'))
    elif '.io' in newLinks[i][j]:
        newRefs.append(newLinks[i][j].split('.io'))
    elif '.gov' in newLinks[i][j]:
        newRefs.append(newLinks[i][j].split('.gov'))
    elif '.org' in newLinks[i][j]:
        newRefs.append(newLinks[i][j].split('.org'))
    elif '.dev' in newLinks[i][j]:
        newRefs.append(newLinks[i][j].split('.dev'))
    else:
        newRefs.append(newLinks[i][j].split('.com'))

newLinks = []
newRefs = []

for i in range(3):
    links = driver.find_elements(By.CLASS_NAME,"yuRUbf")
    driver.implicitly_wait(3)
    for k in range(1):
        for j in range(9):
            newLinks.append(links[j].text.split('https://'))
    if i == 0:
        for j in range(1):
            for k in range(9):
                purifyExtensions(k,1)
                #newRefs.append(newLinks[k][1].split('.com'))
        driver.find_element(By.XPATH, '//*[@id="pnnext"]/span[2]').click()
    elif i == 1:
        for j in range(1):
            for k in range(9):
                if k ==4:
                    purifyExtensions(k+9,0)
                    #newRefs.append(newLinks[k + 9][0].split('.com'))
                else:
                    purifyExtensions(k + 9, 1)
                    #newRefs.append(newLinks[k + 9][1].split('.com'))
        driver.find_element(By.XPATH, '//*[@id="pnnext"]/span[2]').click()
    elif i == 2:
        for j in range(1):
            for k in range(9):
                purifyExtensions(k+18,1)
                #newRefs.append(newLinks[k+18][1].split('.com'))

driver.close()

workbook = xlsxwriter.Workbook('import_file.xlsx')
worksheet = workbook.add_worksheet()

worksheet.set_column('A:A', len(newLinks))
worksheet.set_column('B:B', len(newLinks))

text1 = 'A{n:.2f}'
text2 = 'B{n:.2f}'
for j in range(len(newLinks)):
    for k in range(1):
        stringLink = newLinks[j][k]
        stringRef = newRefs[j][k]
        worksheet.write(text1.format(n = j+1), stringLink)
        worksheet.write(text2.format(n = j+1), 'https://' + stringRef + '.com')

workbook.close()


# ##### BEGIN GPL LICENSE BLOCK #####
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU General Public License
#  as published by the Free Software Foundation; either version 2
#  of the License, or (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software Foundation,
#  Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# ##### END GPL LICENSE BLOCK #####

import bpy
from bpy.types import Header, Menu, Panel, PropertyGroup

from fd_datablocks import enums, const

import os

from bpy.props import (StringProperty,
                       BoolProperty,
                       IntProperty,
                       FloatProperty,
                       FloatVectorProperty,
                       BoolVectorProperty,
                       PointerProperty,
                       CollectionProperty,
                       EnumProperty)

def find_node(material, nodetype):
    if material and material.node_tree:
        ntree = material.node_tree

        for node in ntree.nodes:
            if getattr(node, "type", None) == nodetype:
                return node

    return None

def find_node_input(node, name):
    for input in node.inputs:
        if input.name == name:
            return input

    return None

def panel_node_draw(layout, id_data, output_type, input_name):
    if not id_data.use_nodes:
        layout.operator("cycles.use_shading_nodes", icon='NODETREE')
        return False

    ntree = id_data.node_tree

    node = find_node(id_data, output_type)
    if not node:
        layout.label(text="No output node")
    else:
        input = find_node_input(node, input_name)
        layout.template_node_view(ntree, node, input)

    return True

class PANEL_scenes(Panel):
    bl_space_type = "VIEW_3D"
    bl_region_type = "TOOLS"
    bl_category = "Scenes"
    bl_context = "objectmode"
    bl_label = " "
    bl_options = {'HIDE_HEADER'}
    #bl_idname = "mvProject.part_properties"

    def draw_header(self,context):
        layout = self.layout
        row = layout.row(align=True)
        row.label("Scenes:        ",icon='SCENE_DATA')

    @classmethod
    def poll(cls, context):
        return True

    def draw(self, context):
        unit = context.scene.unit_settings
        scene = context.scene
        layout = self.layout
        space = context.space_data

        col = layout.column(align=True)
        box = col.box()
        row = box.row(align=True)
        row.template_ID(context.screen, "scene", new="fd_scene.create_scene", unlink="scene.delete")
        box = col.box()
        row = box.row()
        row.prop(scene, "camera",text="Active Camera")
        row = box.row()
        row.label("Main Units:")
        row.row().prop(unit, "system", expand=True)
        row = box.row()
        row.label("Angle Units:")
        row.row().prop(unit, "system_rotation", expand=True)
        if space.type == 'VIEW_3D' and scene.unit_settings.system == 'NONE':
            row = box.row()
            row.label("Grid Spacing:")
            row.row().prop(space, "grid_scale", expand=True)
        box = col.box()
        scene.mv.PromptPage.draw_prompt_page(box,scene)

class PANEL_worlds(Panel):
    bl_space_type = "VIEW_3D"
    bl_region_type = "TOOLS"
    bl_category = "Worlds"
    bl_context = "objectmode"
    bl_label = " "
    bl_options = {'HIDE_HEADER'}
    #bl_idname = "mvProject.part_properties"

    def draw_header(self,context):
        layout = self.layout
        row = layout.row(align=True)
        row.label("World Management:        ",icon=const.icon_world)

    @classmethod
    def poll(cls, context):
        return True

    def draw(self, context):
        scene = context.scene
        world = context.scene.world
        layout = self.layout
        col = layout.column(align=True)
        box = col.box()
        row = box.row(align=True)
        row.template_ID(context.scene, "world", new="world.new")
        box = col.box()
        if not panel_node_draw(box, world, 'OUTPUT_WORLD', 'Surface'):
            box.prop(world, "horizon_color", text="Color")
        box = col.box()
        world.mv.PromptPage.draw_prompt_page(box,world)

class PANEL_materials(Panel):
    bl_space_type = "VIEW_3D"
    bl_region_type = "TOOLS"
    bl_category = "Materials"
    bl_context = "objectmode"
    bl_label = " "
    bl_options = {'HIDE_HEADER'}

    def draw_header(self,context):
        layout = self.layout
        row = layout.row(align=True)
        row.label("Material Management:        ",icon=const.icon_material)

    @classmethod
    def poll(cls, context):
        return True

    def draw(self, context):
        layout = self.layout
        box = layout.box()
        row = box.row()
        row.operator("fd_material.apply_materials_from_pointers",text="Assign Materials",icon=const.icon_material)
        row.operator("fd_material.clear_unused_materials_from_file",text="Clear Unused",icon='ZOOMOUT')
        row.operator("fd_material.clear_all_materials_from_file",text="Clear All",icon='PANEL_CLOSE')
        box.template_list("MATERIAL_UL_matslots", "", bpy.data, "materials", context.scene.mv, "active_material_index", rows=5)
        if len(bpy.data.materials) > 0:
            box = layout.box()
            material = bpy.data.materials[context.scene.mv.active_material_index]
            material.mv.draw_properties(box,material)

class PANEL_libraries(Panel):
    bl_space_type = "VIEW_3D"
    bl_region_type = "TOOLS"
    bl_category = "Libraries"
    bl_context = "objectmode"
    bl_label = " "
    bl_options = {'HIDE_HEADER'}

    def draw_header(self,context):
        layout = self.layout
        row = layout.row(align=True)
        row.label("Library Management:        ",icon=const.icon_library)

    @classmethod
    def poll(cls, context):
        return True

    def draw(self, context):
        dm = context.scene.mv.dm
        layout = self.layout
        col = layout.column(align=True)
        box = col.box()
        if os.path.exists(dm.Libraries.path):
            Libraries = context.scene.mv.dm.Libraries
            row = box.row(align=True)
            row.prop(dm.Libraries,"path",text="",icon='FILE_TICK')
#             box = col.box()
#             row = box.row(align=True)
#             Libraries.draw_active_pointer_library_menus(row)
        else:
            row = box.row(align=True)
            row.prop(dm.Libraries,"path",text="",icon='ERROR')
        dm.Specgroups.draw_spec_groups(box)

#------REGISTER
classes = [
           PANEL_scenes,
           PANEL_worlds,
           PANEL_materials,
           PANEL_libraries
           ]

def register():
    for c in classes:
        bpy.utils.register_class(c)

def unregister():
    for c in classes:
        bpy.utils.unregister_class(c)

if __name__ == "__main__":
    register()


import os
from unittest import TestCase

import jinja2

from apply.issue.issure_js_auto_code.db_util import res_to_dict
from config.db_conf import localhost_oa_engine
from util.str_util import to_lower_camel, to_snake, to_upper_camel

class Form:

    @staticmethod
    def get_tables(db):
        sql = "select TABLE_NAME from INFORMATION_SCHEMA.TABLES where TABLE_SCHEMA = %(db)s"
        res = localhost_oa_engine.execute(sql, {"db": db})
        return res_to_dict(res)

    @staticmethod
    def get_table_info(table_name, db=None):
        # 表名,表注释
        # 字段名,字段类型,字段注释, 枚举值

        # json数据 -- class数据-- 类数据
        sql = """SELECT COL.COLUMN_NAME, COL.COLUMN_TYPE, COL.COLUMN_COMMENT, COL.DATA_TYPE
        FROM INFORMATION_SCHEMA.COLUMNS COL
        Where COL.table_schema = %(db)s AND COL.TABLE_NAME = %(table_name)s"""

        args = {"db": db, "table_name": table_name}
        res = localhost_oa_engine.execute(sql, args)
        data = res_to_dict(res)
        for item in data:
            item["COLUMN_NAME"] = to_lower_camel(item["COLUMN_NAME"])
        return data

    @staticmethod
    def to_file(data_dic, path, resource_dir, template_file):
        template_loader = jinja2.FileSystemLoader(searchpath=resource_dir)
        template_env = jinja2.Environment(loader=template_loader)
        template = template_env.get_template(template_file)
        output_text = template.render(data_dic)
        with open(path, "w", encoding="utf-8") as f:
            f.write(output_text)

class TestAutoCode(TestCase):

    def test_run(self):
        table = "organization"
        rows = Form.get_table_info(table, "oa")
        data = {
            "list": rows
        }
        Form.to_file(data, f"{table}.vue", os.path.dirname(__file__), "templates/vue.template")

    def test_run_vue_js(self):
        tables = Form.get_tables("oa")
        for table in tables:
            table_name = table["TABLE_NAME"]
            table_upper_caml = to_upper_camel(table_name)
            rows = Form.get_table_info(table_name, "oa")
            data = {
                "list": rows,
                "tableUpperCaml": table_upper_caml,
                "tableConst": to_snake(table_name).upper(),
            }
            Form.to_file(data, f"tmp/{table_upper_caml}.vue", os.path.dirname(__file__), "templates/vue.template")

    def test_run_index_js(self):
        tables = Form.get_tables("oa")
        table_infos = []
        for table in tables:
            table_name = table["TABLE_NAME"]
            table_upper_caml = to_upper_camel(table_name)
            table_infos.append({
                "tableUpperCaml": table_upper_caml,
                "tableLowerCaml": to_lower_camel(table_name),
                "tableConst": to_snake(table_name).upper(),
            })
        data = {
            "list": table_infos
        }
        Form.to_file(data, "tmp/routes.js", os.path.dirname(__file__), "templates/routes.template")

    def test_run_config_js(self):
        tables = Form.get_tables("oa")
        table_infos = []
        for table in tables:
            table_name = table["TABLE_NAME"]
            table_infos.append({
                "tableLowerCaml": to_lower_camel(table_name),
                "tableConst": to_snake(table_name).upper(),
            })
        data = {
            "list": table_infos
        }
        Form.to_file(data, "tmp/config.js", os.path.dirname(__file__), "templates/config.template")


import cv2
import numpy as np

from .base import BaseTask

class BlurAndPHash(BaseTask):
    def __init__(self):
        super().__init__(taskID=4, taskName='BlurAndPHash')
        self.thresholdLaplacian = 120
        self.thresholdDiffStop = 120
        self.thresholdDiffPre = 25
        self.hashLen = 32
        self.preStopPHash = None
        self.prePHash = None
        self.n = 0

    def exec(self, inputData):
        frame, isLastFrame = inputData
        if isLastFrame:
            return None, isLastFrame

        currPHash = self.getPHash(frame)
        if currPHash is None:
            return None

        if self.preStopPHash is None:
            self.preStopPHash = currPHash
            self.prePHash = currPHash
            return frame, isLastFrame

        diffStop = self.hamDistance(self.preStopPHash, currPHash)
        diffPre = self.hamDistance(self.prePHash, currPHash)
        self.prePHash = currPHash

        if diffStop >= self.thresholdDiffStop \
                or diffPre <= self.thresholdDiffPre:
            return None

        self.n += 1
        if self.n <= 3:
            return None
        self.n = 0
        self.preStopPHash = currPHash
        return frame, isLastFrame

    def getPHash(self, img):
        pHash = None
        laplacian = cv2.Laplacian(img, cv2.CV_64F).var()
        if laplacian <= self.thresholdLaplacian:
            return pHash
        imgGray = cv2.resize(
            cv2.cvtColor(img, cv2.COLOR_RGB2GRAY),
            (self.hashLen, self.hashLen),
            cv2.INTER_AREA)
        height, width = imgGray.shape[:2]
        matrixOriginal = np.zeros(
            (height, width),
            np.float32)
        matrixOriginal[:height, :width] = imgGray

        matrix = cv2.dct(cv2.dct(matrixOriginal))
        matrix.resize(self.hashLen, self.hashLen)
        matrixFlatten = matrix.flatten()

        medianValue = sum(matrixFlatten) * 1. / len(matrixFlatten)
        pHash = 0
        for i in matrixFlatten:
            pHash <<= 1
            if i >= medianValue:
                pHash += 1
        return pHash

    @staticmethod
    def hamDistance(x, y):
        tmp = x ^ y
        distance = 0

        while tmp > 0:
            distance += tmp & 1
            tmp >>= 1

        return distance


try:
	from zohocrmsdk.src.com.zoho.crm.api.exception import SDKException
	from zohocrmsdk.src.com.zoho.crm.api.util import Constants
except Exception:
	from ..exception import SDKException
	from ..util import Constants

class Backup(object):
	def __init__(self):
		"""Creates an instance of Backup"""

		self.__rrule = None
		self.__id = None
		self.__start_date = None
		self.__scheduled_date = None
		self.__status = None
		self.__requester = None
		self.__key_modified = dict()

	def get_rrule(self):
		"""
		The method to get the rrule

		Returns:
			string: A string representing the rrule
		"""

		return self.__rrule

	def set_rrule(self, rrule):
		"""
		The method to set the value to rrule

		Parameters:
			rrule (string) : A string representing the rrule
		"""

		if rrule is not None and not isinstance(rrule, str):
			raise SDKException(Constants.DATA_TYPE_ERROR, 'KEY: rrule EXPECTED TYPE: str', None, None)

		self.__rrule = rrule
		self.__key_modified['rrule'] = 1

	def get_id(self):
		"""
		The method to get the id

		Returns:
			int: An int representing the id
		"""

		return self.__id

	def set_id(self, id):
		"""
		The method to set the value to id

		Parameters:
			id (int) : An int representing the id
		"""

		if id is not None and not isinstance(id, int):
			raise SDKException(Constants.DATA_TYPE_ERROR, 'KEY: id EXPECTED TYPE: int', None, None)

		self.__id = id
		self.__key_modified['id'] = 1

	def get_start_date(self):
		"""
		The method to get the start_date

		Returns:
			datetime: An instance of datetime
		"""

		return self.__start_date

	def set_start_date(self, start_date):
		"""
		The method to set the value to start_date

		Parameters:
			start_date (datetime) : An instance of datetime
		"""

		from datetime import datetime

		if start_date is not None and not isinstance(start_date, datetime):
			raise SDKException(Constants.DATA_TYPE_ERROR, 'KEY: start_date EXPECTED TYPE: datetime', None, None)

		self.__start_date = start_date
		self.__key_modified['start_date'] = 1

	def get_scheduled_date(self):
		"""
		The method to get the scheduled_date

		Returns:
			datetime: An instance of datetime
		"""

		return self.__scheduled_date

	def set_scheduled_date(self, scheduled_date):
		"""
		The method to set the value to scheduled_date

		Parameters:
			scheduled_date (datetime) : An instance of datetime
		"""

		from datetime import datetime

		if scheduled_date is not None and not isinstance(scheduled_date, datetime):
			raise SDKException(Constants.DATA_TYPE_ERROR, 'KEY: scheduled_date EXPECTED TYPE: datetime', None, None)

		self.__scheduled_date = scheduled_date
		self.__key_modified['scheduled_date'] = 1

	def get_status(self):
		"""
		The method to get the status

		Returns:
			string: A string representing the status
		"""

		return self.__status

	def set_status(self, status):
		"""
		The method to set the value to status

		Parameters:
			status (string) : A string representing the status
		"""

		if status is not None and not isinstance(status, str):
			raise SDKException(Constants.DATA_TYPE_ERROR, 'KEY: status EXPECTED TYPE: str', None, None)

		self.__status = status
		self.__key_modified['status'] = 1

	def get_requester(self):
		"""
		The method to get the requester

		Returns:
			Requester: An instance of Requester
		"""

		return self.__requester

	def set_requester(self, requester):
		"""
		The method to set the value to requester

		Parameters:
			requester (Requester) : An instance of Requester
		"""

		try:
			from zohocrmsdk.src.com.zoho.crm.api.backup.requester import Requester
		except Exception:
			from .requester import Requester

		if requester is not None and not isinstance(requester, Requester):
			raise SDKException(Constants.DATA_TYPE_ERROR, 'KEY: requester EXPECTED TYPE: Requester', None, None)

		self.__requester = requester
		self.__key_modified['requester'] = 1

	def is_key_modified(self, key):
		"""
		The method to check if the user has modified the given key

		Parameters:
			key (string) : A string representing the key

		Returns:
			int: An int representing the modification
		"""

		if key is not None and not isinstance(key, str):
			raise SDKException(Constants.DATA_TYPE_ERROR, 'KEY: key EXPECTED TYPE: str', None, None)

		if key in self.__key_modified:
			return self.__key_modified.get(key)

		return None

	def set_key_modified(self, key, modification):
		"""
		The method to mark the given key as modified

		Parameters:
			key (string) : A string representing the key
			modification (int) : An int representing the modification
		"""

		if key is not None and not isinstance(key, str):
			raise SDKException(Constants.DATA_TYPE_ERROR, 'KEY: key EXPECTED TYPE: str', None, None)

		if modification is not None and not isinstance(modification, int):
			raise SDKException(Constants.DATA_TYPE_ERROR, 'KEY: modification EXPECTED TYPE: int', None, None)

		self.__key_modified[key] = modification


# encoding=utf-8
import logging
import logging.config
import os
import sys
import time
import traceback
import datetime

def init_log(name='root'):
    path = os.path.dirname(__file__)

    config_file = path + os.sep + 'logger.conf'
    log_path = os.path.join(os.path.abspath(__file__ + ('/..' * 3)), 'zz_logs')
    if not os.path.exists(log_path):
        os.makedirs(log_path)

    log_path = os.path.join(log_path, str(datetime.datetime.now().date()) + '.log')

    if os.path.isfile(config_file) is False:
        raise Exception("Config file {} not found".format(config_file))

    datalines = list()
    with open(config_file, 'r') as f:
        for data in f.readlines():
            if '$path' in data:
                data = data.replace('$path', log_path)
            datalines.append(data)
    f = open(config_file + '_bak', 'w')
    f.writelines(datalines)
    f.close()

    del datalines

    logging.config.fileConfig(config_file + '_bak')

    # os.remove(config_file + '_bak')

    return logging.getLogger(name)

# decorator print log
def addlog(name=''):
    begin = time.time()

    def _addlog(func):
        def wapper(*args, **kwargs):
            data = None
            begin1 = time.time()
            try:
                s = traceback.extract_stack()
                file = s[-2][0]
                __project_name = os.path.abspath(__file__ + ('/..' * 3))
                file_name = file[file.find(__project_name) + len(__project_name) + 1:file.rfind(r'.')]
                func_descrip = (file_name + '.' + func.__name__) if name == '' else name

                log.info('Start Execute：%s ...' % func_descrip)
                data = func(*args, **kwargs)

                inner_secs = time.time() - begin1

                log.info('Complete：%s , Time Consume: %s, Total Time: %s ' % (func_descrip,
                                                         time_str(inner_secs), time_str(time.time() - begin)))

            except Exception as e:
                # traceback.print_exc()
                log.exception('Failure Calling Time Consume:%s, Total Time:%s, Err Message:%s', time_str(time.time() - begin1),
                              time_str(time.time() - begin), e)
                # traceback.print_exc(file=open(log_file, 'a'))
                sys.exit(0)

            return data

        return wapper

    return _addlog

def time_str(second):
    return ('%.2f sec' % second) if second < 60 else ('%.2f min' % (second / 60.0))

log = init_log()

# example
@addlog()
def log_test1():
    time.sleep(1)

@addlog(name='test2')
def log_test2():
    time.sleep(1)
    log_test1()
    time.sleep(2)
    raise ValueError('A very specific bad thing happened.')

if __name__ == "__main__":
    col = 'aaaa'
    missing_rate = 0.26587

    log.info('%s has missing rate as %f' % (col, missing_rate))
    # log_test2()
    #
    # __project_name = os.path.abspath(__file__ + ('/..' * 3))
    # print(__project_name)

    log.debug('debug')

    log.info('test  -   debug')

    log.warning('warining')


import os
import pathlib
import pandas as pd
import keyring
import dropbox
from dropbox.exceptions import AuthError

# Directory
BASE_DIR = os.path.dirname(os.path.abspath(__file__))

dropbox_home = "https://www.dropbox.com/home/"
dropbox_app = "MAD_WahooToGarmin"
dropbox_app_dir = "/Apps/WahooFitness/"
DROPBOX_ACCESS_TOKEN = keyring.get_password("dropbox", dropbox_app)

# https://practicaldatascience.co.uk/data-science/how-to-use-the-dropbox-api-with-python

def dropbox_connect():
    """Create a connection to Dropbox."""

    try:
        dbx = dropbox.Dropbox(DROPBOX_ACCESS_TOKEN)
    except AuthError as e:
        print("Error connecting to Dropbox with access token: " + str(e))
    return dbx

def dropbox_list_files(path):
    """Return a Pandas dataframe of files in a given Dropbox folder path in the Apps directory.
    """

    dbx = dropbox_connect()

    try:
        files = dbx.files_list_folder(path).entries
        files_list = []
        for file in files:
            if isinstance(file, dropbox.files.FileMetadata):
                metadata = {
                    "filename": file.name,
                    "path_display": file.path_display,
                    "client_modified": pd.Timestamp(file.client_modified).isoformat(),
                    "server_modified": pd.Timestamp(file.server_modified).isoformat(),
                }
                files_list.append(metadata)

        df = pd.DataFrame.from_records(files_list)
        return df.sort_values(by="server_modified", ascending=False)

    except Exception as e:
        print("Error getting list of files from Dropbox: " + str(e))

def dropbox_download_file(dropbox_file_path, local_file_path):
    """Download a file from Dropbox to the local machine."""

    try:
        dbx = dropbox_connect()

        with open(local_file_path, "wb") as f:
            metadata, result = dbx.files_download(path=dropbox_file_path)
            f.write(result.content)
    except Exception as e:
        print("Error downloading file from Dropbox: " + str(e))

def dropbox_upload_file(local_path, local_file, dropbox_file_path):
    """Upload a file from the local machine to a path in the Dropbox app directory.

    Args:
        local_path (str): The path to the local file.
        local_file (str): The name of the local file.
        dropbox_file_path (str): The path to the file in the Dropbox app directory.

    Example:
        dropbox_upload_file('.', 'test.csv', '/stuff/test.csv')

    Returns:
        meta: The Dropbox file metadata.
    """

    try:
        dbx = dropbox_connect()

        local_file_path = pathlib.Path(local_path) / local_file

        with local_file_path.open("rb") as f:
            meta = dbx.files_upload(
                f.read(), dropbox_file_path, mode=dropbox.files.WriteMode("overwrite")
            )

            return meta
    except Exception as e:
        print("Error uploading file to Dropbox: " + str(e))

if __name__ == "__main__":
    print("here")


import re

import pep8
import six

"""
Guidelines for writing new hacking checks

 - Use only for Octavia specific tests. OpenStack general tests
   should be submitted to the common 'hacking' module.
 - Pick numbers in the range O3xx. Find the current test with
   the highest allocated number and then pick the next value.
 - Keep the test method code in the source file ordered based
   on the O3xx value.
 - List the new rule in the top level HACKING.rst file
 - Add test cases for each new rule to
   octavia/tests/unit/test_hacking.py

"""

log_translation = re.compile(
    r"(.)*LOG\.(audit|error|info|warn|warning|critical|exception)\(\s*('|\")")
author_tag_re = (re.compile("^\s*#\s*@?(a|A)uthor"),
                 re.compile("^\.\.\s+moduleauthor::"))
_all_hints = set(['_', '_LI', '_LE', '_LW', '_LC'])
_all_log_levels = {
    # NOTE(yamamoto): Following nova which uses _() for audit.
    'audit': '_',
    'error': '_LE',
    'info': '_LI',
    'warn': '_LW',
    'warning': '_LW',
    'critical': '_LC',
    'exception': '_LE',
}
log_translation_hints = []
for level, hint in six.iteritems(_all_log_levels):
    r = "(.)*LOG\.%(level)s\(\s*((%(wrong_hints)s)\(|'|\")" % {
        'level': level,
        'wrong_hints': '|'.join(_all_hints - set([hint])),
    }
    log_translation_hints.append(re.compile(r))

assert_trueinst_re = re.compile(
    r"(.)*assertTrue\(isinstance\((\w|\.|\'|\"|\[|\])+, "
    "(\w|\.|\'|\"|\[|\])+\)\)")
assert_equal_in_end_with_true_or_false_re = re.compile(
    r"assertEqual\((\w|[][.'\"])+ in (\w|[][.'\", ])+, (True|False)\)")
assert_equal_in_start_with_true_or_false_re = re.compile(
    r"assertEqual\((True|False), (\w|[][.'\"])+ in (\w|[][.'\", ])+\)")
assert_equal_with_true_re = re.compile(
    r"assertEqual\(True,")
assert_equal_with_false_re = re.compile(
    r"assertEqual\(False,")
mutable_default_args = re.compile(r"^\s*def .+\((.+=\{\}|.+=\[\])")
assert_equal_end_with_none_re = re.compile(r"(.)*assertEqual\(.+, None\)")
assert_equal_start_with_none_re = re.compile(r".*assertEqual\(None, .+\)")
assert_not_equal_end_with_none_re = re.compile(
    r"(.)*assertNotEqual\(.+, None\)")
assert_not_equal_start_with_none_re = re.compile(
    r"(.)*assertNotEqual\(None, .+\)")
assert_no_xrange_re = re.compile(
    r"\s*xrange\s*\(")

def _directory_to_check_translation(filename):
    return True

def assert_true_instance(logical_line):
    """Check for assertTrue(isinstance(a, b)) sentences

    O316
    """
    if assert_trueinst_re.match(logical_line):
        yield (0, "O316: assertTrue(isinstance(a, b)) sentences not allowed")

def assert_equal_or_not_none(logical_line):
    """Check for assertEqual(A, None) or assertEqual(None, A) sentences,

    assertNotEqual(A, None) or assertNotEqual(None, A) sentences

    O318
    """
    msg = ("O318: assertEqual/assertNotEqual(A, None) or "
           "assertEqual/assertNotEqual(None, A) sentences not allowed")
    res = (assert_equal_start_with_none_re.match(logical_line) or
           assert_equal_end_with_none_re.match(logical_line) or
           assert_not_equal_start_with_none_re.match(logical_line) or
           assert_not_equal_end_with_none_re.match(logical_line))
    if res:
        yield (0, msg)

def no_translate_debug_logs(logical_line, filename):
    """Check for 'LOG.debug(_('

    As per our translation policy,
    https://wiki.openstack.org/wiki/LoggingStandards#Log_Translation
    we shouldn't translate debug level logs.

    * This check assumes that 'LOG' is a logger.
    O319
    """
    if _directory_to_check_translation(filename) and logical_line.startswith(
            "LOG.debug(_("):
        yield(0, "O319 Don't translate debug level logs")

def validate_log_translations(logical_line, physical_line, filename):
    # Translations are not required in the test directory
    if "octavia/tests" in filename:
        return
    if pep8.noqa(physical_line):
        return
    msg = "O320: Log messages require translations!"
    if log_translation.match(logical_line):
        yield (0, msg)

    if _directory_to_check_translation(filename):
        msg = "O320: Log messages require translation hints!"
        for log_translation_hint in log_translation_hints:
            if log_translation_hint.match(logical_line):
                yield (0, msg)

def use_jsonutils(logical_line, filename):
    msg = "O321: jsonutils.%(fun)s must be used instead of json.%(fun)s"

    # Some files in the tree are not meant to be run from inside Octavia
    # itself, so we should not complain about them not using jsonutils
    json_check_skipped_patterns = [
    ]

    for pattern in json_check_skipped_patterns:
        if pattern in filename:
            return

    if "json." in logical_line:
        json_funcs = ['dumps(', 'dump(', 'loads(', 'load(']
        for f in json_funcs:
            pos = logical_line.find('json.%s' % f)
            if pos != -1:
                yield (pos, msg % {'fun': f[:-1]})

def no_author_tags(physical_line):
    for regex in author_tag_re:
        if regex.match(physical_line):
            physical_line = physical_line.lower()
            pos = physical_line.find('moduleauthor')
            if pos < 0:
                pos = physical_line.find('author')
            return pos, "O322: Don't use author tags"

def assert_equal_true_or_false(logical_line):
    """Check for assertEqual(True, A) or assertEqual(False, A) sentences

    O323
    """
    res = (assert_equal_with_true_re.search(logical_line) or
           assert_equal_with_false_re.search(logical_line))
    if res:
        yield (0, "O323: assertEqual(True, A) or assertEqual(False, A) "
               "sentences not allowed")

def no_mutable_default_args(logical_line):
    msg = "O324: Method's default argument shouldn't be mutable!"
    if mutable_default_args.match(logical_line):
        yield (0, msg)

def assert_equal_in(logical_line):
    """Check for assertEqual(A in B, True), assertEqual(True, A in B),

    assertEqual(A in B, False) or assertEqual(False, A in B) sentences

    O338
    """
    res = (assert_equal_in_start_with_true_or_false_re.search(logical_line) or
           assert_equal_in_end_with_true_or_false_re.search(logical_line))
    if res:
        yield (0, "O338: Use assertIn/NotIn(A, B) rather than "
               "assertEqual(A in B, True/False) when checking collection "
               "contents.")

def no_log_warn(logical_line):
    """Disallow 'LOG.warn('

    O339
    """
    if logical_line.startswith('LOG.warn('):
        yield(0, "O339:Use LOG.warning() rather than LOG.warn()")

def no_xrange(logical_line):
    """Disallow 'xrange()'

    O340
    """
    if assert_no_xrange_re.match(logical_line):
        yield(0, "O340: Do not use xrange().")

def factory(register):
    register(assert_true_instance)
    register(assert_equal_or_not_none)
    register(no_translate_debug_logs)
    register(validate_log_translations)
    register(use_jsonutils)
    register(no_author_tags)
    register(assert_equal_true_or_false)
    register(no_mutable_default_args)
    register(assert_equal_in)
    register(no_log_warn)
    register(no_xrange)


"""empty message

Revision ID: a5cfe890710d
Revises: 7352c721e0a4
Create Date: 2023-05-28 16:47:42.177222

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'a5cfe890710d'
down_revision = '7352c721e0a4'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table('images', schema=None) as batch_op:
        batch_op.drop_column('url_small')
        batch_op.drop_column('url_full')
        batch_op.drop_column('url_regular')

    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table('images', schema=None) as batch_op:
        batch_op.add_column(sa.Column('url_regular', sa.VARCHAR(length=500), autoincrement=False, nullable=True))
        batch_op.add_column(sa.Column('url_full', sa.VARCHAR(length=500), autoincrement=False, nullable=True))
        batch_op.add_column(sa.Column('url_small', sa.VARCHAR(length=500), autoincrement=False, nullable=True))

    # ### end Alembic commands ###


import requests
from bs4 import BeautifulSoup
import json

def get_description(url):
    response = requests.get(url)
    if response is not None:
        soup = BeautifulSoup(response.text, 'html.parser')

    description = {}

    l1 = []
    l2 = []
    for item in soup.find_all("span", class_="adPage__content__features__key"):
        if item is not None:
            l1.append(item.text)

    for item in soup.find_all("span", class_="adPage__content__features__value"):
        if item is not None:
            l2.append(item.text)

    for index in range(len(l2)):
        description[l1[index]] = l2[index]

    extra = []
    for index in range(len(l2)+1,len(l1)):
        extra.append(index)
    description['Extra_features'] = extra

    if soup.find('h1') is not None:
        car_model = soup.find('h1').text

    des = soup.find('div', class_=("adPage__content__description grid_18"))
    if des is not None:
        description["description"] = des.text
    print(description)
    desc = {}

    desc[car_model] = description

    file_name = "description"

    with open(file_name,"a") as json_file:
        json.dump(desc, json_file)


"""Module containing class `UntagClipsCommand`."""

import logging
import random
import time

from django.db import transaction

from vesper.command.clip_set_command import ClipSetCommand
from vesper.django.app.models import Job, Tag, TagEdit, TagInfo
import vesper.command.command_utils as command_utils
import vesper.django.app.model_utils as model_utils
import vesper.util.archive_lock as archive_lock
import vesper.util.text_utils as text_utils
import vesper.util.time_utils as time_utils

_logger = logging.getLogger()

class TagClipsCommand(ClipSetCommand):

    extension_name = 'tag_clips'

    def __init__(self, args):

        super().__init__(args, True)

        get_opt = command_utils.get_optional_arg
        self._clip_count = get_opt('clip_count', args)

    def execute(self, job_info):
        self._job_info = job_info
        clip_indices = self._get_tag_clip_indices()
        self._tag_clips(clip_indices)
        return True

    def _get_tag_clip_indices(self):

        if self._clip_count is None:
            # tag all clips

            return None

        clip_count = self._count_clips()

        if clip_count <= self._clip_count:
            # tag all clips

            return None

        # If we get here, a clip count is specified and it is less than
        # the number of untagged clips.

        _logger.info('Getting indices of clips to tag...')

        indices = random.sample(range(clip_count), self._clip_count)

        return frozenset(indices)

    def _count_clips(self):

        value_tuples = self._create_clip_query_values_iterator()
        count = 0

        for station, mic_output, date, detector in value_tuples:

            clips = model_utils.get_clips(
                station=station,
                mic_output=mic_output,
                date=date,
                detector=detector,
                annotation_name=self._annotation_name,
                annotation_value=self._annotation_value,
                tag_name=self._tag_name,
                tag_excluded=True,
                order=False)

            count += clips.count()

        return count

    def _tag_clips(self, clip_indices):

        start_time = time.time()

        value_tuples = self._create_clip_query_values_iterator()

        clip_index = 0
        total_clip_count = 0
        total_tagged_count = 0

        for station, mic_output, date, detector in value_tuples:

            # Get clip for this station, mic_output, date, and detector.
            clips = model_utils.get_clips(
                station=station,
                mic_output=mic_output,
                date=date,
                detector=detector,
                annotation_name=self._annotation_name,
                annotation_value=self._annotation_value,
                tag_name=self._tag_name,
                tag_excluded=True,
                order=False)

            # Get list of clip IDs.
            clip_ids = clips.values_list('pk', flat=True)

            # Get IDs of clips to tag.
            tag_clip_ids = \
                self._get_tag_clip_ids(clip_ids, clip_index, clip_indices)
            clip_count = len(clip_ids)
            tagged_count = len(tag_clip_ids)
            clip_index += clip_count

            # Tag clips.
            try:
                self._tag_clip_batch(tag_clip_ids)
            except Exception as e:
                batch_text = \
                    _get_batch_text(station, mic_output, date, detector)
                command_utils.log_and_reraise_fatal_exception(
                    e, f'Tagging of clips for {batch_text}')

            # Log clip counts.
            if tagged_count == clip_count:
                prefix = 'Tagged'
            else:
                untagged_count = clip_count - tagged_count
                prefix = (
                    f'Tagged {tagged_count} and left untagged '
                    f'{untagged_count} of')
            count_text = text_utils.create_count_text(clip_count, 'clip')
            batch_text = _get_batch_text(station, mic_output, date, detector)
            _logger.info(f'{prefix} {count_text} for {batch_text}.')

            total_clip_count += clip_count
            total_tagged_count += tagged_count

        # Log total clip counts and tagging rate.
        if total_tagged_count == total_clip_count:
            prefix = 'Tagged'
        else:
            total_untagged_count = total_clip_count - total_tagged_count
            prefix = (
                f'Tagged {total_tagged_count} and left untagged '
                f'{total_untagged_count} of')
        count_text = text_utils.create_count_text(total_clip_count, 'clip')
        elapsed_time = time.time() - start_time
        timing_text = command_utils.get_timing_text(
            elapsed_time, total_clip_count, 'clips')
        _logger.info(f'{prefix} a total of {count_text}{timing_text}.')

    def _get_tag_clip_ids(self, clip_ids, start_clip_index, clip_indices):

        if clip_indices is None:
            # tagging all clips

            return clip_ids

        else:
            # not tagging all clips

            clip_index = start_clip_index
            tag_clip_ids = []

            for clip_id in clip_ids:

                if clip_index in clip_indices:
                    tag_clip_ids.append(clip_id)

                clip_index += 1

            return tag_clip_ids

    def _tag_clip_batch(self, clip_ids):

        with archive_lock.atomic():

            with transaction.atomic():

                # See note in untag_clips_command.py about maximum
                # chunk size. I'm not certain we have to do the same
                # thing here, but it seems likely that we do, for a
                # similar reason.
                max_chunk_size = 900

                tag_info = TagInfo.objects.get(name=self._tag_name)
                action = TagEdit.ACTION_SET
                creation_time = time_utils.get_utc_now()
                creating_job = Job.objects.get(id=self._job_info.job_id)

                for i in range(0, len(clip_ids), max_chunk_size):

                    chunk = clip_ids[i:i + max_chunk_size]

                    # Create tags.
                    Tag.objects.bulk_create([
                        Tag(
                            clip_id=clip_id,
                            info=tag_info,
                            creation_time=creation_time,
                            creating_user=None,
                            creating_job=creating_job,
                            creating_processor=None)
                        for clip_id in chunk])

                    # Create tag edits.
                    TagEdit.objects.bulk_create([
                        TagEdit(
                            clip_id=clip_id,
                            info=tag_info,
                            action=action,
                            creation_time=creation_time,
                            creating_user=None,
                            creating_job=creating_job,
                            creating_processor=None)
                        for clip_id in chunk])

def _get_batch_text(station, mic_output, date, detector):
    return (
        f'station "{station.name}", mic output "{mic_output.name}", '
        f'date {date}, and detector "{detector.name}"')


import string, itertools

ascii_lowercases = list(string.ascii_lowercase)

MAX_WORD_LENGTH = 5

for i in range(1, MAX_WORD_LENGTH + 1):
    charlist = [[x for x in ascii_lowercases]] * i

    for combinations in itertools.product(*charlist):
        combinations = "".join(combinations)
        with open("../wordlist.txt", "a") as file:
            file.write(combinations + "\n")
    print("finished!", i)


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression

if __name__ == '__main__':
    # 将csv数据读取为pandas对象
    fund = pd.read_csv('./csv/001112.csv', dtype={'fcode': str})
    # 转化时间字符串为时间
    fund['fdate'] = pd.to_datetime(fund['fdate'])
    # 设置时间列为索引，并升序排列
    fund = fund.set_index('fdate').sort_index(ascending=False)
    # x轴为数据2017后的索引 y为2017后的NAV
    x = fund.loc['2017'].index
    y = fund.loc['2017']['NAV']
    # 将xy数据转化为矩阵:
    # 将时间转化为时间戳
    x = x.astype(np.int64)
    # 将时间戳转化为1列多行的二维数组
    x = x.values.reshape(-1, 1)
    y = y.values.reshape(-1, 1)

    # 放入数据开始训练
    lr = LinearRegression()
    lr.fit(x, y)
    # 构建一个时间戳，来预测y轴的值
    test_x = pd.to_datetime(np.array(['2017-9-30', '2017-10-1'])).astype(np.int64).values.reshape(-1, 1)
    # 预测到Y轴的值 [[1.41483561]
    #  [1.41626252]]
    new_y = lr.predict(test_x)
    # 把拟合线画出来：如果y为训练预测出的值，则线条为直线拟合线
    x_date = fund.loc['2017'].index
    # 走势点图
    plt.scatter(x_date, fund.loc['2017']['NAV'])
    plt.plot(x_date, lr.predict(x), 'r')
    plt.show()
    print(new_y)


import sys, re
from argparse import ArgumentParser #import the library

parser = ArgumentParser(description = 'Classify a sequence as DNA or RNA') #create one ArgumentParser
parser.add_argument("-s", "--seq", type = str, required = True, help = "Input sequence") #add the first argument
parser.add_argument("-m", "--motif", type = str, required = False, help = "Motif") #add the second argument

if len(sys.argv) == 1: #print the help message only if no arguments are supplied on the command line
    parser.print_help()
    sys.exit(1)

args = parser.parse_args() #parser the argument

args.seq = args.seq.upper() #convert the sequence  in upper case

if 'U' in args.seq and 'T' in args.seq: #if it finds U and T in the sequence return a message that the sequence have a mutagenic bases
	print ('The sequence have a mutagenic bases') # if it finds this condition it does not execute the others command line
	sys.exit ()

if re.search('^[ACGTU]+$', args.seq): #search in the sequence the pattern within the string
    if re.search('T', args.seq): #if it finds T in the sequence return a message that the sequence is DNA
        print ('The sequence is DNA')
    elif re.search('U', args.seq):
        print ('The sequence is RNA') #if it finds U in the sequence return a message that the sequence is RNA
    else:
        print ('The sequence can be DNA or RNA') #if it finds T and  U in the sequence return a message that the sequence can be DNA or RNA
else:
    print ('The sequence is not DNA') #else the sequence is not DNA

if args.motif:
    args.motif = args.motif.upper() #converte the motif in upper case
    print(f'Motif search enabled: looking for motif in sequence') #to find simple motifs in the sequence, besides printing the type of molecule (DNA or RNA)
    if re.search(args.motif, args.seq):
        print("FOUND")
    else:
        print("NOT FOUND")


import datetime

def rest_sec_of_day():
    """
    :return: 截止到目前当日剩余时间
    """
    today = datetime.datetime.strptime(str(datetime.date.today()), "%Y-%m-%d")
    tomorrow = today + datetime.timedelta(days=1)
    nowTime = datetime.datetime.now()
    return (tomorrow - nowTime).seconds  # 获取秒


import pandas as pd
import pickle
df=pd.read_csv(r'C:/Users/SAIDHANUSH/spam-ham.csv')

df['Category'].replace('spam',0,inplace=True)
df['Category'].replace('ham',1,inplace=True)

x=df['Message']
y=df['Category']

from sklearn.feature_extraction.text import CountVectorizer
cv=CountVectorizer()

x=cv.fit_transform(x)

pickle.dump(cv,open('transform.pkl','wb'))

from sklearn.model_selection import train_test_split
x_train,x_test,y_train,y_test=train_test_split(x,y,random_state=4,test_size=0.2)

from sklearn.tree import DecisionTreeClassifier
clf=DecisionTreeClassifier()
clf.fit(x_train,y_train)
pr=clf.predict(x_test)
from sklearn.metrics import accuracy_score
print(accuracy_score(y_test,pr))

pickle.dump(clf,open('nlp_model1.pkl','wb'))


#!/usr/bin/env python3
"""
Name: bgp_neighbor_prefix_received.py
Description: NXAPI: display bgp neighbor summary info
"""
our_version = 109
script_name = "bgp_neighbor_prefix_received"
# standard libraries
import argparse
from concurrent.futures import ThreadPoolExecutor

# local libraries
from nxapi_netbox.args.args_cookie import ArgsCookie
from nxapi_netbox.args.args_nxapi_tools import ArgsNxapiTools
from nxapi_netbox.general.log import get_logger
from nxapi_netbox.netbox.netbox_session import netbox, get_device_mgmt_ip
from nxapi_netbox.vault.vault import get_vault
from nxapi_netbox.nxapi.nxapi_bgp_unicast_summary import (
    NxapiBgpUnicastSummaryIpv4,
    NxapiBgpUnicastSummaryIpv6,
)

def get_parser():
    help_afi = "address family to query. one of ipv4 or ipv6."
    help_nonzero = (
        "if specified, only display neighbors with non-zero prefixes received"
    )
    ex_prefix = "Example: "
    ex_afi = "{} --afi ipv6".format(ex_prefix)
    ex_nonzero = "{} --nonzero".format(ex_prefix)

    parser = argparse.ArgumentParser(
        description="DESCRIPTION: display bgp unicast summary info via NXAPI",
        parents=[ArgsCookie, ArgsNxapiTools],
    )
    default = parser.add_argument_group(title="DEFAULT SCRIPT ARGS")
    mandatory = parser.add_argument_group(title="MANDATORY SCRIPT ARGS")

    default.add_argument(
        "--afi",
        dest="afi",
        required=False,
        choices=["ipv4", "ipv6"],
        default="ipv4",
        help="{} {}".format(help_afi, ex_afi),
    )

    default.add_argument(
        "--nonzero",
        dest="nonzero",
        required=False,
        default=False,
        action="store_true",
        help="{} {}".format(help_nonzero, ex_nonzero),
    )

    parser.add_argument(
        "--version", action="version", version="{} v{}".format("%(prog)s", our_version)
    )
    return parser.parse_args()

def get_device_list():
    try:
        return cfg.devices.split(",")
    except:
        log.error(
            "exiting. Cannot parse --devices {}.  Example usage: --devices leaf_1,spine_2,leaf_2".format(
                cfg.devices
            )
        )
        exit(1)

def print_header():
    print(fmt.format("ip", "hostname", "neighbor", "prefix_rx"))

def print_output(futures):
    for future in futures:
        output = future.result()
        if output == None:
            continue
        for line in output:
            print(line)

def collect_prefix_rx(ip, bgp):
    lines = list()
    for neighbor in bgp.neighbor_info:
        bgp.neighbor = neighbor
        try:
            prefixreceived = int(bgp.prefixreceived)
        except:
            log.warning(
                "collect_prefix_rx. {} skipping neighbor {}. cannot convert bgp.prefixreceived {} to int()".format(
                    bgp.hostname, bgp.neighbor, bgp.prefixreceived
                )
            )
            continue
        if prefixreceived == 0 and cfg.nonzero == True:
            continue
        lines.append(fmt.format(ip, bgp.hostname, bgp.neighbor, bgp.prefixreceived))
    lines.append("")
    return lines

def get_instance(ip, vault):
    """
    return a list of NxapiBgpUnicastSummary*() instances based on cfg.afi
    """
    if cfg.afi == "ipv4":
        return NxapiBgpUnicastSummaryIpv4(
            vault.nxos_username, vault.nxos_password, ip, log
        )
    elif cfg.afi == "ipv6":
        return NxapiBgpUnicastSummaryIpv6(
            vault.nxos_username, vault.nxos_password, ip, log
        )
    else:
        log.error("exiting. Unknown afi {}".format(cfg.afi))
        exit(1)

def worker(device, vault):
    ip = get_device_mgmt_ip(nb, device)
    instance = get_instance(ip, vault)
    instance.nxapi_init(cfg)
    instance.vrf = cfg.vrf
    instance.refresh()
    return collect_prefix_rx(ip, instance)

def get_fmt():
    fmt_ipv6 = "{:<15} {:<18} {:<40} {:>9}"
    fmt_ipv4 = "{:<15} {:<18} {:<15} {:>9}"
    if cfg.afi == "ipv4":
        return fmt_ipv4
    else:
        return fmt_ipv6

cfg = get_parser()
log = get_logger(script_name, cfg.loglevel, "DEBUG")
vault = get_vault(cfg.vault)
vault.fetch_data()
nb = netbox(vault)

devices = get_device_list()

fmt = get_fmt()
print_header()

executor = ThreadPoolExecutor(max_workers=len(devices))
futures = list()
for device in devices:
    args = [device, vault]
    futures.append(executor.submit(worker, *args))
print_output(futures)


import datetime
import table
import restaurant

class Restaurant:
  def __init__(self):
    self.tables = []
    self.name = "Restaurant Dingo"
    for i in range(8):
      self.tables.append(table.Table(i))

  def get_tables(self):
    return self.tables

  def print_tables(self):
    for i in range(8):
      print("Table " + str(i))

def loop_opening_hours(action):
  dt = datetime.datetime.now()
  newdate = dt.replace(hour=12, minute=0)
  for i in range(12, 20):
    newdate = dt.replace(hour=i, minute=0)
    action(newdate)


import json
from pathlib import Path
import numpy as np
import torch
import torch.utils.data

from PIL import Image
from panopticapi.utils import rgb2id

from utils.utils import masks_to_boxes
from dataset.utils import make_coco_transforms

city2int = {
    "aachen": 0,
    "bremen": 1,
    "darmstadt": 2,
    "erfurt": 3,
    "hanover": 4,
    "krefeld": 5,
    "strasbourg": 6,
    "tubingen": 7,
    "weimar": 8,
    "bochum": 9,
    "cologne": 10,
    "dusseldorf": 11,
    "hamburg": 12,
    "jena": 13,
    "monchengladbach": 14,
    "stuttgart": 15,
    "ulm": 16,
    "zurich": 17,
    "frankfurt": 18,
    "lindau": 19,
    "munster": 20,
    "berlin": 21,
    "bielefeld": 22,
    "bonn": 23,
    "leverkusen": 24,
    "mainz": 25,
    "munich": 26,
}

int2city = {v: k for k, v in city2int.items()}

def imgid2int(id):
    city, f, s = id.split("_")
    return int(int(s) + int(f) * 1e6 + city2int[city] * 1e12)

def int2imgid(num):
    cityn = num // int(1e12)
    f = (num - int(cityn * 1e12)) // int(1e6)
    s = num % int(1e6)
    return int2city[cityn] + "_" + str(f).zfill(6) + "_" + str(s).zfill(6)

class CityscapesPanoptic:
    def __init__(
        self, img_folder, ann_folder, ann_file, transforms=None, return_masks=True
    ):
        with open(ann_file, "r") as f:
            self.coco = json.load(f)

        # sort 'images' field so that they are aligned with 'annotations'
        # i.e., in alphabetical order
        self.coco["images"] = sorted(self.coco["images"], key=lambda x: x["id"])

        self.img_folder = img_folder
        self.ann_folder = ann_folder
        self.ann_file = ann_file
        self.transforms = transforms
        self.return_masks = return_masks

    def __getitem__(self, idx):
        ann_info = (
            self.coco["annotations"][idx]
            if "annotations" in self.coco
            else self.coco["images"][idx]
        )
        city = ann_info["image_id"].split("_")[0]
        img_path = (
            Path(self.img_folder) / city / (ann_info["image_id"] + "_leftImg8bit.png")
        )
        ann_path = Path(self.ann_folder) / ann_info["file_name"]

        img = Image.open(img_path).convert("RGB")
        w, h = img.size
        if "segments_info" in ann_info:
            masks = np.asarray(Image.open(ann_path), dtype=np.uint32)
            masks = rgb2id(masks)

            ids = np.array([ann["id"] for ann in ann_info["segments_info"]])
            masks = masks == ids[:, None, None]

            masks = torch.as_tensor(masks, dtype=torch.uint8)
            labels = torch.tensor(
                [ann["category_id"] for ann in ann_info["segments_info"]],
                dtype=torch.int64,
            )

        target = {}
        target["image_id"] = torch.tensor(
            [
                imgid2int(
                    ann_info["image_id"] if "image_id" in ann_info else ann_info["id"]
                )
            ]
        )
        if self.return_masks:
            target["masks"] = masks
        target["labels"] = labels

        target["boxes"] = masks_to_boxes(masks)

        target["size"] = torch.as_tensor([int(h), int(w)])
        target["orig_size"] = torch.as_tensor([int(h), int(w)])
        if "segments_info" in ann_info:
            for name in ["iscrowd", "area"]:
                target[name] = torch.tensor(
                    [ann[name] for ann in ann_info["segments_info"]]
                )

        if self.transforms is not None:
            img, target = self.transforms(img, target)

        return img, target

    def __len__(self):
        return len(self.coco["images"])

    def get_height_and_width(self, idx):
        img_info = self.coco["images"][idx]
        height = img_info["height"]
        width = img_info["width"]
        return height, width

def build_cityscapes_panoptic(image_set, args):
    img_folder_root = Path(args.coco_path)
    ann_folder_root = Path(args.coco_panoptic_path)
    assert img_folder_root.exists(), f"provided path {img_folder_root} does not exist"
    assert ann_folder_root.exists(), f"provided path {ann_folder_root} does not exist"

    ann_file = {
        "train": "/content/drive/MyDrive/cityscapes/gtFine/cityscapes_panoptic_train.json",
        "val": "/content/drive/MyDrive/cityscapes/gtFine/cityscapes_panoptic_val.json",
    }

    img_folder_path = {
        "train": "/content/drive/MyDrive/cityscapes/leftImg8bit/train",
        "val": "/content/drive/MyDrive/cityscapes/leftImg8bit/val",
    }

    ann_folder = {
        "train": "/content/drive/MyDrive/cityscapes/gtFine/cityscapes_panoptic_train",
        "val": "/content/drive/MyDrive/cityscapes/gtFine/cityscapes_panoptic_val",
    }

    dataset = CityscapesPanoptic(
        img_folder_path[image_set],
        ann_folder[image_set],
        ann_file[image_set],
        transforms=make_coco_transforms(image_set),
        return_masks=args.masks,
    )

    return dataset

def build_dataset(image_set, args):
    if args.dataset_file == "coco_panoptic":
        # to avoid making panopticapi required for coco
        return build_cityscapes_panoptic(image_set, args)
    raise ValueError(f"dataset {args.dataset_file} not supported")