# -*- coding: utf-8 -*-

import datetime
import json
import logging

from django.conf import settings
from django.db import models
from django.db.transaction import atomic
from django.utils.functional import cached_property
from django.utils.translation import ugettext as _
from kubernetes import client as k8s_client

from bkmonitor.utils.consul import BKConsul
from metadata import config
from metadata.utils import consul_tools

from .cluster import BCSClusterInfo
from .replace import ReplaceConfig
from .utils import ensure_data_id_resource, is_equal_config

logger = logging.getLogger("metadata")

class BCSResource(models.Model):
    """kubernetes资源描述"""

    # 资源信息配置
    # 资源组
    GROUP: str = "monitoring.coreos.com"
    # 版本信息
    VERSION: str = "v1"
    # 资源复数名
    PLURAL: str = None
    PLURALS: str = None
    # 使用用途
    USAGE: str = None

    # 资源集群信息
    # 集群ID，此处和cluster.ClusterInfo.
    cluster_id = models.CharField("集群ID", max_length=128)
    namespace = models.CharField("归属命名空间名", max_length=512)
    name = models.CharField("资源名", max_length=128)

    # 数据元信息
    # 此处的元信息记录的是data_id，因为实际的数据上报后可以继续分为多个结果表(result_table)，
    # 而此处记录为data_id话就可以预留后续拆分结果表的能力
    bk_data_id = models.BigIntegerField("数据源ID", db_index=True)
    is_custom_resource = models.BooleanField("是否自定义资源", default=True)
    # 用于渲染配置时使用
    is_common_data_id = models.BooleanField("是否使用公共data_id", default=True)

    record_create_time = models.DateTimeField("资源记录创建时间")
    resource_create_time = models.DateTimeField("资源在集群中创建时间")

    class Meta:
        abstract = True
        # 对于同一个资源，只可以分配一次资源
        unique_together = ("cluster_id", "namespace", "name")

    @staticmethod
    def make_bcs_client(cluster_id: int) -> k8s_client.ApiClient:
        """
        返回一个bcs操作句柄
        :param cluster_id: 集群ID
        :return:
        """
        try:
            cluster_info = BCSClusterInfo.objects.get(cluster_id=cluster_id)
        except BCSClusterInfo.DoesNotExist as err:
            logger.exception("failed to get bcs client for cluster->[%s] as cluster info is not exists.", cluster_id)
            raise ValueError(_("集群[%s]不存在") % cluster_id) from err

        return cluster_info.api_client

    def should_refresh_own_dataid(self):
        """
        判断该resource是否应该向k8s刷新只属于自己的dataid resource
        """
        # 该resource使用的非公共dataid，则需要刷新
        if not self.is_common_data_id:
            return True
        # 该resource存在配置，则需要刷新
        if ReplaceConfig.objects.filter(
            is_common=False,
            custom_level=ReplaceConfig.CUSTOM_LEVELS_RESOURCE,
            cluster_id=self.cluster_id,
            resource_name=self.name,
            resource_namespace=self.namespace,
            resource_type=self.PLURAL,
        ).exists():
            return True
        return False

    @classmethod
    def refresh_custom_resource(cls, cluster_id):
        """
        刷新自定义资源信息，追加部署的资源，更新未同步的资源
        :param cluster_id: 集群ID
        :return: True | False
        """
        api_client = cls.make_bcs_client(cluster_id=cluster_id)
        custom_client = k8s_client.CustomObjectsApi(api_client)

        # 1. 获取所有命名空间下的本资源信息
        resource_list = custom_client.list_cluster_custom_object(
            group=config.BCS_RESOURCE_GROUP_NAME,
            version=config.BCS_RESOURCE_VERSION,
            plural=config.BCS_RESOURCE_DATA_ID_RESOURCE_PLURAL,
        )
        logger.info("cluster->[%s] got resource->[%s] total->[%s]", cluster_id, cls.PLURAL, len(resource_list["items"]))

        resource_items = {}
        # 遍历所有的资源信息生成字典
        for resource in resource_list["items"]:
            name = resource["metadata"]["name"]
            resource_items[name] = resource

        # 2.遍历所有的resource,对符合条件的数据进行更新操作,只操作已经更新为非公共dataid的resource
        for item in cls.objects.filter(cluster_id=cluster_id):
            # 判断是否需要刷新独立的dataid resource
            if not item.should_refresh_own_dataid():
                continue
            # 检查k8s集群里是否已经存在对应resource
            if item.config_name not in resource_items.keys():
                # 如果k8s_resource不存在，则增加
                ensure_data_id_resource(api_client=api_client, resource_name=item.config_name, config_data=item.config)
                logger.info("cluster->[%s] add new resource->[%s]", cluster_id, item.config)
            else:
                # 否则检查信息是否一致，不一致则更新
                if not is_equal_config(item.config, resource_items[item.config_name]):
                    ensure_data_id_resource(
                        api_client=api_client, resource_name=item.config_name, config_data=item.config
                    )
                    logger.info("cluster->[%s] update resource->[%s]", cluster_id, item.config)

    @classmethod
    @atomic(config.DATABASE_CONNECTION_NAME)
    def refresh_resource(cls, cluster_id: int, common_data_id: int) -> bool:
        """
        刷新集群资源信息，追加未发现的资源,删除已不存在的资源
        :param cluster_id: 集群ID
        :param common_data_id: 公共data_id，所有的资源都会注册到该data_id下
        :return: True | False
        """
        api_client = cls.make_bcs_client(cluster_id=cluster_id)
        custom_client = k8s_client.CustomObjectsApi(api_client)

        # 1. 获取所有命名空间下的本资源信息
        resource_list = custom_client.list_cluster_custom_object(
            group=cls.GROUP, version=cls.VERSION, plural=cls.PLURALS
        )
        logger.info("cluster->[%s] got resource->[%s] total->[%s]", cluster_id, cls.PLURAL, len(resource_list["items"]))

        resource_name_list = []
        # 遍历所有的资源信息
        bulk_create_data = []
        for resource in resource_list["items"]:
            namespace = resource["metadata"]["namespace"]
            name = resource["metadata"]["name"]
            resource_name_list.append("{}_{}".format(namespace, name))

            # 2. 判断是否已经注册
            if cls.objects.filter(
                cluster_id=cluster_id,
                namespace=namespace,
                name=name,
            ).exists():
                # 如果已经存在，继续下一个
                logger.info(
                    "cluster->[%s] resource->[%s] under namespace->[%s] is already exists, nothing will do.",
                    cluster_id,
                    cls.PLURAL,
                    resource["metadata"]["namespace"],
                )
                continue

            # 组装资源记录关系信息，用于批量创建
            bulk_create_data.append(
                cls(
                    cluster_id=cluster_id,
                    namespace=namespace,
                    name=name,
                    bk_data_id=common_data_id,
                    # 由于是公共data_id批量创建，都是公共的data_id
                    is_common_data_id=True,
                    # 由于是拉取集群获得的信息，所以都是自定义资源
                    is_custom_resource=True,
                    record_create_time=datetime.datetime.now(),
                    resource_create_time=datetime.datetime.now(),
                )
            )
            logger.info(
                "cluster->[%s] now create resource->[%s] name->[%s] under namespace->[%s] with data_id->[%s]"
                " success.",
                cluster_id,
                cls.PLURAL,
                name,
                namespace,
                common_data_id,
            )

        # 批量创建
        cls.objects.bulk_create(bulk_create_data)

        # 删除已经不存在的resource映射
        resource_name_set = set(resource_name_list)
        for item in cls.objects.filter(cluster_id=cluster_id):
            key = f"{item.namespace}_{item.name}"
            if key not in resource_name_set:
                item.delete()
                logger.info(
                    "cluster->[%s] delete monitor info->[%s] name->[%s] namespace->[%s] update success.",
                    cluster_id,
                    cls.PLURAL,
                    item.name,
                    item.namespace,
                )

        logger.info("cluster->[%s] all resource->[%s] update success.", cluster_id, cls.PLURAL)
        return True

    @cached_property
    def bcs_client(self) -> k8s_client.ApiClient:
        return self.make_bcs_client(self.cluster_id)

    @cached_property
    def bk_env_label(self) -> str:
        try:
            cluster_info = BCSClusterInfo.objects.get(cluster_id=self.cluster_id)
        except BCSClusterInfo.DoesNotExist as err:
            logger.exception("cluster->[%s] is not exists, err: %s", self.cluster_id, err)
            raise ValueError(_("集群[{}]不存在").format(self.cluster_id))
        return cluster_info.bk_env or settings.BCS_CLUSTER_BK_ENV_LABEL

    @cached_property
    def config_name(self) -> str:

        prefix = "common" if self.is_common_data_id else "custom"
        end = "custom" if self.is_custom_resource else "system"

        if self.bk_env_label:
            return f"{self.bk_env_label}-{prefix}-{self.USAGE}-{self.name}-{end}"
        return f"{prefix}-{self.USAGE}-{self.name}-{end}"

    @cached_property
    def config(self) -> dict:
        # 获取全局replace配置
        replace_config = ReplaceConfig.get_common_replace_config()
        # 获取集群层级replace配置
        cluster_replace_config = ReplaceConfig.get_cluster_replace_config(cluster_id=self.cluster_id)
        # 获取resource层级replace配置
        custom_replace_config = ReplaceConfig.get_resource_replace_config(
            cluster_id=self.cluster_id,
            resource_name=self.name,
            resource_namespace=self.namespace,
            resource_type=self.PLURAL,
        )

        # 将replace配置逐层覆盖
        replace_config[ReplaceConfig.REPLACE_TYPES_METRIC].update(
            cluster_replace_config[ReplaceConfig.REPLACE_TYPES_METRIC]
        )
        replace_config[ReplaceConfig.REPLACE_TYPES_DIMENSION].update(
            cluster_replace_config[ReplaceConfig.REPLACE_TYPES_DIMENSION]
        )
        replace_config[ReplaceConfig.REPLACE_TYPES_METRIC].update(
            custom_replace_config[ReplaceConfig.REPLACE_TYPES_METRIC]
        )
        replace_config[ReplaceConfig.REPLACE_TYPES_DIMENSION].update(
            custom_replace_config[ReplaceConfig.REPLACE_TYPES_DIMENSION]
        )

        cluster = BCSClusterInfo.objects.get(cluster_id=self.cluster_id)
        labels = {"usage": self.USAGE, "isCommon": "false", "isSystem": "false"}

        result = {
            "apiVersion": f"{config.BCS_RESOURCE_GROUP_NAME}/{config.BCS_RESOURCE_VERSION}",
            "kind": f"{config.BCS_RESOURCE_DATA_ID_RESOURCE_KIND}",
            "metadata": {
                "name": cluster.compose_dataid_resource_name(self.config_name),
                "labels": cluster.compose_dataid_resource_label(labels),
            },
            "spec": {
                "dataID": self.bk_data_id,
                "labels": {"bcs_cluster_id": self.cluster_id, "bk_biz_id": str(cluster.bk_biz_id)},
                "metricReplace": replace_config[ReplaceConfig.REPLACE_TYPES_METRIC],
                "dimensionReplace": replace_config[ReplaceConfig.REPLACE_TYPES_DIMENSION],
            },
        }

        # 追加一个指定的资源对应描述
        result["spec"]["monitorResource"] = {"namespace": self.namespace, "kind": self.PLURAL, "name": self.name}

        return result

    def change_data_id(self, data_id: int):
        """
        将resource使用的data_id改为新的data_id
        :param data_id: 新的data_id
        :param is_common_data_id: 更改的新data_id是否公共data_id，默认修改后，就不是公共data_id
        :return:
        """

        # 0.检查cluster的公共dataid信息，判断是否与传入的dataid发生重复了,阻挡重复的情况
        cluster = BCSClusterInfo.objects.get(cluster_id=self.cluster_id)
        for usage, register_info in cluster.DATASOURCE_REGISTER_INFO.items():
            common_data_id = getattr(cluster, register_info["datasource_name"])
            if common_data_id == data_id:
                logger.error(
                    "input change dataid->[%s] conflict with common dataid->[%s] in cluster->[%s]",
                    data_id,
                    usage,
                    self.cluster_id,
                )
                raise ValueError(_("不允许传入公共dataid"))

        # 1. 修改当前资源的data_id
        self.bk_data_id = data_id

        # dataid发生修改，说明一定是改为非公共dataid，则对应调整标志位
        self.is_common_data_id = False

        self.save()
        logger.info(
            "cluster->[%s] namespace->[%s] resource->[%s] name->[%s] change to data_id->[%s] is_common->[%s]",
            self.cluster_id,
            self.namespace,
            self.PLURAL,
            self.name,
            self.bk_data_id,
            self.is_common_data_id,
        )

        # 2. 更新data_id资源
        ensure_data_id_resource(api_client=self.bcs_client, resource_name=self.config_name, config_data=self.config)

        return

    def make_data_id_resource_config(self):
        """
        渲染生成data_id配置文件
        :return:
        """

    def to_json(self):
        return {
            "cluster_id": self.cluster_id,
            "namespace": self.namespace,
            "name": self.name,
            "bk_data_id": self.bk_data_id,
            "is_custom_resource": self.is_custom_resource,
            "is_common_data_id": self.is_common_data_id,
            "resource_type": self.PLURAL,
            "resource_usage": self.USAGE,
        }

    def refresh_info_to_consul(self):
        """
        刷新当前资源dataid到consul
        仅添加当前dataid到对应路径，不会对其他dataid做修改，或者删除
        :return:
        """
        try:
            if not BCSClusterInfo.objects.filter(cluster_id=self.cluster_id).exists():
                logger.error("filter data_id:{}, cluster_id:{} failed".format(self.bk_data_id, self.cluster_id))
                return
            cluster_infos = BCSClusterInfo.objects.filter(cluster_id=self.cluster_id)
            path = "{}/project_id/{}/cluster_id/{}".format(
                config.CONSUL_PATH, cluster_infos[0].project_id, self.cluster_id
            )
            hash_consul = consul_tools.HashConsul()
            # 试图获取当前project_id, cluster_id下的data_id_list
            _, val = hash_consul.get(path)
            vals = []
            if val is not None:
                vals = json.loads(val.get("Value", "[]"))
            # 若当前data_id_list中没有这个data_id，则添加，否则直接返回
            append_data_id_list = [
                data_id for data_id in [cluster_infos[0].K8sMetricDataID, self.bk_data_id] if data_id not in vals
            ]
            if len(append_data_id_list) == 0:
                return
            vals.extend(append_data_id_list)
            hash_consul.put(path, vals)

        except Exception as e:
            logger.error("loads key:{}, value:{} failed:{}".format(path, val, e))

    @classmethod
    def refresh_all_to_consul(cls):
        """
        刷新资源与k8s集群对应的所有时序data_id到consul
        :return:
        """
        # 所有的时序dataid = k8scluster的默认K8sMetricDataID + resource(PodMonitorInfo,ServiceMonitorInfo) 的data_id_list
        path_template = config.CONSUL_PATH + "/project_id/{}/cluster_id/{}"
        info_dict = {}
        cluster_dict = {}
        # 获取所有的k8s_cluster, 构造一个字典cluster_dict {"cluster_id": "project_id"}
        # 同时初始化 构造出一个字典info_dict {"config.CONSUL_PATH/project_id/{}/cluster_id/{}": "[data_id]"}
        bcs_clusters = BCSClusterInfo.objects.all()
        for cluster in bcs_clusters:
            info_key = path_template.format(cluster.project_id, cluster.cluster_id)
            info_dict[info_key] = {cluster.K8sMetricDataID}
            cluster_dict[cluster.cluster_id] = cluster.project_id
        # 获取所有resource
        pod_monitor_infos = PodMonitorInfo.objects.all()
        service_monitor_infos = ServiceMonitorInfo.objects.all()
        try:
            # 遍历并构造出一个字典{"{}/project_id/{}/cluster_id/{}": "[data_id]"}
            for resources in [pod_monitor_infos, service_monitor_infos]:
                for resource in resources:
                    dict_key = path_template.format(cluster_dict[resource.cluster_id], resource.cluster_id)
                    info_dict.get(dict_key, set()).add(resource.bk_data_id)

            # 构造完成之后覆盖consul中的内容
            hash_consul = consul_tools.HashConsul()
            for path, vals in info_dict.items():
                hash_consul.put(path, vals)
        except Exception as e:
            logger.error("refresh all info into consul failed:{}".format(e))

    @classmethod
    def clean_all_bcs_info(cls):
        """
        清除consul上所有资源对应的所有data_id信息
        :return:
        """
        # 直接删除所有的 {}/project_id 下的所有kv
        hash_consul = consul_tools.HasrhConsul()
        root_path = config.CONSUL_PATH + "/project_id"
        bk_consul = BKConsul(
            host=hash_consul.host, port=hash_consul.port, scheme=hash_consul.scheme, verify=hash_consul.verify
        )
        bk_consul.kv.delete(root_path, recurse=True)

class PodMonitorInfo(BCSResource):
    """kubernetes中的pod Monitor信息"""

    PLURAL = "PodMonitor"
    PLURALS = "podmonitors"
    USAGE = "metric"

class ServiceMonitorInfo(BCSResource):
    """kubernetes中的service Monitor信息"""

    PLURAL = "ServiceMonitor"
    PLURALS = "servicemonitors"
    USAGE = "metric"

class LogCollectorInfo(BCSResource):
    """kubernetes中的log Collector信息"""

    GROUP = "bk.tencent.com"
    VERSION = "v1alpha1"
    PLURAL = "BkLogConfig"
    PLURALS = "bklogconfigs"
    USAGE = "log"

    @classmethod
    @atomic(config.DATABASE_CONNECTION_NAME)
    def refresh_resource(cls, cluster_id: int, common_data_id: int) -> bool:
        """
        刷新集群资源信息，追加未发现的资源,删除已不存在的资源
        :param cluster_id: 集群ID
        :param common_data_id: 公共data_id，兼容参数，日志采集场景下不允许公共的dataid
        :return: True | False
        """
        api_client = cls.make_bcs_client(cluster_id=cluster_id)
        custom_client = k8s_client.CustomObjectsApi(api_client)

        # 1. 获取所有命名空间下的本资源信息
        resource_list = custom_client.list_cluster_custom_object(
            group=cls.GROUP, version=cls.VERSION, plural=cls.PLURALS
        )
        logger.info("cluster->[%s] got resource->[%s] total->[%s]", cluster_id, cls.PLURAL, len(resource_list["items"]))

        resource_name = []
        # 遍历所有的资源信息
        for resource in resource_list["items"]:
            namespace = resource["metadata"]["namespace"]
            name = resource["metadata"]["name"]
            resource_name.append("{}_{}".format(namespace, name))
            data_id = resource["spec"]["data_id"]

            # 2. 判断是否已经注册
            if cls.objects.filter(
                cluster_id=cluster_id,
                namespace=namespace,
                name=name,
            ).exists():
                # 如果已经存在，继续下一个
                logger.info(
                    "cluster->[%s] resource->[%s] under namespace->[%s] is already exists, nothing will do.",
                    cluster_id,
                    cls.PLURAL,
                    resource["metadata"]["namespace"],
                )
                continue

            # 3. 创建新的资源记录关系信息
            cls.objects.create(
                cluster_id=cluster_id,
                namespace=namespace,
                name=name,
                bk_data_id=data_id,
                # 由于是公共data_id批量创建，都是公共的data_id
                is_common_data_id=False,
                # 由于是拉取集群获得的信息，所以都是自定义资源
                is_custom_resource=True,
                record_create_time=datetime.datetime.now(),
                resource_create_time=datetime.datetime.now(),
            )
            logger.info(
                "cluster->[%s] now create resource->[%s] name->[%s] under namespace->[%s] with data_id->[%s]"
                " success.",
                cluster_id,
                cls.PLURAL,
                name,
                namespace,
                data_id,
            )

        # 删除已经不存在的resource映射
        for item in cls.objects.filter(cluster_id=cluster_id):
            key = "{}_{}".format(item.namespace, item.name)
            if key not in resource_name:
                item.delete()
                logger.info(
                    "cluster->[%s] delete log collector info->[%s] name->[%s] namespace->[%s] update success.",
                    cluster_id,
                    cls.PLURAL,
                    item.name,
                    item.namespace,
                )

        logger.info("cluster->[%s] log resource->[%s] update success.", cluster_id, cls.PLURAL)
        return True


import types
from logging import debug

from fred_webadmin.webwidgets.gpyweb.gpyweb import tagid, attr, table, tbody, tr, td, label
from fred_webadmin.webwidgets.details.abstractdetaillayout import \
    AbstractDetailLayout
from fred_webadmin.webwidgets.details.sectionlayouts import SectionLayout

class DetailLayout(AbstractDetailLayout):
    pass

class TableDetailLayout(DetailLayout):
    columns_count = 2
    tattr_list = table.tattr_list

    def __init__(self, *content, **kwd):
        super(TableDetailLayout, self).__init__(*content, **kwd)
        self.tag = u'table'
        self.cssc = 'detail_table'
        if self.detail.is_nested:
            self.cssc = 'nested_' + self.cssc

        self.create_layout()

    def create_layout(self):
        detail = self.detail
        self.add(tbody(tagid('tbody')))

        for field in detail.fields.values():
            label_str = self.get_label_name(field)
            self.tbody.add(tr(td(attr(cssc='left_label'), label(label_str)),
                              td(field)
                             )
                          )

class SectionDetailLayout(TableDetailLayout):
    def create_section(self, section_spec):
        section_layout = SectionLayout
        if len(section_spec) > 2:
            section_layout = section_spec[2]
            # List of layouts, first for detail without history,
            # second for detail with history.
            if isinstance(section_layout, types.ListType):
                if not self.detail.history:
                    section_layout = section_layout[0]
                else:
                    section_layout = section_layout[1]
        return section_layout(self.detail, section_spec)

    def create_layout(self):
        self.add(tbody(tagid('tbody')))

        for section_spec in self.detail.sections:
            self.tbody.add(tr(td(self.create_section(section_spec))))

class DirectSectionDetailLayout(SectionDetailLayout):
    ''' Layout that renders section_layout detail directly (without any wrapper tag)'''
    def __init__(self, *content, **kwd):
        super(DirectSectionDetailLayout, self).__init__(*content, **kwd)
        self.tag = ''

    def create_layout(self):
        for section_spec in self.detail.sections:
            self.add(self.create_section(section_spec))

class TableColumnsDetailLayout(DetailLayout):
    def __init__(self, *content, **kwd):
        super(TableColumnsDetailLayout, self).__init__(*content, **kwd)
        self.tag = u''
        self.create_layout()

    def create_layout(self):
        detail = self.detail

        for field in detail.fields.values():
            self.add(td(field))

class TableRowDetailLayout(TableColumnsDetailLayout):
    tattr_list = tr.tattr_list

    def __init__(self, *content, **kwd):
        super(TableRowDetailLayout, self).__init__(*content, **kwd)
        self.tag = u'tr'

class OnlyFieldsDetailLayout(DetailLayout):
    ''' Detail, that renders only fields, directly without anything.
        Fields are separated by comma.
    '''
    def __init__(self, *content, **kwd):
        super(OnlyFieldsDetailLayout, self).__init__(*content, **kwd)
        self.tag = u''
        self.enclose_content = True
        self.create_layout()

    def create_layout(self):
        for i, field in enumerate(self.detail.fields.values()):
            if i != 0:
                self.add(',')
            self.add(field)


import FileLog
import experts
import basic_groups
import ground_truths
import Util
from ground_truths import DataSet

from constants import _TIMEDELTAS_FILE_DELTA_INDEX
from constants import _TIMEDELTAS_FILE_URL_INDEX
from constants import _TIMEDELTAS_FILE_USER_ID_INDEX

from params import _DELTAS
from params import _SIZE_EXPERTS
from params import _SIZE_TOP_NEWS

_NUM_SEC_PER_HOUR = 3600
_LOG_FILE = 'a_hour_thresholds.log'
_DATA_DIR = '../data/HourThresholds/'
_BREAKDOWN = False

# enum Expert type
class ExpertGroup:
  precision = None
  fscore = None
  ci = None
  union = None

class GroupCount:
  population = 0
  common = 0
  union = 0
  other = 0

  non_experts = 0

  precision = 0
  fscore = 0
  ci = 0

  def add(self, gc):
    self.population += gc.population
    self.common += gc.common
    self.union += gc.union
    self.other += gc.other
    self.non_experts += gc.non_experts
    self.precision += gc.precision
    self.fscore += gc.fscore
    self.ci += gc.ci

def run():

  Util.ensure_dir_exist(_DATA_DIR)
  category = None
  seeds = Util.load_seeds() #read twitter data

  gt_rankings = ground_truths.get_gt_rankings(seeds, DataSet.TESTING,
                                              category)
  log('Num ground_truth_rankings: %s' % len(gt_rankings))
  target_news = ground_truths.find_target_news(gt_rankings, _SIZE_TOP_NEWS)
  log('Size target_news: %s' % len(target_news))

  for delta in _DELTAS:
    (num_users, newsaholics,
     active_users, common_users) = basic_groups.group_users(delta, category)
    population = newsaholics.union(active_users).union(common_users)
    log('Num newsaholics: %s' % len(newsaholics))
    log('Num active: %s' % len(active_users))
    log('Num common: %s' % len(common_users))
    log('Num users (population): %s' % len(population))

    # -- Get experts --
    ExpertGroup.precision = experts.select_experts_precision(
        newsaholics.union(active_users), num_users, delta, _SIZE_EXPERTS,
        category)
    ExpertGroup.fscore = experts.select_experts_fscore(len(target_news),
                                                   num_users,
                                                   delta, _SIZE_EXPERTS,
                                                   category)
    ExpertGroup.ci = experts.select_experts_ci(num_users, delta, _SIZE_EXPERTS,
                                           category)
    ExpertGroup.union = experts.select_all_experts(ExpertGroup.precision,
                                             ExpertGroup.fscore,
                                             ExpertGroup.ci)

    log('Num experts (precision): %s' % len(ExpertGroup.precision))
    log('Num experts (fscore): %s' % len(ExpertGroup.fscore))
    log('Num experts (ci): %s' % len(ExpertGroup.ci))
    log('Num all experts: %s' % len(ExpertGroup.union))

    non_experts = population.difference(ExpertGroup.union)
    log('Num non_experts: %s' % len(non_experts))

    # other_users = population.difference(all_experts).difference(common_users)

    # -- counting --

    total_num_tweets = 0
    hour_to_num_tweets = {}
    with open('../data/FolkWisdom/time_deltas.tsv') as in_file:
      for line in in_file:
        tokens = line.split('\t')
        time_delta_in_sec = int(tokens[_TIMEDELTAS_FILE_DELTA_INDEX])
        url = tokens[_TIMEDELTAS_FILE_URL_INDEX].strip()
        user_id = tokens[_TIMEDELTAS_FILE_USER_ID_INDEX]

        if time_delta_in_sec > 0 and url in target_news:
          current_hour = time_delta_in_sec / _NUM_SEC_PER_HOUR
          total_num_tweets += 1

          if current_hour not in hour_to_num_tweets:
            hour_to_num_tweets[current_hour] = GroupCount()
          gcount = hour_to_num_tweets[current_hour]

          gcount.population += 1
          if user_id in ExpertGroup.union:
            gcount.union += 1
            if user_id in ExpertGroup.precision:
              gcount.precision += 1
            if user_id in ExpertGroup.fscore:
              gcount.fscore += 1
            if user_id in ExpertGroup.ci:
              gcount.ci += 1
          else:
            gcount.non_experts += 1
            if user_id in common_users:
              gcount.common += 1

            #  print >> sys.stderr, 'Error, a user in expert union but not belongs to any expert group'

          # elif user_id in common_users:
          #   gcount.common += 1
          # else :
          #   gcount.other += 1

          # if user_id in non_experts:
          #   gcount.non_experts += 1

    gcount = GroupCount()
    with open(_DATA_DIR + 'hour_thresholds_%s.tsv' % delta, 'w') as out_file:
      for hour in hour_to_num_tweets.keys():
        gc = hour_to_num_tweets[hour]
        gcount.add(gc)
        percentage = (gcount.population / float(total_num_tweets)) * 100.0
        percentage_common = (gcount.common / float(total_num_tweets)) * 100.0
        percentage_other = (gcount.other / float(total_num_tweets)) * 100.0
        percentage_experts = (gcount.union / float(total_num_tweets)) * 100.0
        percentage_non_experts = (gcount.non_experts / float(total_num_tweets)) * 100.0

        out_file.write('%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' % (hour, percentage,
                                                             percentage_non_experts,
                                                             percentage_experts,
                                                             percentage_common,
                                                             (gcount.precision / float(total_num_tweets)) * 100.0,
                                                             (gcount.fscore / float(total_num_tweets)) * 100.0,
                                                             (gcount.ci / float(total_num_tweets)) * 100.0))
    log('hour\tpopulation\tnon_experts\texperts\tcommon\tprecision\tfscore\tci')

def log(message):
  """Helper method to modularize the format of log messages.

  Keyword Arguments:
  message -- A string to log.
  """
  FileLog.log(_LOG_FILE, message)

if __name__ == "__main__":
  run()


import re

class Indicator(object):

    """docstring for Indicator"""
    indicator_re = '!(?P<eighteens>[1-9A-H])(?P<ones>[1-9A-H])'
    indicator_rhi_re = '!(?P<one>[1-9A-H]{2})(?P<two>[1-9A-H]{2}(?P<three>[1-9A-H]{2}))'

    def __init__(self, value=None):
        super(Indicator, self).__init__()

        self.encoded = Value

    @staticmethod
    def is_indicator(value):
        if re.match(Indicator.indicator_re, value):
            return True
        return False

    def decode_digit(self, value):
        if value >= 1 and value <= 9:
            return int(value)
        else:
            return ord(value) - ord('A') + 10

    def decode_single_indicator(self, value):
        val1 = self.decode_digit(value[0])
        val2 = self.decode_digit(value[1])

        return 18 * val1 + val2

    def decode(self, value=None):
        if not value:
            value = self.encoded

        if value.startswith('!'):
            value = value[1:]
        val_1 = decode_single_indicator(self, value)
        if len(value) == 2:
            return val_1
        elif len(value) == 6:
            val_2 = decode_single_indicator(self, value[2:])
            val_3 = decode_single_indicator(self, value[4:])
            return val_1, val_2, val_3
        else:
            raise Exception("Invalid indicator: %s" % value)

    def encode(self, value):
        pass


import scrapy

from ..Utils import Utils
class DeputadosSpider(scrapy.Spider):
    name = "deputados"

    def start_requests(self):
        deps_file = open("../lista_deputados.txt", "r")

        deps_urls = deps_file.read().splitlines()

        for url in deps_urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        main_response= Utils(response)

        main_data = main_response.main(gender="M")

        yield main_data


from collections import defaultdict, deque, Counter
from numbers import Number
from typing import Dict

class AverageTracker(defaultdict, Dict[str, list]):

    def __init__(self) -> None:
        defaultdict.__init__(self, self._ADeque)

        self.totals = Counter()

        # avg: avg_bandwidth | total: total_bandwidth
        # basically keep the avg display name and total display name separate
        self.total_names: Dict[str, str] = {}

        # maybe regular name is "bandwidth" and the display is "bandwidth(mb)"
        self.display_names: Dict[str, str] = {}

    def add(self, name: str, value: Number):
        self[name].append(value)
        if name in self.total_names:
            self.totals[name] += value

    def track(self, name: str, display_name: str, total_name: str = None):
        if total_name:
            self.total_names[name] = total_name
        if display_name:
            self.display_names[name] = display_name

    def get_average(self, name: str) -> str:
        return '%.2f' % (sum(self.get(name)) / len(self[name]))

    def get_stats(self):
        d = {}
        for key in self:
            if len(self[key]) > 1:
                d[self.display_names.get(key) or key] = self.get_average(key)
            if key in self.total_names:
                d[self.total_names[key]] = self.get_total(key)

        return d

    def get_total(self, key) -> str:
        value = self.totals[key]

        if not isinstance(value, int): return '%.2f' % value
        else: return str(int)

    class _ADeque(deque):

        def __init__(self) -> None:
            super().__init__([], 1000)

if __name__ == '__main__':
    averages = AverageTracker()
    averages['bandwidth'].append(500)
    averages['bandwidth'].append(250)
    averages['bandwidth'].append(50)

    print(averages.get_average('bandwidth'))


import requests, json
from bs4 import BeautifulSoup

blog_soup = ""
blog_data = ""

def extract_data(username):
	blog_URL = "https://codeforces.com/blog/" + username
	blog_page = requests.get(blog_URL)

	global blog_soup, blog_data
	blog_soup = BeautifulSoup(blog_page.content, 'html.parser')
	blog_data = {}

	for a in blog_soup.find_all('a', href=True):
	    if a.text and (a['href'])[:12] == "/blog/entry/" and (a['href'])[::-1][:9] != 'stnemmoc#':
	    	blog_data[a.text] = a['href']
	return save_data(blog_data)

def save_data(blog_data):
	with open('blog_data.txt', 'w') as outfile:
		json.dump(blog_data, outfile)

def output_HTML():
	global blog_soup
	print(blog_soup.prettify)

def output_data():
	global blog_data
	print(blog_data)


#!/usr/bin/env python
# -*- coding: UTF-8 -*-
from libshorttext.classifier import *
'''
https://www.csie.ntu.edu.tw/~cjlin/libshorttext/doc/libshorttext.html#installation-and-data-format

https://www.csie.ntu.edu.tw/~cjlin/libshorttext/doc/_sources/classifier.txt
LibShortText是一个开源的Python短文本（包括标题、短信、问题、句子等）分类工具包。它在LibLinear的基础上针对短文本进一步优化，
主要特性有：
支持多分类
直接输入文本，无需做特征向量化的预处理
二元分词（Bigram），不去停顿词，不做词性过滤
基于线性核SVM分类器（参见SVM原理简介：最大间隔分类器），训练和测试的效率极高
提供了完整的API，用于特征分析和Bad Case检验
'''
# training
model_path="m1.model"
train_file_path = "demo/train_file"
model,svm = train_text(train_file_path)
model.save(model_path, True)

# testing
test_file_path = "demo/test_file"
m = TextModel()
m.load(model_path)
predict_result = predict_text(test_file_path,m)
print("Accuracy = {0:.4f}% ({1}/{2})".format(
		predict_result.get_accuracy()*100,
		sum(ty == py for ty, py in zip(predict_result.true_y, predict_result.predicted_y)),
		len(predict_result.true_y)))

# ***
# optimization finished, #iter = 8
# Objective value = -986.801900
# nSV = 18974
# Accuracy = 87.1800% (4359/5000)


import itertools
import random

import discord

# WORDS = ["scary", "lel", "lol", "aiya", "bruh", "breh", "oop", "<:notlikeduck:764579516755738664>",
#          "<:tourist_mad:764579517515300914>", "<:confus:764579516755738664>", "<:foncus:766364807732527114>"]
WORDS = [
    "scary", "terrible", "rude", "lol", "aiya", "bruh", "oop", "true", "ok", "cool", "hi", "hehe", "gah", "issoke",
    "yes", "epic", "popping", "fair", "omg", ":0", "i-", "indeed", "anyway", "funny", "ugh", "idk",
    "<:notlikeduck:764579516755738664>", "<:confus:764579516755738664>", "<:foncus:766364807732527114>",
    ":clown:", "<:lemondamage:928558941330358272>", "<:lemonthink:928558940210466836>"
]
EMOTES = [":notlikeduck:", ":tourist_mad:", ":confus:", ":foncus:"]
WORD_CHANCE = int(len(WORDS) * 1.25)
COWWIN = ["galen colin", "colin galen", "calen golin", "golin calen", "cowwin", "galen", "colin", "calin golin"]
COWWIN.extend([x.replace(' ', '_') for x in COWWIN if ' ' in x])
COLIN_ORZ = ["try again", "try another time"]

def is_hm(text):
    return set(text) <= set("hm") and len(text) >= 2

async def spam(message):
    msg = message.content.lower()
    if message.author.id == 145266128363585536 and any([msg in orz for orz in COLIN_ORZ]):
        reply = random.choice(COWWIN)
    elif any([emote in msg for emote in itertools.chain(EMOTES, WORDS)]):
        idx = random.randint(0, WORD_CHANCE - 1)
        if idx < len(WORDS):
            reply = WORDS[idx]
        else:
            reply = message.content
    elif is_hm(msg):
        if random.randint(1, 10) == 1:
            reply = "yumm"
        else:
            reply_size = min(500, len(message.content))
            reply = ""
            if random.randint(0, 1) == 0:
                reply += 'm' * random.randint(1, reply_size // 2)
            reply += 'h'
            reply += 'm' * random.randint(1, reply_size)
    elif "sir" in msg:
        reply = f"hello {message.author.mention} sir"
    elif "sad" in msg:
        reply = f"{random.choice(['so', 'very'])} sad!"
    elif 'q' in msg and random.randint(1, 3) == 1:
        reply = "q is the best letter!"
    elif random.randint(1, 5) == 1:
        reply = random.choice(WORDS)
    elif ' ' not in message.content and ':' not in message.content and random.randint(1, 8) == 1:
        reply = message.content[::-1]
    elif ' ' in message.content and random.randint(1, 10) == 1:
        words = message.content.split()
        words.reverse()
        reply = ' '.join(words)
    else:
        return
    await message.channel.send(reply)


__author__ = 'ONS-BIG-DATA'

# Use scikitlearn to look at machine learning techniques
# for identifying park homes from Zoopla data
# Karen Gask
# 31/07/15

# Import packages
import numpy as np
import pandas as pd
from sklearn.cross_validation import train_test_split, cross_val_score, StratifiedKFold
from scipy.stats import sem

# Import all homes data collected from Zoopla API
# This is for PO, SO, NR and TR postcode areas
homes = pd.read_csv('C:/Users/ONS-BIG-DATA/Documents/Zoopla/park_homes_logistic2.csv')

print(homes.head())
print(homes.dtypes)

####### Data wrangling

# Create a function which z-scores for rental and sale prices independently
# Adds a new variable, zprice, from price
# (X(i) - X(mean)) / (X(StDev))
def zscores(df):
    # Sale
    df['saleprice'] = np.where(df['listing_status']=='sale', df['price'], 0)
    df['zsaleprice'] = np.where(df['listing_status']=='sale', (df['saleprice'] - df['saleprice'].mean()) / (df['saleprice'].std()), 0)
    # Rent
    df['rentprice'] = np.where(df['listing_status']=='rent', df['price'], 0)
    df['zrentprice'] = np.where(df['listing_status']=='rent', (df['rentprice'] - df['rentprice'].mean()) / (df['rentprice'].std()), 0)
    # Combine both into one z-scored price variable
    df['zprice'] = df[['zsaleprice','zrentprice']].sum(axis=1)
    df = df.drop(['saleprice','rentprice','zsaleprice','zrentprice'], axis=1)
    return df

zscores(homes)

# Need to see whether certain phrases are in the property description
def add_caravan_booleans(df):
    df['description_lower'] = df['description'].str.lower()
    df['holiday_park'] = np.where(df['description_lower'].str.contains("holiday park"), 1, 0)
    df['holiday_village'] = np.where(df['description_lower'].str.contains("holiday village"), 1, 0)
    df['chalet_text'] = np.where(df['description_lower'].str.contains("chalet"), 1, 0)
    df['park_home'] = np.where(df['description_lower'].str.contains("park home"), 1, 0)
    df['static_home'] = np.where(df['description_lower'].str.contains("static home"), 1, 0)
    df['mobile_home'] = np.where(df['description_lower'].str.contains("mobile home"), 1, 0)
    df['park_lodge'] = np.where(df['description_lower'].str.contains("park lodge"), 1, 0)
    df['static_caravan'] = np.where(df['description_lower'].str.contains("static caravan"), 1, 0)
    return df

add_caravan_booleans(homes)

# Property type has lots of different values (eg. detached house, semi-detached house etc.)
# Explore number of cases per property type
print(homes.groupby(by=['property_type'])['listing_id'].count())
# I can narrow these down in feature selection later

# Need to one hot encode property_type as this needs to be a numeric value for scikit learn
homes_prop_type = pd.get_dummies(homes['property_type'])
homes = pd.concat([homes, homes_prop_type], axis=1)

# Convert listing_status (whether for sale or for rent to a numeric value, 1=sale, 0=rent)
homes['listing_status'] = np.where(homes['listing_status']=='sale',1,0)

# Scikit learn needs a numpy array, not a pandas dataframe
# Keep only relevant variables which may be able to predict whether you have a park home

# x = explantory features
x = homes[['listing_status','num_bedrooms','zprice','holiday_park','holiday_village','chalet_text','park_home','static_home',
           'mobile_home','park_lodge','static_caravan','Barn conversion','Bungalow','Chalet','Cottage',
           'Country house','Detached bungalow','Detached house','End terrace house','Equestrian property','Farm','Farmhouse',
           'Flat','Land','Link-detached house','Lodge','Maisonette','Mews house','Mobile/park home','Parking/garage',
           'Semi-detached bungalow','Semi-detached house','Studio','Terraced bungalow','Terraced house','Town house','Villa']].astype(np.float64)
X = np.array(x)

# y = label or target variable which you are trying to predict. Must be a number
y = homes[['parkhome']]
y = np.array(y)
y = y.ravel()
print(sum(y), "cases labelled as park homes out of", len(y), "(%0.2f per cent)" % (sum(y)/len(y)*100))

# Print information about x and y
print(X[0], y[0])
print(X.shape, y.shape)
print(type(X), type(y))

####### Principal components analysis

from sklearn.decomposition import PCA
pca = PCA(n_components=10)
pca.fit(X)
print(pca.explained_variance_ratio_)

# Plot first two eigenvectors with different colours for y (whether caravan home or not)
pca = PCA(n_components=2).fit(X)
X_reduced = pca.transform(X)
import matplotlib.pyplot as plt
cmap = 'jet'
plt.scatter(X_reduced[:,0], X_reduced[:,1], c=y, cmap=cmap)
plt.show()

####### Machine learning

# Train / Test split here
# Train to decide upon the best type of model eg. logistic regression, SVM etc.
# Cross validation to a final evaluation of the goodness-of-fit of the chosen model
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=10)

# Check out percentage of park homes in training and testing datasets
def positive_class(y):
    print("%0.2f per cent" % (sum(y)/len(y)*100), "of cases are park homes")

positive_class(y_train)
positive_class(y_test)

# Create a K-fold cross validation iterator with F1 score instead of accuracy
# K is set to 10 - change if need be
def fit_evaluate_cv(model, X, y):
    model.fit(X, y)
    cv = StratifiedKFold(y, 10, shuffle=True, random_state=0)
    scores = cross_val_score(model, X, y, cv=cv, scoring='f1')
    print(scores)
    print("Mean F1 score: %0.3f (+/- %0.3f)" % (np.mean(scores), sem(scores)))
    precision = cross_val_score(model, X, y, cv=cv, scoring='precision')
    print("Mean precision: %0.3f" % (np.mean(precision)))
    recall = cross_val_score(model, X, y, cv=cv, scoring='recall')
    print("Mean recall: %0.3f" % (np.mean(recall)))

# Feature selection with 10-fold cross validation
from sklearn.feature_selection import RFECV
def select_features(model, X, y):
    feature_names = list(x.columns.values)
    rfecv = RFECV(model, step=1, cv=10)
    rfecv = rfecv.fit(X, y)
    print("Features sorted by their rank:")
    print(sorted(zip(map(lambda x: round(x, 4), rfecv.ranking_), feature_names)))

# Run model with features selected in select_features function
def fit_evaluate_cv_selected_features(model, X, y):
    rfecv = RFECV(model, step=1, cv=5, scoring='f1')
    rfecv = rfecv.fit(X, y)
    X_transformed = rfecv.transform(X)
    print("Shape of full training set: ",X.shape)
    print("Shape of reduced training set: ", X_transformed.shape)
    fit_evaluate_cv(model, X_transformed, y)

# Logistic regression with all features
# L1 regularisation better than L2 if you have less observations than features
# In my case I have many more observations than features so L2 thought to be better
# L2 also performs slightly better here
from sklearn.linear_model import LogisticRegression
log = LogisticRegression(class_weight='auto', penalty='l2') # class_weight is for unbalanced datasets, L2 regularisation
fit_evaluate_cv(log, X_train, y_train)
# Select features to use
select_features(log, X_train, y_train)
# Logistic regression with reduced number of features
fit_evaluate_cv_selected_features(log, X_train, y_train)
# So reducing number of features from 37 to 3 increases mean F1 score from 0.828 to 0.867

# Decision tree
# Entropy better than Gini here - gini F1=0.494, entropy F1=0.862
from sklearn.tree import DecisionTreeClassifier
dtree = DecisionTreeClassifier(criterion='entropy', max_depth=9, class_weight="auto")
# Use with all features
fit_evaluate_cv(dtree, X_train, y_train)
# Can also do feature_importances_ here

# Random forests
# Entropy better than Gini here - gini F1=0.895, entropy F1=0.900
from sklearn.ensemble import RandomForestClassifier
rf = RandomForestClassifier(criterion='entropy')
fit_evaluate_cv(rf, X_train, y_train)
feature_names = list(x.columns.values)
print("Features sorted by their rank:")
print(sorted(zip(map(lambda x: round(x, 4), rf.feature_importances_), feature_names), reverse=True))

# Support Vector Machines grid search for optimisation
# cv: If cv is an integer input, if y is binary, StratifiedKFold used, so StratifiedKFold is used here
# Also used class_weight='auto' to add weight to unbalanced y_train
from sklearn import svm, grid_search
import time
start_time = time.time()
param_grid = [ {'C': [1, 10, 100, 1000], 'gamma': [0.001, 0.01, 0.1, 1.0], 'kernel': ['linear','rbf']} ]
svmsearch = svm.SVC(class_weight='auto')
svmmodel = grid_search.GridSearchCV(svmsearch, param_grid, cv=3, scoring='f1')
svmmodel.fit(X_train, y_train)
print(svmmodel.best_params_)
print("--- %s seconds ---" % (time.time() - start_time))
# Can't use RFE with an rbf kernel

# Create SVM with the best parameters
svmmodel = svm.SVC(C=100, gamma=0.1, kernel='rbf')
# Use with all features
fit_evaluate_cv(svmmodel, X_train, y_train)

# Finally apply final chosen model (SVM) to remaining 20% testing set
from sklearn.metrics import confusion_matrix, f1_score
svmtesting = svm.SVC(C=100, gamma=0.1, kernel='rbf', class_weight='auto')
svmtesting.fit(X_test, y_test)
y_pred = svmtesting.predict(X_test)

print("F1 score", f1_score(y_test, y_pred))
print("Confusion matrix", confusion_matrix(y_test, y_pred))


from multiprocessing.resource_sharer import stop
import sys
import socket
import time
import threading

class Client:
    def __init__(self,host = "127.0.0.1",port = 3456):
        self.host = host
        self.port = port
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.do_runing = True
        self.BUFFER_SIZE = 2048
        self.count = 0
        self.connect()

    def connect(self):
        try:
            self.sock.connect((self.host,self.port))
        except socket.error as e:
            print(str(e))

    def handle_recieve(self):
        while self.do_runing:
            resp = self.sock.recv(self.BUFFER_SIZE)
            if resp:
                print("{} :Received data : {}\r".format(self.count,resp.decode()))
                self.count+=1

            time.sleep(0.1)

    def handle_send(self):
        while self.do_runing:
            data = input('{}:{}>'.format(self.host,self.port))
            if data:
                print("Send data : {}\r".format(data))
                self.sock.sendall(data.encode())

            time.sleep(0.2)

    def run(self):
        print("Starting : {}:{}".format(self.host,self.port))
        thread = threading.Thread(target=self.handle_recieve)
        thread.setDaemon(True)
        thread.start()

        self.handle_send()

if __name__ == "__main__":
    try:
        client = Client()
        client.run()
    except KeyboardInterrupt:
        print("Stop process..")
        sys.exit()


import os
import unittest
import sys
from selenium import webdriver

username = os.environ.get("LT_USERNAME")
access_key = os.environ.get("LT_ACCESS_KEY")

class FirstSampleTest(unittest.TestCase):

    # setUp runs before each test case
    def setUp(self):
        desired_caps = {
            'LT:Options': {
                "user": username,
                "accessKey": access_key,
                "build": "UnitTest-Selenium-Sample",
                "name": "UnitTest-Selenium-Test",
                "platformName": "Windows 11",
                "selenium_version": "4.0.0"
            },
            "browserName": "Chrome",
            "browserVersion": "latest",
        }

        self.driver = webdriver.Remote(
            command_executor="http://hub.lambdatest.com:80/wd/hub",
            desired_capabilities=desired_caps)

# tearDown runs after each test case

    def tearDown(self):
        self.driver.quit()

    def test_multiple_tabs(self):
        # try:
        driver = self.driver

        driver.get('https://www.lambdatest.com/')
        driver.maximize_window()
        driver.execute_script("window.open('about:blank','secondtab');")
        driver.switch_to.window("secondtab")
        driver.get('https://www.lambdatest.com/')
        driver.switch_to.window(1)

if __name__ == "__main__":
    unittest.main()


from django.contrib.auth import views as auth_views
from django.urls import path

from .views import (
    ListUsersView, register_user, create_user, UserLoginView)

app_name = 'users'
urlpatterns = [
    path('', ListUsersView.as_view(), name='list'),
    path('nuevo/', create_user, name='create'),
    path('registro/<str:dni>/', register_user, name='register'),
    # path('perfil/', profile, name='profile'),
    path('ingresar/', UserLoginView.as_view(), name='login'),
    path('salir/', auth_views.LogoutView.as_view(
        template_name='users/logout.html'),
        name='logout'),
]


pairs = ""
with open("input.txt", "r") as f:
    while True:
        line = f.readline()
        if not line:
            break
        pairs += line

def splitPairs(pairs):
    newPairs = pairs.split("\n")
    draft_pairs = []
    draft_pairsv2 = []
    for i in range(len(newPairs)):
        draft_pairs.append(newPairs[i].split(","))
    for j in range(len(draft_pairs)):
        draft_pairsv2.append(
            [draft_pairs[j][0].split("-"), draft_pairs[j][1].split("-")]
        )
    return draft_pairsv2

ranges_final = []

def findContainers(splitPairs):
    for i in range(len(splitPairs)):
        ranges = []
        ranges2 = []
        boundary1_2 = int(splitPairs[i][1][0])
        boundary2_2 = int(splitPairs[i][1][1])
        boundary2 = int(splitPairs[i][0][1])
        boundary1 = int(splitPairs[i][0][0])
        integer = 0
        integer2 = 0
        while integer < (boundary2 - boundary1 + 1):
            ranges.append(boundary1 + integer)
            integer += 1
        while integer2 < (boundary2_2 - boundary1_2 + 1):
            ranges2.append(boundary1_2 + integer2)
            integer2 += 1
        ranges_final.append([ranges, ranges2])
    return ranges_final

def fullyContain(ranges):
    count = 0
    for i in range(len(ranges)):
        stretch1 = ranges[i][0]
        stretch2 = ranges[i][1]
        if set(stretch1) >= set(stretch2):
            count += 1
        elif set(stretch1) <= set(stretch2):
            count += 1
    return count

def overlap(ranges):
    count = 0
    for i in range(len(ranges)):
        stretch1 = ranges[i][0]
        stretch2 = ranges[i][1]
        if list(set(stretch1) & set(stretch2)):
            count +=1
    return count

print(overlap(findContainers(splitPairs(pairs))))


from datetime import datetime, timedelta
from pathlib import Path
import random
from typing import NamedTuple, Optional, Union

from OpenSSL import crypto

from provisioningserver.path import get_tentative_data_path
from provisioningserver.utils.snap import running_in_snap, SnapPaths

class Certificate(NamedTuple):
    """A self-signed X509 certificate with an associated key."""

    key: crypto.PKey
    cert: crypto.X509

    @classmethod
    def from_pem(cls, material: Union[bytes, str]):
        """Return a Certificate from PEM encoded material.

        The material is expected to contain both the certificate and private
        key.

        """
        key = crypto.load_privatekey(crypto.FILETYPE_PEM, material)
        cert = crypto.load_certificate(crypto.FILETYPE_PEM, material)
        return cls(key, cert)

    def cn(self) -> str:
        """Return the certificate CN."""
        return self.cert.get_subject().CN

    def expiration(self) -> Optional[datetime]:
        """Return the certificate expiration."""
        expiry = self.cert.get_notAfter()
        if expiry is None:
            return None
        return datetime.strptime(expiry.decode("ascii"), "%Y%m%d%H%M%SZ")

    def public_key_pem(self) -> str:
        """Return PEM-encoded public key."""
        return crypto.dump_publickey(crypto.FILETYPE_PEM, self.key).decode(
            "ascii"
        )

    def private_key_pem(self) -> str:
        """Return PEM-encoded private key."""
        return crypto.dump_privatekey(crypto.FILETYPE_PEM, self.key).decode(
            "ascii"
        )

    def certificate_pem(self) -> str:
        """Return PEM-encoded certificate."""
        return crypto.dump_certificate(crypto.FILETYPE_PEM, self.cert).decode(
            "ascii"
        )

    def cert_hash(self) -> str:
        """Return the SHA-256 digest for the certificate."""
        return self.cert.digest("sha256").decode("ascii")

def generate_certificate(
    cn, key_bits=4096, validity=timedelta(days=3650)
) -> Certificate:
    """Generate an X509 certificate with an RSA private key."""
    key = crypto.PKey()
    key.generate_key(crypto.TYPE_RSA, key_bits)

    cert = crypto.X509()
    cert.get_subject().CN = cn
    cert.set_serial_number(random.randint(0, (1 << 128) - 1))
    cert.gmtime_adj_notBefore(0)
    cert.gmtime_adj_notAfter(int(validity.total_seconds()))
    cert.set_pubkey(key)
    cert.sign(key, "sha512")
    return Certificate(key, cert)

def get_maas_cert_tuple():
    """Return a 2-tuple with certificate and private key paths.

    The format is the same used by python-requests."""
    if running_in_snap():
        cert_dir = SnapPaths.from_environ().common / "certificates"
        private_key = cert_dir / "maas.key"
        certificate = cert_dir / "maas.crt"
    else:
        private_key = Path(
            get_tentative_data_path("/etc/maas/certificates/maas.key")
        )
        certificate = Path(
            get_tentative_data_path("/etc/maas/certificates/maas.crt")
        )
    if not private_key.exists() or not certificate.exists():
        return None
    return str(certificate), str(private_key)


r""" Dataloader builder for few-shot semantic segmentation dataset  """
from torchvision import transforms
from torch.utils.data import DataLoader
import albumentations as A
import albumentations.pytorch
from PIL import Image
import numpy as np
import cv2

from data.pascal import DatasetPASCAL
from data.coco import DatasetCOCO
from data.fss import DatasetFSS

class Compose(A.Compose):
    def __init__(self, transforms, bbox_params=None, keypoint_params=None, additional_targets=None, p=1):
        super().__init__(transforms, bbox_params=bbox_params, keypoint_params=keypoint_params, additional_targets=additional_targets, p=p)

    def __call__(self, image, mask):
        augmented = super().__call__(image=np.array(image), mask=np.array(mask))
        return augmented['image'], augmented['mask']

class FSSDataset:

    @classmethod
    def initialize(cls, benchmark, img_size, datapath, use_original_imgsize, apply_cats_augmentation=False, apply_pfenet_augmentation=False):

        cls.datasets = {
            'pascal': DatasetPASCAL,
            'coco': DatasetCOCO,
            'fss': DatasetFSS,
        }

        cls.img_mean = [0.485, 0.456, 0.406]
        cls.img_std = [0.229, 0.224, 0.225]
        cls.datapath = datapath
        cls.use_original_imgsize = use_original_imgsize

        cats_augmentation = [
            A.ToGray(p=0.2),
            A.Posterize(p=0.2),
            A.Equalize(p=0.2),
            A.Sharpen(p=0.2),
            A.RandomBrightnessContrast(p=0.2),
            A.Solarize(p=0.2),
            A.ColorJitter(p=0.2),
        ]

        scale_limit = (0.9, 1.1) if benchmark == 'coco' else (0.8, 1.25)

        pfenet_augmentation = [
            A.RandomScale(scale_limit=scale_limit, p=1.),
            A.Rotate(limit=10, p=1.),
            A.GaussianBlur((5, 5), p=0.5),
            A.HorizontalFlip(p=0.5),
            A.PadIfNeeded(img_size, img_size, border_mode=cv2.BORDER_CONSTANT,
                value=[x * 255 for x in cls.img_mean], mask_value=0),
            A.RandomCrop(img_size, img_size),
        ]

        cls.trn_transform = Compose([
            *(cats_augmentation if apply_cats_augmentation else ()),
            *(pfenet_augmentation if apply_pfenet_augmentation else ()),
            A.Resize(img_size, img_size),
            A.Normalize(cls.img_mean, cls.img_std),
            A.pytorch.transforms.ToTensorV2(),
        ])

        cls.transform = Compose([
            A.Resize(img_size, img_size),
            A.Normalize(cls.img_mean, cls.img_std),
            A.pytorch.transforms.ToTensorV2(),
        ])

    @classmethod
    def build_dataloader(cls, benchmark, bsz, nworker, fold, split, shot=1):
        # Force randomness during training for diverse episode combinations
        # Freeze randomness during testing for reproducibility
        shuffle = split == 'trn'
        nworker = nworker if split == 'trn' else 0
        transform = cls.trn_transform if split == 'trn' else cls.transform

        dataset = cls.datasets[benchmark](cls.datapath, fold=fold, transform=transform, split=split, shot=shot, use_original_imgsize=cls.use_original_imgsize)
        dataloader = DataLoader(dataset, batch_size=bsz, shuffle=shuffle, num_workers=nworker)

        return dataloader


#!/usr/bin/env python
# -*- coding: utf-8 -*-

for i in range(3):
    try:
        # assert语句失败将引发AssertionError
        assert i != 1, 'i不能为1'
        print(i)
    except AssertionError as ae:
        print(ae)
        # exit(-1)
    else:
        print('%s OK' % i)
    finally:
        print('结束第%d轮循环\n-----------' % i)

try:
    n = 0
    print(10 / n)
except ValueError as e:
    print('ValueError:', e)
except ZeroDivisionError as e:
    print('ZeroDivisionError:', e)
else:
    print('No error.')
finally:
    print('Finally.')
print('End')
print('-' * 40)

# 转换异常的类型
try:
    try:
        10 / 0
    except ZeroDivisionError:
        raise ValueError('Convert error type.')
except ValueError as ve:
    print(ve)


#SQLITE3 Database API

import sqlite3

def main():
    print('Connecting')
    db=sqlite3.connect('db-api.db')         #creates a db file
    cur=db.cursor()                         #creating a cursor to connect and execute programs
    cur.execute('DROP TABLE IF EXISTS test')        #execute is used to execute sql command
    cur.execute("""CREATE TABLE test(
        id INTEGER PRIMARY KEY,
        string TEXT,
        number INTEGER )"""
    )
    cur.execute("""INSERT INTO test values(1,'hello','57')""")
    cur.execute("""INSERT INTO test (string,number) values ('yeee',29)""")
    db.commit()          #commit
    for row in cur.execute('SELECT * FROM test'):
        print(row)
    db.close()           #close

main()


import os
from celery import Celery
from celery.schedules import crontab

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings')

app = Celery('config')

app.config_from_object('django.conf:settings', namespace='CELERY')

app.autodiscover_tasks()

app.conf.beat_schedule = {
    'get_tracks_from_playlist_every_minute': {
        'task': 'music.tasks.get_tracks_from_api_playlist',
        'schedule': crontab(minute='*/20'),
    },
    'get_playlist_original_id_every_minute': {
        'task': 'music.tasks.get_playlist_original_id',
        'schedule': crontab(minute='*/20'),
    },
    'get_playlists_from_keywords_every_minute': {
        'task': 'music.tasks.get_playlists_from_keywords',
        'schedule': crontab(minute='*/20'),
    }
}


# coding:utf-8
# !/user/bin/python
'''
Created on 2017年11月25日
@author: 线程类
'''
import hashlib
import pickle
import threading

from flask import request

from app.share.caheFiles import LOCK_DICT

class MyThread(threading.Thread):

    def __init__(self, func, args=None, kwargs=None):
        super(MyThread, self).__init__()
        self.func = func
        self.args = args
        self.kwargs = kwargs
        sign = request.url if hasattr(request, "endpoint") else None
        self.key = hashlib.sha1(pickle.dumps((self.func.func_name, sign))).hexdigest()
        self.result = None

    @property
    def lock(self):
        lock = LOCK_DICT.get(self.key)
        if not lock:
            lock = threading.RLock()
            LOCK_DICT[self.key] = lock
        return lock

    def run(self):
        """"重写start方法,加锁,逻辑处理异常抛出异常,这里可以拓展结果返回"""
        try:
            if self.lock.acquire():
                self.result = self.func(*self.args)
        except:
            raise
        finally:
            self.lock.release()

    def get_result(self):
        try:
            return self.result  # 获取时候需要join(), 不然主线程比子线程跑的快,会拿不到结果
        except:
            raise


"""
### A service to perform geometric operations on geospatial datasets.

The *geometry service* offers the ability to apply geometric operations on one or more spatial files. Following the upload of a spatial file, the requested geometric operation generates, promptly or asynchronously, a new spatial file available for download. Depending on the operation, the generated file could

- contain new geometries (**constructive** operations) constructed from the original ones,
- be a filtered subset of the initial file (**filter** operations), or
- be a result of a spatial join (**join** operations) of two files.
"""

import os, sys
from apispec import APISpec
from apispec_webframeworks.flask import FlaskPlugin
import tempfile
from geometry_service.database import db
from ._version import __version__
from .api.doc_components import add_components
from .loggers import logger

# OpenAPI documentation
logger.debug('Initializing OpenAPI specification.')
spec = APISpec(
    title="Geometry API",
    version=__version__,
    info=dict(
        description=__doc__,
        contact={"email": "pmitropoulos@getmap.gr"}
    ),
    externalDocs={"description": "GitHub", "url": "https://github.com/OpertusMundi/geometry-service"},
    openapi_version="3.0.2",
    plugins=[FlaskPlugin()],
)
logger.debug('Adding OpenAPI specification components.')
add_components(spec)

# Check environment variables
if os.getenv('DATABASE_URI') is None:
    logger.fatal('Environment variable not set [variable="DATABASE_URI"]')
    sys.exit(1)
if os.getenv('SECRET_KEY') is None:
    logger.fatal('Environment variable not set [variable="SECRET_KEY"]')
    sys.exit(1)
if os.getenv('OUTPUT_DIR') is None:
    logger.fatal('Environment variable not set [variable="OUTPUT_DIR"]')
    sys.exit(1)
if os.getenv('INPUT_DIR') is None:
    logger.fatal('Environment variable not set [variable="INPUT_DIR"]')
    sys.exit(1)
if os.getenv('WORKING_DIR') is None:
    working_dir = os.path.join(tempfile.gettempdir(), os.getenv('FLASK_APP'))
    os.environ['WORKING_DIR'] = working_dir
    logger.info('Set environment variable [WORKING_DIR="%s"]', working_dir)
if os.getenv('CORS') is None:
    os.environ['CORS'] = '*'
    logger.info('Set environment variable [CORS="*"]')

# Create directories
for path in [os.environ['WORKING_DIR'], os.environ['OUTPUT_DIR']]:
    try:
        os.makedirs(path)
    except OSError:
        pass
    else:
        logger.info("Created directory: %s.", path)

def create_app():
    """Create flask app."""
    from flask import Flask, make_response, g, request
    from flask_cors import CORS
    from shutil import rmtree
    from geometry_service.database.model import Queue
    from geometry_service.api import constructive, filter_, join, jobs, misc

    logger.debug('Initializing app.')
    app = Flask(__name__)
    app.config.from_mapping(
        SECRET_KEY=os.environ['SECRET_KEY'],
        SQLALCHEMY_DATABASE_URI=os.environ['DATABASE_URI'],
        SQLALCHEMY_TRACK_MODIFICATIONS=False,
        JSON_SORT_KEYS=False,
        EXECUTOR_TYPE="thread",
        EXECUTOR_MAX_WORKERS="1"
    )
    db.init_app(app)

    #Enable CORS
    if os.getenv('CORS') is not None:
        if os.getenv('CORS')[0:1] == '[':
            origins = json.loads(os.getenv('CORS'))
        else:
            origins = os.getenv('CORS')
        cors = CORS(app, origins=origins)

    # Register Blueprints
    constructive.executor.init_app(app)
    filter_.executor.init_app(app)
    join.executor.init_app(app)
    logger.debug('Registering blueprints.')
    # Add blueprints
    app.register_blueprint(constructive.bp)
    app.register_blueprint(filter_.bp)
    app.register_blueprint(join.bp)
    app.register_blueprint(jobs.bp)
    app.register_blueprint(misc.bp)

    # Register documentation
    logger.debug('Registering documentation.')
    with app.test_request_context():
        for view in app.view_functions.values():
            spec.path(view=view)

    @app.route("/", methods=['GET'])
    def index():
        """The index route, returns the JSON OpenAPI specification."""
        logger.info('Generating the OpenAPI document...')
        return make_response(spec.to_dict(), 200)

    @app.teardown_request
    def clean_working_path(error=None):
        if request.values.get('response') == 'prompt':
            try:
                session = getattr(g, 'session')
                rmtree(session['working_path'])
            except:
                pass

    # Register cli commands
    with app.app_context():
        import geometry_service.cli

    logger.debug('Created app.')
    return app


import nltk
print(nltk.corpus.gutenberg.fileids())
emma = nltk.Text(nltk.corpus.gutenberg.words('austen-emma.txt'))
print(emma.concordance("surprize"))

from nltk.corpus import gutenberg
for fileid in gutenberg.fileids():
    num_chars = len(gutenberg.raw(fileid))
    num_words = len(gutenberg.words(fileid))
    num_sents = len(gutenberg.sents(fileid))
    num_vocab = len(set([w.lower() for w in gutenberg.words(fileid)]))
print(int(num_chars/num_words), int(num_words/num_sents), int(num_words/num_vocab), fileid)

# 网络和聊天文本
from nltk.corpus import webtext
for fileid in webtext.fileids():
    print(fileid, webtext.raw(fileid)[:65], '...')

from nltk.corpus import nps_chat
chatroom = nps_chat.posts('10-19-20s_706posts.xml')
print(chatroom[123])


import unittest
import redis
import time

from redismutex import RedisMutex
from redismutex.errors import (
    BlockTimeExceedError, MutexLockError, MutexUnlockError
)

REDIS_CONN = redis.StrictRedis(host='localhost', port=6379, db=4)

class TestRedisMutex(unittest.TestCase):
    """Tests for basic locking and unlocking
    """

    def setUp(self):
        self.redis = REDIS_CONN
        self.mutex = RedisMutex(self.redis)
        self.key = str(time.time())

    def test_basic_flow(self):

        self.mutex = self.mutex.acquire_lock(self.key)

        # Check if key and value are set properly
        self.assertEqual(self.mutex.key, self.key)
        self.assertTrue(self.mutex.value)
        self.assertIsInstance(self.mutex.value, str)

        self.mutex = self.mutex.release_lock()

        # Check if key and value are reset to None after unlock
        self.assertIsNone(self.mutex.key)
        self.assertIsNone(self.mutex.value)

    def test_as_context_manager(self):

        with self.mutex.acquire_lock(self.key):

            # Check if key and value are set properly
            self.assertEqual(self.mutex.key, self.key)
            self.assertTrue(self.mutex.value)
            self.assertIsInstance(self.mutex.value, str)

        # Check if key and value are reset to None after unlock
        self.assertIsNone(self.mutex.key)
        self.assertIsNone(self.mutex.value)

    def test_invalid_connection(self):
        """Check if correct redis connection is provided to RedisMutex
        """
        with self.assertRaises(TypeError):
            RedisMutex('invlaid-connection')

        with self.assertRaises(TypeError):
            RedisMutex(1234)

        with self.assertRaises(TypeError):
            RedisMutex({'foo': 'bar'})

    def test_generate_unique_id(self):
        """mutex.generate_unique_id() should always generate a different
        string and should be independent of any parameter.
        """
        unique_ids = []
        for i in range(100):
            unique_ids.append(self.mutex.generate_unique_id())

        self.assertIsInstance(unique_ids[0], str)

        set_unique_ids = set(unique_ids)
        self.assertEqual(len(unique_ids), len(set_unique_ids))

    def test_lock_already_exists(self):
        """When a lock on the given key already exists.
        """

        # Create a lock using a new mutex
        new_mutex = RedisMutex(self.redis, block_time=10, expiry=12)
        new_mutex = new_mutex.acquire_lock(self.key)

        self.mutex.block_time = 1
        with self.assertRaises(BlockTimeExceedError):
            self.mutex.acquire_lock(self.key)

        # A blocking mutex will raise a MutexLockError instead of
        # BlockTimeExceedError as blcok time does not comes into play
        # during locking of a non blocking mutex.
        self.mutex.blocking = False
        with self.assertRaises(MutexLockError):
            self.mutex.acquire_lock(self.key)

        new_mutex.release_lock()

    def test_unlock_without_locking(self):
        """When release_lock() or unlock() is called before acquiring
        a valid mutex lock.
        """
        with self.assertRaises(MutexUnlockError):
            self.mutex.release_lock()

        with self.assertRaises(MutexUnlockError):
            self.mutex.unlock()

class TestRedisMutexKeyExpiry(unittest.TestCase):
    """Tests related to mutex key expiration
    """

    def setUp(self):
        self.redis = REDIS_CONN
        self.mutex = RedisMutex(self.redis, block_time=1, expiry=2)
        self.key = str(time.time())

    def test_key_expire(self):
        """When unlock is attempted after key is expired
        """
        with self.assertRaises(MutexUnlockError):
            # Acquire lock and release after the expiration of the key.
            # This would raise MutexLockError as the key does not
            # exists in redis anymore and mutex is trying to delete it.
            self.mutex.acquire_lock(self.key)
            time.sleep(2.5)
            self.mutex.release_lock()

    def test_key_override(self):
        """When unlock is attempted after mutex key is overriden by
        another mutex after expiration.
        """
        new_mutex = RedisMutex(self.redis, block_time=1, expiry=2)

        with self.assertRaises(MutexUnlockError):
            # Acquire lock and release after the expiration of the key.
            # After the expiration a new mutex acquires the lock for the
            # same key. This key cannot be deleted by the old mutex as
            # the unique value generated for the key will be different
            # and would hence raise a MutexLockValueError
            self.mutex.acquire_lock(self.key)
            time.sleep(2.5)
            new_mutex = new_mutex.acquire_lock(self.key)
            self.mutex.release_lock()

        # cleanup
        new_mutex = new_mutex.release_lock()

    def test_new_mutex_lock_after_expiry(self):
        """When a new mutex tries to acquire an expried lock
        """
        new_mutex = RedisMutex(self.redis, block_time=1, expiry=2)

        self.mutex.acquire_lock(self.key)
        # Let the key expire...
        time.sleep(2.5)

        # As the key has expired, a new mutex should be able to
        # acquire lock with the same key
        new_mutex.acquire_lock(self.key)

        self.assertEqual(new_mutex.key, self.key)
        self.assertTrue(new_mutex.value)
        self.assertIsInstance(new_mutex.value, str)

        new_mutex.release_lock()


from bs4 import BeautifulSoup

from ..crawler import Crawler
from .generic import ImageGetter
from .customExceptions import DownloaderInvaildParam

class WallhavenImage(ImageGetter):

    def __init__(self, search=None, limit=None, category="toplist", nsfw=True, ratios=None):
        """[Init for wallhaven.cc iterator image]

        Get image url from wallhaven.cc

        Keyword Arguments:
            search {str} -- key work search (default: {None})
            limit {[int]} -- [limit image urls get] (default: {None} -- unlimit)
            category {str} -- [wallhaven.cc category] (default: {"toplist"}) -- vaild value: toplist, random, latest
            nsfw {[bool]} -- nsfw flag (defautl: {True})
            ratios {[str]} -- ratio image (defautl: {None}) -- valid value: [None, "16x9", "16x10", "21x9", "32x9", "48x9"]
        """
        # self._validate_kwargs()
        if search == category == None:
            raise DownloaderInvaildParam("Required search or category is not None")
        if category not in ["toplist", "random", "latest"]:
            raise DownloaderInvaildParam('category vaild value: ["toplist", "random", "latest"]')

        self.search = search
        self.limit = limit
        self.category = category
        self.nsfw = bool(nsfw)
        self.ratios = ratios

    def __iter__(self):
        # self._validate_kwargs()

        self.crawler = Crawler()
        self.current_page = 0
        self.number_image = 0

        self.pre_load_urls = []
        self._generate_url = self._generator_url()

        return self

    def _validate_kwargs(self, **kwargs):
        pass

    def _generator_url(self):
        if self.search:
            url = f"https://wallhaven.cc/search?q={self.search}"
        else:
            url = f"https://wallhaven.cc/{self.category}"
        if self.ratios:
            params = f"categories=111&purity=10{self.nsfw:d}&topRange=1M&ratios={self.ratios}&sorting=toplist&order=desc"
        else:
            params = f"categories=111&purity=10{self.nsfw:d}&topRange=1M&sorting=toplist&order=desc"
        while True:
            self.current_page += 1
            yield f"{url}?{params}&page={self.current_page}"

    def _parse_image_url(self, html):
        soup = BeautifulSoup(html, "lxml")
        figure_elems = soup.select("ul > li > figure")
        for figure_elem in figure_elems:
            link_split = figure_elem.select_one("a.preview")["href"].split("/")
            is_png = figure_elem.select_one("div.thumb-info > span.png")
            if is_png:
                image_ext = "png"
            else:
                image_ext = "jpg"
            url = f"https://w.wallhaven.cc/full/{link_split[-1][:2]}/wallhaven-{link_split[-1]}.{image_ext}"
            yield url

    def get_urls(self):
        res = []
        self.current_page = 0
        self.crawler = Crawler()
        number_image_credit = self.limit
        for page_url in self._generate_url():
            html = self.crawler.get_text(page_url)
            urls = list(self._parse_image_url(html))
            res.extend(urls[:number_image_credit])
            number_image_credit -= len(urls)
            if number_image_credit > 0:
                continue
            return res
        return res

    def __next__(self):
        if len(self.pre_load_urls) == 0:
            page_url = next(self._generate_url)
            html = self.crawler.get_text(page_url)
            self.pre_load_urls = list(self._parse_image_url(html))

        image_url = self.pre_load_urls.pop(0)
        self.number_image += 1
        if self.limit and self.number_image > self.limit:
            raise StopIteration
        return image_url


def solution(number, k):
    answer = ''
    stack = [number[0]]
    left = ''
    for num in number[1:]:
        # 들어오는 값이 stack 의 끝 값보다 크면 stack값을 제거한다
        while len(stack) > 0 and stack[-1] < num and k > 0:
            k -= 1
            stack.pop()
        # 새로운 값을 삽입
        stack.append(num)
    if k != 0:
        # 값을 큰 순서대로 넣었기 때문에 제거 안한만큼 스택 뒤에서 제거해주면 된다.
        stack = stack[:-k]
    answer = ''.join(stack)
    return answer

solution("4177252841", 4)

프로그래머스/PR_큰 수 만들기.py


from dbt.adapters.factory import get_adapter, register_adapter

try:
    from dbt.flags import set_flags

    SET_FLAGS = True
except ImportError:
    SET_FLAGS = False
from dbt.task.base import ConfiguredTask

from dbt_coves.core.exceptions import MissingCommand

class BaseTask:
    """
    Base Task Class
    """

    needs_config = False
    needs_dbt_project = False

    def __init__(self, args, config=None):
        self.args = args

    @classmethod
    def register_parser(cls, sub_parsers, base_subparser):
        raise NotImplementedError()

    @classmethod
    def get_instance(cls, flags, coves_config=None):
        instance = cls(flags.args)
        instance.coves_flags = flags
        return instance

    def run(self) -> int:
        raise NotImplementedError()

class BaseConfiguredTask(ConfiguredTask, BaseTask):
    """
    Task class that requires a configuration
    """

    needs_config = True
    needs_dbt_project = True

    def __init__(self, args, config):
        super().__init__(args, config)
        try:
            self.adapter = get_adapter(self.config)
        except KeyError:
            register_adapter(self.config)
            self.adapter = get_adapter(self.config)
        self.coves_config = None
        self.coves_flags = None

    @classmethod
    def from_args(cls, args):
        if SET_FLAGS:
            set_flags(args)
        config = cls.ConfigType.from_args(args)
        try:
            return cls(args, config)
        # class cannot be instantiated with abstract methods.
        # that means a subcommand is missing.
        except TypeError:
            raise MissingCommand(cls.arg_parser)

    @classmethod
    def get_instance(cls, flags, coves_config):
        instance = cls.from_args(flags.args)
        instance.coves_config = coves_config
        instance.coves_flags = flags
        return instance

class NonDbtBaseTask(BaseTask):
    """
    Task class that requires a configuration
    """

    needs_config = True
    needs_dbt_project = False

    def __init__(self, args, config):
        super().__init__(args, config)
        self.coves_config = config
        self.coves_flags = None

    @classmethod
    def get_instance(cls, flags, coves_config):
        instance = cls(flags.args, coves_config)
        instance.coves_config = coves_config
        instance.coves_flags = flags
        return instance

    @classmethod
    def run(cls) -> int:
        raise MissingCommand(cls.arg_parser)

class NonDbtBaseConfiguredTask(BaseTask):
    """
    Task class that requires a configuration
    """

    needs_config = True
    needs_dbt_project = False

    def __init__(self, args, config):
        super().__init__(args, config)
        self.coves_config = config
        self.coves_flags = None

    @classmethod
    def get_instance(cls, flags, coves_config):
        instance = cls(flags.args, coves_config)
        instance.coves_config = coves_config
        instance.coves_flags = flags
        return instance


# -*- coding: utf-8 -*-
"""
Created on Wed Mar 29 16:01:44 2023

Source: https://huggingface.co/EleutherAI/gpt-neo-2.7B

GPT-Neo 2.7B - a transformer model designed using EleutherAI's replication of the
GPT-3 architecture. The model is available on HuggingFace. Although it can be used
for different tasks, the model is best at what it was pretrained for, which is
generating texts from a prompt.

The task in this script is text generation.

There are also a 1.3B and 6B versions.

"""

import torch
from transformers import AutoTokenizer, GPTNeoForCausalLM

tokenizer = AutoTokenizer.from_pretrained("EleutherAI/gpt-neo-2.7B")
model = GPTNeoForCausalLM.from_pretrained("EleutherAI/gpt-neo-2.7B")

inputs = tokenizer("Hello, my dog is cute", return_tensors="pt")
outputs = model(**inputs, labels=inputs["input_ids"])

input_ids = inputs["input_ids"]

gen_tokens = model.generate(
    input_ids,
    do_sample=True,
    temperature=0.9,
    max_length=100,
)

gen_text = tokenizer.batch_decode(gen_tokens)[0]

print("=========================================================")
print(gen_text)


from django.db import models

# Create your models here.
from courses.models import Lesson
from users.models import User

class TableModel(models.Model):

    class Meta:
        verbose_name = 'Посещаемость - Успеваемость'
        verbose_name_plural = 'Посещаемость - Успеваемость'
        ordering = ['id']

    is_here = models.BooleanField(default=None, blank=True, null=True, verbose_name='Посещаемость')
    score = models.IntegerField(blank=True, null=True, default=None, verbose_name='Оценка за ДЗ')
    student = models.ForeignKey(User, on_delete=models.CASCADE,
                                related_name='table_student', verbose_name='Студент')
    lesson = models.ForeignKey(Lesson, on_delete=models.CASCADE,
                               related_name='lesson', verbose_name='Урок')
    date_of_lesson = models.DateField()


"""
This module is a set of useful function, while famous library does not seem to contain.
"""
## standard libraries

## 3rd party libraries
import numpy as np
from scipy.special import psi

## local libraries

def ratio_tanh_x(x):
    """
    Calculating f(x)=tanh(x)/x.
    While lim_{x -> 0} f(x) = 1, overflow is occured at zero point.
    Thus, this function is conditioned by zero point and other points.
    """
    zero_condition = 1e-20
    ret_val = np.zeros(x.shape)
    zero_ind = np.abs(x) < zero_condition
    ret_val[~zero_ind] = np.tanh(x[~zero_ind])/x[~zero_ind]
    ret_val[zero_ind] = 1
    return ret_val

def logcosh(x:np.ndarray):
    """
    Calculating a log cosh(x).
    When absolute value of x is very large, this function are overflow,
    so we avoid it.
    """
    return np.abs(x) + np.log((1 + np.exp(-2 * np.abs(x)))/2)

def multipsi(x:float, d:int):
    """
    Calculate multivariate digamma function f(x,d):
    f(x,d) = sum_{j=1}^d psi(x + (1-j)/2)
    + Input
        1. x: float value > d
        2. d: integer
    + Output
        f(x,d)
    """
    return (psi(x) + (1 - np.arange(1, d+1))/2).sum()

def rgmm(ratio:np.ndarray, mean:np.ndarray, precision:np.ndarray, size:int=1, data_seed:int = -1):
    """
    Generate data following to mixture of a Gaussian distribution with ratio, mean, and precision.
    Assigning the data size and seed is admissible for this function.

    + Input:
        + ratio: K dimensional ratio vector, i.e. (K-1) dimensional simplex.
        + mean: K times M dimensional vector, where K is the number of component.
        + precision: K dimensional R_+ vector representing a precision for each distribution.
        + size: (optional) the number of data, default is 1.
        + data_seed: (optional) seed to gerante data.
    """
    if data_seed > 0: np.random.seed(data_seed)
    data_label = np.random.multinomial(n = 1, pvals = ratio, size = size)
    data_label_arg = np.argmax(data_label, axis = 1)
    M = mean.shape[1]
    X = np.array([np.random.normal(loc=mean[data_label_arg[i],:], scale=1/precision[data_label_arg[i],:]) for i in range(size)])
    return (X, data_label, data_label_arg)

# def logpdf_mixture_dist(x:np.ndarray, param:dict, component_log_dist:Callable[[np.ndarray, np.ndarray, np.ndarray], np.ndarray]):
#     """
#     対数尤度の計算
#     確率分布が混合分布の時を想定:
#     $\log p(x|w) = \sum_{i = 1}^n \log p(x_i|w) = \sum_{i = 1}^n \log \exp(L_{ik}(w)) =  \sum_{i = 1}^n \{\hat{L}_{i} + \log \exp(L_{ik}(w) - \hat{L}(i)) \}$,
#     where L_{ik} = \log a_k + \log p(x_i|w, y_{ik} = 1)\hat{L}(i) = \max_{k} L_{ik}, p(x_i|w, y_{ik}=1):i番目のサンプルのk番目のクラスタの確率分布
#
#     + 入力:
#         1. x:入力データ(n*M)
#         2. param: 確率分布のパラメータ(ratio: 混合比, mean: 各クラスタの平均値 K*M, scale: 各クラスタのscale(正規分布における標準偏差) K*M)
#         3. component_log_dist: 各クラスタの対数確率密度の値 logp(x|w,y)
#     """
#     n = x.shape[0]
#     K = len(param["ratio"])
#     loglik = np.zeros((n,K))
#     for k in range(K):
#         if param["scale"].ndim == 2:
#             loglik[:,k] = np.log(param["ratio"][k]) + component_log_dist(test_x, param["mean"][k,:],  param["scale"][k,:])
#         elif param["scale"].ndim == 3:
#             loglik[:,k] = np.log(param["ratio"][k]) + component_log_dist(test_x, param["mean"][k,:],  param["scale"][k,:,:])
#         else:
#             raise ValueError("Error precision, dimension of precision must be 2 or 3!")
#     max_loglik = loglik.max(axis = 1)
#     norm_loglik = loglik - np.repeat(max_loglik,K).reshape(n,K)
#     return (np.log(np.exp(norm_loglik).sum(axis = 1)) + max_loglik)


n = int(input()) # 저울추의 개수
w = list(map(int, input().split())) # 저울추의 무게
w.sort()
answer = 1
for i in range(n):
    if(w[i] <= answer):
        answer += w[i]
    else:
        break
print(answer)

######### 처음 시도했던 풀이 (조합) -> 메모리초과 ##########
# from itertools import combinations

# w.sort()
# candi = [i for i in w]

# def getcandidate(n):
#     com = list(combinations(w, n))
#     for i in range(len(com)):
#         candi.append(sum(com[i]))

# for i in range(2, n+1):
#     getcandidate(i)

# candi = list(set(candi))
# candi.sort()
# # print(candi)

# i = 1
# while(i in candi):
#     i += 1

# print(i)


from djangosaml2.backends import Saml2Backend
import logging
from typing import Any, Optional, Tuple
from django.conf import settings
from django.core.exceptions import (ImproperlyConfigured,
                                    MultipleObjectsReturned)

from administration.models import Student, Employee

logger = logging.getLogger('djangosaml2')

class CustomSaml2Backend(Saml2Backend):
    def is_authorized(self, attributes, attribute_mapping):
        if attributes.get('hrEduPersonHomeOrg', [''])[0]=='riteh.hr':
            return True
        return False

    def get_or_create_user(self,
            user_lookup_key: str, user_lookup_value: Any, create_unknown_user: bool,
            idp_entityid: str, attributes: dict, attribute_mapping: dict, request
        ) -> Tuple[Optional[settings.AUTH_USER_MODEL], bool]:
        """ Look up the user to authenticate. If he doesn't exist, this method creates him (if so desired).
            The default implementation looks only at the user_identifier. Override this method in order to do more complex behaviour,
            e.g. customize this per IdP.
        """
        UserModel = self._user_model

        # Construct query parameters to query the userModel with. An additional lookup modifier could be specified in the settings.
        user_query_args = {
            user_lookup_key + getattr(settings, 'SAML_DJANGO_USER_MAIN_ATTRIBUTE_LOOKUP', ''): user_lookup_value
        }

        # Lookup existing user
        # Lookup existing user
        user, created = None, False
        try:
            user = UserModel.objects.get(**user_query_args)
        except MultipleObjectsReturned:
            logger.error("Multiple users match, model: %s, lookup: %s", UserModel._meta, user_query_args)
        except UserModel.DoesNotExist:
            # Create new one if desired by settings
            if create_unknown_user:
                user = UserModel(**{ user_lookup_key: user_lookup_value })

               	user.save()
                if attributes.get('hrEduPersonPrimaryAffiliation', [''])[0]=='student':
                    student = Student(user=user)
                    student.area_of_study = attributes.get('hrEduPersonScienceArea', [''])[0]
                    student.category = attributes.get('hrEduPersonStudentCategory', [''])[0]
                    student.jmbag = attributes.get('hrEduPersonUniqueNumber', [''])[0]

                    student.save()
                else:
                	employee = Employee(user=user)
                	employee.save()

                created = True
                logger.debug('New user created: %s', user)
            else:
                logger.error('The user does not exist, model: %s, lookup: %s', UserModel._meta, user_query_args)

        return user, created


# -*- coding: utf-8 -*-
"""
Created on Wed May  8 06:09:20 2019

@author: Anupam Sharma
"""

"""
Code Challenge
  Name:
    Operations Function
  Filename:
    operation.py
  Problem Statement:
    Write following functions for list operations. Take list as input from the User
    Add(), Multiply(), Largest(), Smallest(), Sorting(), Remove_Duplicates(), Print()
    Only call Print() function to display the results in the below displayed
    format (i.e all the functions must be called inside the print() function
    and only the Print() is to be called in the main script)

  Input:
    5,2,6,2,3
  Output:
    Sum = 18
    Multiply = 360
    Largest = 6
    Smallest = 2
    Sorted = [2, 2, 3, 5, 6]
    Without Duplicates = [2, 3, 5, 6]
"""

add=0
multiply=1
i=input("enter a number").split()
print(i)
for l in i:
    print(l)
    int(l)
    add = add + int(l)
    multiply=multiply * int(l)
print(add)
print(multiply)
sort=list(i)


import sys
# n 입력
n = int(input())
# print(n)

# 학생 정보 입력받을 리스트
student = []
for i in range(n):
    student.append(sys.stdin.readline().split())
# print(student)

# 문제에서 주어진 조건대로 정렬
student.sort(key=lambda student: (-int(student[1]), int(student[2]), -int(student[3]), student[0]))

for i in student:
    print(i[0])


import requests
import json
from os import path

'''
This is for testing anomaly detection models
'''
anomaly_files_path_conn = path.join('Anomaly_detection', 'inputfile', 'Anamoly_test.json')
anomaly_files_path_dns = path.join('Anomaly_detection', 'inputfile', 'sampl.json')
anomaly_files_path_http = path.join('Anomaly_detection', 'inputfile', 'http_test.json')

DGA = path.join('freq_master', 'data', 'input', 'input.txt')
phis = path.join('StreamingPhish', 'data', 'input.txt')
useragent = path.join('User_agent', 'input', 'multiple.txt')

# for anomaly detection
path_list = {anomaly_files_path_conn:'/detect-anamaly-conn',
             anomaly_files_path_dns:'/detect-anamaly-dns',
             anomaly_files_path_http:'/detect-anamaly-http',
             }

# for dga phishing and useragent
path_list1 = { DGA:'/measure',phis:'/phishService', useragent:'/analyze_user_for_traffic'}

for i, v in path_list.items():
    print(i)
    try:
        with open(i) as f:
            data = json.load(f)

        response = requests.post('http://127.0.0.1:5500'+v, json=data)

        if response.status_code == 200:
            print('Success')
            print("Status code: ", response.status_code)
            response_Json = response.json()
            print("Printing Post JSON data")
            print(response_Json)
            print('\n\n')

    except Exception as e:
            pass

for i,v in path_list1.items():
    with open(i) as f:
        data = f.read()

    response = requests.post('http://127.0.0.1:5500'+v, data=data)
    if response.status_code == 200:
        print('Success')
        print("Status code: ", response.status_code)
        print("Printing Post JSON data")
        print(response.content)
        print('\n\n')


import re

filename = input().strip()
replace_count = int(input())
needle = input()
replace = input()

file = open(filename, 'r')
data = file.read()
file.close()
if replace_count == 0:
    print(data)
else:
    needle = needle.replace('\n', '')
    needle = needle.replace('\r', '')
    print(re.sub(re.escape(needle), replace, data, replace_count, flags=re.IGNORECASE))