#!/usr/bin/env python

import json
import threading

from gwcli.node import UIGroup, UINode, UIRoot
from gwcli.hostgroup import HostGroups
from gwcli.storage import Disks
from gwcli.client import Clients, CHAP
from gwcli.utils import (this_host, response_message, GatewayAPIError,
                         GatewayError, APIRequest, console_message, valid_iqn)

import ceph_iscsi_config.settings as settings
from ceph_iscsi_config.utils import (normalize_ip_address, format_lio_yes_no)
from ceph_iscsi_config.gateway import GWTarget

import rtslib_fb.root as root

from gwcli.ceph import CephGroup

# FIXME - code is using a self signed cert common across all gateways
# the embedded urllib3 package will issue warnings when ssl cert validation is
# disabled - so this disable_warnings stops the user interface from being
# bombed
from requests.packages import urllib3

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

class ISCSIRoot(UIRoot):

    def __init__(self, shell, scan_threads=1, endpoint=None):
        UIRoot.__init__(self, shell)

        self.error = False
        self.error_msg = ''
        self.interactive = True           # default interactive mode
        self.scan_threads = scan_threads

        if settings.config.api_secure:
            self.http_mode = 'https'
        else:
            self.http_mode = 'http'

        if endpoint is None:

            self.local_api = ('{}://localhost:{}/'
                              'api'.format(self.http_mode,
                                           settings.config.api_port))

        else:
            self.local_api = endpoint

        self.config = {}
        # Establish the root nodes within the UI, for the different components

        self.disks = Disks(self)
        self.ceph = CephGroup(self)
        self.target = ISCSITarget(self)

    def refresh(self):
        self.config = self._get_config()

        if not self.error:

            if 'disks' in self.config:
                self.disks.refresh(self.config['disks'])
            else:
                self.disks.refresh({})

            if 'gateways' in self.config:
                self.target.gateway_group = self.config['gateways']
            else:
                self.target.gateway_group = {}

            if 'controls' in self.config:
                self.target.controls = self.config['controls']
            else:
                self.target.controls = {}

            if 'clients' in self.config:
                self.target.client_group = self.config['clients']
            else:
                self.target.client_group = {}

            self.target.refresh()

            self.ceph.refresh()

        else:
            # Unable to get the config, tell the user and exit the cli
            self.logger.critical("Unable to access the configuration "
                                 "object : {}".format(self.error_msg))
            raise GatewayError

    def _get_config(self, endpoint=None):

        if not endpoint:
            endpoint = self.local_api

        api = APIRequest(endpoint + "/config")
        api.get()

        if api.response.status_code == 200:
            try:
                return api.response.json()
            except Exception:
                self.error = True
                self.error_msg = "Malformed REST API response"
                return {}
        else:
            self.error = True
            self.error_msg = "REST API failure, code : " \
                             "{}".format(api.response.status_code)
            return {}

    def export_ansible(self, config):

        ansible_vars = []
        ansible_vars.append("seed_monitor: {}".format(self.ceph.local_ceph.healthy_mon))
        ansible_vars.append("cluster_name: {}".format(settings.config.cluster_name))
        ansible_vars.append("gateway_keyring: {}".format(settings.config.gateway_keyring))
        ansible_vars.append("deploy_settings: true")
        ansible_vars.append("perform_system_checks: true")
        ansible_vars.append('gateway_iqn: "{}"'.format(config['gateways']['iqn']))
        ansible_vars.append('gateway_ip_list: "{}"'.format(
            ",".join(config['gateways'].get('ip_list', []))))
        ansible_vars.append("# rbd device definitions")
        ansible_vars.append("rbd_devices:")

        disk_template = ("  - {{ pool: '{}', image: '{}', size: '{}', "
                         "host: '{}', state: 'present' }}")

        for disk in self.disks.children:
            ansible_vars.append(disk_template.format(disk.pool,
                                                     disk.image,
                                                     disk.size_h,
                                                     disk.owner))

        ansible_vars.append("# client connections")
        ansible_vars.append("client_connections:")
        client_template = ("  - {{ client: '{}', image_list: '{}', "
                           "chap: '{}', status: 'present' }}")

        for client in sorted(config['clients'].keys()):
            client_metadata = config['clients'][client]
            lun_data = client_metadata['luns']
            sorted_luns = [s[0] for s in sorted(lun_data.items(),
                                                key=lambda v: v[1]['lun_id'])]
            chap = CHAP(client_metadata['auth']['chap'])
            ansible_vars.append(client_template.format(client,
                                                       ','.join(sorted_luns),
                                                       chap.chap_str))
        for var in ansible_vars:
            print(var)

    def export_copy(self, config):

        fmtd_config = json.dumps(config, sort_keys=True,
                                 indent=4, separators=(',', ': '))
        print(fmtd_config)

    def ui_command_export(self, mode='ansible'):
        """
        Print the configuration in a format that can be used by ceph-ansible or
        as a backup.

        The export command supports two modes:

        ansible - The configuration will be printed in a format that can be
                  used for the ceph-ansible iscsigws.yml.
        copy - This prints the internal configuration. It can used for backup
               or for support requests.
        """

        valid_modes = ['ansible', 'copy']

        self.logger.debug("CMD: export mode={}".format(mode))

        if mode not in valid_modes:
            self.logger.error("Invalid export mode requested - supported "
                              "modes are: {}".format(','.join(valid_modes)))
            return

        current_config = self._get_config()
        if not current_config.get('gateways'):
            self.logger.error("Export requested, but the config is empty")
            return

        if mode == "ansible":
            self.export_ansible(current_config)
        elif mode == 'copy':
            self.export_copy(current_config)

    def ui_command_info(self):
        self.logger.debug("CMD: info")

        if settings.config.trusted_ip_list:
            display_ips = ','.join(settings.config.trusted_ip_list)
        else:
            display_ips = 'None'

        console_message("HTTP mode          : {}".format(self.http_mode))
        console_message("Rest API port      : {}".format(settings.config.api_port))
        console_message("Local endpoint     : {}".format(self.local_api))
        console_message("Local Ceph Cluster : {}".format(settings.config.cluster_name))
        console_message("2ndary API IP's    : {}".format(display_ips))

class ISCSITarget(UIGroup):
    help_intro = '''
                 The iscsi-target group defines the ISCSI Target that the
                 group of gateways will be known as by iSCSI initiators (clients).

                 Only one iSCSI target is allowed, but each target can consist
                 of 2-4 gateway nodes. Multiple gateways are needed to deliver
                 high availability storage to the iSCSI client.

                 '''

    def __init__(self, parent):
        UIGroup.__init__(self, 'iscsi-target', parent)
        self.gateway_group = {}
        self.client_group = {}

    def ui_command_create(self, target_iqn):
        """
        Create an iSCSI target. This target is defined across all gateway nodes,
        providing the client with a single 'image' for iscsi discovery.

        Only ONE iSCSI target is supported, at this time.
        """

        self.logger.debug("CMD: /iscsi create {}".format(target_iqn))

        defined_targets = [tgt.name for tgt in self.children]
        if len(defined_targets) > 0:
            self.logger.error("Only ONE iscsi target image is supported")
            return

        # We need LIO to be empty, so check there aren't any targets defined
        local_lio = root.RTSRoot()
        current_target_names = [tgt.wwn for tgt in local_lio.targets]
        if current_target_names:
            self.logger.error("Local LIO instance already has LIO configured "
                              "with a target - unable to continue")
            return

        # OK - this request is valid, but is the IQN usable?
        if not valid_iqn(target_iqn):
            self.logger.error("IQN name '{}' is not valid for "
                              "iSCSI".format(target_iqn))
            return

        # 'safe' to continue with the definition
        self.logger.debug("Create an iscsi target definition in the UI")

        local_api = ('{}://localhost:{}/api/'
                     'target/{}'.format(self.http_mode,
                                        settings.config.api_port,
                                        target_iqn))

        api = APIRequest(local_api)
        api.put()

        if api.response.status_code == 200:
            self.logger.info('ok')
            # create the target entry in the UI tree
            Target(target_iqn, self)
        else:
            self.logger.error("Failed to create the target on the local node")

            raise GatewayAPIError("iSCSI target creation failed - "
                                  "{}".format(response_message(api.response,
                                                               self.logger)))

    def ui_command_clearconfig(self, confirm=None):
        """
        The 'clearconfig' command allows you to return the configuration to an
        unused state: LIO on each gateway will be cleared, and gateway
        definitions in the configuration object will be removed.

        > clearconfig confirm=true

        In order to run the clearconfig command, all clients and disks *must*
        have already have been removed.
        """

        self.logger.debug("CMD: clearconfig confirm={}".format(confirm))

        confirm = self.ui_eval_param(confirm, 'bool', False)
        if not confirm:
            self.logger.error("To clear the configuration you must specify "
                              "confirm=true")
            return

        # get a new copy of the config dict over the local API
        # check that there aren't any disks or client listed
        local_api = ("{}://localhost:{}/api/"
                     "config".format(self.http_mode,
                                     settings.config.api_port))

        api = APIRequest(local_api)
        api.get()

        if api.response.status_code != 200:
            self.logger.error("Unable to get fresh copy of the configuration")
            raise GatewayAPIError

        try:
            current_config = api.response.json()
        except Exception:
            self.logger.error("Malformed REST API response")
            raise GatewayAPIError

        num_clients = len(current_config['clients'].keys())
        num_disks = len(current_config['disks'].keys())

        if num_clients > 0 or num_disks > 0:
            self.logger.error("Clients({}) and Disks({}) must be removed first"
                              " before clearing the gateway "
                              "configuration".format(num_clients,
                                                     num_disks))
            return

        self.clear_config(current_config['gateways'])

    def clear_config(self, gateway_group):

        # we need to process the gateways, leaving the local machine until
        # last to ensure we don't fall foul of the api auth check
        gw_list = [gw_name for gw_name in gateway_group
                   if isinstance(gateway_group[gw_name], dict)]

        this_gw = this_host()
        if this_gw not in gw_list:
            self.logger.warning("Executor({}) must be in gateway list: "
                                "{}".format(this_gw, gw_list))
            return

        gw_list.remove(this_gw)
        gw_list.append(this_gw)

        for gw_name in gw_list:

            gw_api = ('{}://{}:{}/api/'
                      '_gateway/{}'.format(self.http_mode,
                                           gw_name,
                                           settings.config.api_port,
                                           gw_name))

            api = APIRequest(gw_api)
            api.delete()
            if api.response.status_code != 200:
                msg = response_message(api.response, self.logger)
                self.logger.error("Delete of {} failed : {}".format(gw_name,
                                                                    msg))
                raise GatewayAPIError
            else:
                self.logger.debug("- deleted {}".format(gw_name))

        # gateways removed, so lets delete the objects from the UI tree
        self.reset()

        # remove any bookmarks stored in the prefs.bin file
        del self.shell.prefs['bookmarks']

        self.logger.info('ok')

    def refresh(self):

        self.logger.debug("Refreshing gateway & client information")
        self.reset()
        if 'iqn' in self.gateway_group:
            tgt = Target(self.gateway_group['iqn'], self)
            tgt.controls = self.controls

            tgt.gateway_group.load(self.gateway_group)
            tgt.client_group.load(self.client_group)

    def summary(self):
        return "Targets: {}".format(len(self.children)), None

class Target(UINode):

    display_attributes = ["target_iqn", "control_values"]

    help_intro = '''
                 The iscsi target is the name that the group of gateways are
                 known as by the iscsi initiators (clients).
                 '''

    def __init__(self, target_iqn, parent):
        UIGroup.__init__(self, target_iqn, parent)
        self.target_iqn = target_iqn

        self.control_values = []
        self.controls = {}

        self.gateway_group = GatewayGroup(self)
        self.client_group = Clients(self)
        self.host_groups = HostGroups(self)

    def _get_controls(self):
        return self._controls.copy()

    def _set_controls(self, controls):
        self._controls = controls.copy()
        self._refresh_control_values()

    controls = property(_get_controls, _set_controls)

    def summary(self):
        return "Gateways: {}".format(len(self.gateway_group.children)), None

    def ui_command_reconfigure(self, attribute, value):
        """
        The reconfigure command allows you to tune various gateway attributes.
        An empty value for an attribute resets the lun attribute to its
        default.
        attribute : attribute to reconfigure. supported attributes:
          - cmdsn_depth : integer 1 - 512
          - dataout_timeout : integer 2 - 60
          - nopin_response_timeout : integer 3 - 60
          - nopin_timeout : integer 3 - 60
          - immediate_data : [Yes|No]
          - initial_r2t : [Yes|No]
          - first_burst_length : integer 512 - 16777215
          - max_burst_length : integer 512 - 16777215
          - max_outstanding_r2t : integer 1 - 65535
          - max_recv_data_segment_length : integer 512 - 16777215
          - max_xmit_data_segment_length : integer 512 - 16777215
        value     : value of the attribute to reconfigure
        e.g.
        set cmdsn_depth
          - reconfigure attribute=cmdsn_depth value=128
        reset cmdsn_depth
          - reconfigure attribute=cmdsn_depth value=
        """
        settings_list = GWTarget.SETTINGS
        if attribute not in settings_list:
            self.logger.error("supported attributes: {}".format(",".join(
                sorted(settings_list))))
            return

        # Issue the api request for the reconfigure
        gateways_api = ('{}://localhost:{}/api/'
                        'target/{}'.format(self.http_mode,
                                           settings.config.api_port,
                                           self.target_iqn))

        controls = {attribute: value}
        controls_json = json.dumps(controls)
        api_vars = {'mode': 'reconfigure', 'controls': controls_json}

        self.logger.debug("Issuing reconfigure request: controls={}".format(controls_json))
        api = APIRequest(gateways_api, data=api_vars)
        api.put()

        if api.response.status_code != 200:
            self.logger.error("Failed to reconfigure : "
                              "{}".format(response_message(api.response,
                                                           self.logger)))
            return

        config = self.parent.parent._get_config()
        if not config:
            self.logger.error("Unable to refresh local config")
        self.controls = config.get('controls', {})

        self.logger.info('ok')

    def _refresh_control_values(self):
        self.control_values = {}
        settings_list = GWTarget.SETTINGS
        for k in settings_list:
            val = self._controls.get(k)
            default_val = getattr(settings.config, k, None)
            if k in settings.Settings.LIO_YES_NO_SETTINGS:
                if val is not None:
                    val = format_lio_yes_no(val)
                default_val = format_lio_yes_no(default_val)

            if val is None or str(val) == str(default_val):
                self.control_values[k] = default_val
            else:
                self.control_values[k] = "{} (override)".format(val)

class GatewayGroup(UIGroup):

    help_intro = '''
                 The gateway-group shows you the high level details of the
                 iscsi gateway nodes that have been configured. It also allows
                 you to add further gateways to the configuration, but this
                 requires the API service instance to be started on the new
                 gateway host

                 If in doubt, use Ansible :)
                 '''

    def __init__(self, parent):

        UIGroup.__init__(self, 'gateways', parent)

        self.thread_lock = threading.Lock()
        self.check_interval = 10           # check gateway state every 'n' secs
        self.last_state = 0

        # record the shortcut
        shortcut = self.shell.prefs['bookmarks'].get('gateways', None)
        if not shortcut or shortcut is not self.path:

            self.shell.prefs['bookmarks']['gateways'] = self.path
            self.shell.prefs.save()
            self.shell.log.debug("Bookmarked %s as %s."
                                 % (self.path, 'gateways'))

    @property
    def gateways_down(self):
        return len([gw for gw in self.children
                    if gw.state != 'UP'])

    def load(self, gateway_group):
        # define the host entries from the gateway_group dict
        gateway_list = [gw for gw in gateway_group
                        if isinstance(gateway_group[gw], dict)]
        for gateway_name in gateway_list:
            Gateway(self, gateway_name, gateway_group[gateway_name])

        self.check_gateways()

    def ui_command_info(self):
        '''
        List configured gateways.
        '''
        self.logger.debug("CMD: ../gateways/ info")

        for child in self.children:
            console_message(child)

    def check_gateways(self):

        check_thread = threading.Timer(self.check_interval,
                                       self.check_gateways)
        check_thread.daemon = True
        check_thread.start()
        self.refresh()

    def refresh(self, mode='auto'):

        self.thread_lock.acquire()

        if len(self.children) > 0:
            for gw in self.children:
                gw.refresh(mode)
        else:
            pass

        gateways_down = self.gateways_down
        if gateways_down != self.last_state:
            if gateways_down == 0:
                self.logger.info("\nAll gateways accessible")
            else:
                err_str = "gateway is" if gateways_down == 1 else "gateways are"
                self.logger.warning("\n{} {} inaccessible - updates will "
                                    "be disabled".format(gateways_down,
                                                         err_str))
            self.last_state = gateways_down

        self.thread_lock.release()

    def ui_command_refresh(self):
        """
        refresh allows you to refresh the connection status of each of the
        configured gateways (i.e. check the up/down state).
        """
        num_gw = len(self.children)
        if num_gw > 0:
            self.logger.debug("{} gateways to refresh".format(num_gw))
            self.refresh(mode='interactive')
        else:
            self.logger.error("No gateways to refresh")

    def ui_command_create(self, gateway_name, ip_address, nosync=False,
                          skipchecks='false'):
        """
        Define a gateway to the gateway group for this iscsi target. The
        first host added should be the gateway running the command

        gateway_name ... should resolve to the hostname of the gateway
        ip_address ..... is the IPv4/IPv6 address of the interface the iscsi
                         portal should use
        nosync ......... by default new gateways are sync'd with the
                         existing configuration by cli. By specifying nosync
                         the sync step is bypassed - so the new gateway
                         will need to have it's rbd-target-gw daemon
                         restarted to apply the current configuration
                         (default = False)
        skipchecks ..... set this to true to force gateway validity checks
                         to be bypassed(default = false). This is a developer
                         option ONLY. Skipping these checks has the potential
                         to result in an unstable configuration.
        """

        ip_address = normalize_ip_address(ip_address)
        self.logger.debug("CMD: ../gateways/ create {} {} "
                          "nosync={} skipchecks={}".format(gateway_name,
                                                           ip_address,
                                                           nosync,
                                                           skipchecks))

        local_gw = this_host()
        current_gateways = [tgt.name for tgt in self.children]

        if gateway_name != local_gw and len(current_gateways) == 0:
            # the first gateway defined must be the local machine. By doing
            # this the initial create uses localhost, and places it's portal IP
            # in the gateway ip list. Once the gateway ip list is defined, the
            # api server can resolve against the gateways - until the list is
            # defined only a request from localhost is acceptable to the api
            self.logger.error("The first gateway defined must be the local "
                              "machine")
            return

        if skipchecks not in ['true', 'false']:
            self.logger.error("skipchecks must be either true or false")
            return

        if local_gw in current_gateways:
            current_gateways.remove(local_gw)

        config = self.parent.parent.parent._get_config()
        if not config:
            self.logger.error("Unable to refresh local config"
                              " over API - sync aborted, restart rbd-target-gw"
                              " on {} to sync".format(gateway_name))

        if nosync:
            sync_text = "sync skipped"
        else:
            sync_text = ("sync'ing {} disk(s) and "
                         "{} client(s)".format(len(config['disks']),
                                               len(config['clients'])))
        if skipchecks == 'true':
            self.logger.warning("OS version/package checks have been bypassed")

        self.logger.info("Adding gateway, {}".format(sync_text))

        gw_api = '{}://{}:{}/api'.format(self.http_mode,
                                         "localhost",
                                         settings.config.api_port)
        gw_rqst = gw_api + '/gateway/{}'.format(gateway_name)
        gw_vars = {"nosync": nosync,
                   "skipchecks": skipchecks,
                   "ip_address": ip_address}

        api = APIRequest(gw_rqst, data=gw_vars)
        api.put()

        msg = response_message(api.response, self.logger)
        if api.response.status_code != 200:
            self.logger.error("Failed : {}".format(msg))
            return

        self.logger.debug("{}".format(msg))
        self.logger.debug("Adding gw to UI")

        # Target created OK, get the details back from the gateway and
        # add to the UI. We have to use the new gateway to ensure what
        # we get back is current (the other gateways will lag until they see
        # epoch xattr change on the config object)
        new_gw_endpoint = ('{}://{}:{}/'
                           'api'.format(self.http_mode,
                                        gateway_name,
                                        settings.config.api_port))

        config = self.parent.parent.parent._get_config(endpoint=new_gw_endpoint)
        gw_config = config['gateways'][gateway_name]
        Gateway(self, gateway_name, gw_config)

        self.logger.info('ok')

    def summary(self):

        up_count = len([gw.state for gw in self.children if gw.state == 'UP'])
        gw_count = len(self.children)

        return ("Up: {}/{}, Portals: {}".format(up_count,
                                                gw_count,
                                                gw_count),
                up_count == gw_count)

    @property
    def interactive(self):
        """determine whether the cli is running in interactive mode"""
        return self.parent.parent.parent.interactive

class Gateway(UINode):

    display_attributes = ["name",
                          "gateway_ip_list",
                          "portal_ip_address",
                          "inactive_portal_ips",
                          "active_luns",
                          "tpgs",
                          "service_state"]

    TCP_PORT = 3260

    def __init__(self, parent, gateway_name, gateway_config):
        """
        Create the LIO element
        :param parent: parent object the gateway group object
        :param gateway_config: dict holding the fields that define the gateway
        :return:
        """

        UINode.__init__(self, gateway_name, parent)

        for k, v in gateway_config.iteritems():
            self.__setattr__(k, v)

        self.state = "DOWN"
        self.service_state = {"iscsi": "DOWN",
                              "api": "DOWN"}

        self.refresh()

    def ui_command_refresh(self):
        """
        The refresh command will initiate a check against the gateway node,
        checking that the API is available, and that the iscsi port is
        listening
        """

        self.refresh()

    def refresh(self, mode="interactive"):

        if mode == 'interactive':
            self.logger.debug("- checking iSCSI/API ports on "
                              "{}".format(self.name))
        self._get_state()

    def _get_state(self):
        """
        Determine iSCSI and gateway API service state using the _ping api
        endpoint
        :return:
        """

        lookup = {200: {"status": "UP",
                        "iscsi": "UP", "api": "UP"},
                  401: {"status": "UNAUTHORIZED",
                        "iscsi": "UNKNOWN", "api": "UP"},
                  403: {"status": "UNAUTHORIZED",
                        "iscsi": "UNKNOWN", "api": "UP"},
                  500: {"status": "UNKNOWN",
                        "iscsi": "UNKNOWN", "api": "UNKNOWN"},
                  503: {"status": "PARTIAL",
                        "iscsi": "DOWN", "api": "UP"},
                  999: {"status": "UNKNOWN",
                        "iscsi": "UNKNOWN", "api": "UNKNOWN"},
                  }

        gw_api = '{}://{}:{}/api/_ping'.format(self.http_mode,
                                               self.name,
                                               settings.config.api_port)
        api = APIRequest(gw_api)
        try:
            api.get()
            rc = api.response.status_code
            if rc not in lookup:
                rc = 999
        except GatewayAPIError:
            rc = 999

        self.state = lookup[rc].get('status')
        self.service_state['iscsi'] = lookup[rc].get('iscsi')
        self.service_state['api'] = lookup[rc].get('api')

    def summary(self):

        state = self.state
        return "{} ({})".format(self.portal_ip_address,
                                state), (state == "UP")


from tensorflow import keras
import numpy as np
import json
from flask import Flask, request, jsonify

app =  Flask(__name__)

modelLSTM =  keras.models.load_model("modelLSTM10.h5")

maxlen = 10

chars = [' ','a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z']

char_to_idx = {'u': 0,'n': 1,'c': 2,'o': 3,'v': 4,'e': 5,'r': 6,' ': 7,'t': 8,'h': 9,'s': 10,'a': 11,'m': 12,'i': 13,'g': 14,'p': 15,'l': 16,'y': 17,'k': 18,'w': 19,'d': 20,'b': 21,'f': 22,'j': 23,'q': 24,'x': 25,'z': 26}

idx_to_char = {0: 'u',1: 'n',2: 'c',3: 'o',4: 'v',5: 'e',6: 'r',7: ' ',8: 't',9: 'h',10: 's', 11: 'a', 12: 'm', 13: 'i',14: 'g',15: 'p',16: 'l',17: 'y',18: 'k',19: 'w',20: 'd',21: 'b',22: 'f',23: 'j',24: 'q',25: 'x',26: 'z'}

def sample(preds, temperature=1.0):
    # helper function to sample an index from a probability array
    preds = np.asarray(preds).astype('float64')
    preds = np.log(preds) / temperature
    exp_preds = np.exp(preds)
    preds = exp_preds / np.sum(exp_preds)
    probas = np.random.multinomial(1, preds, 1)

    return np.argmax(probas)

def textGenerator(seed):

    generatedText = seed+' '

    for diversity in [0.2, 0.5, 1.0, 1.2]:
        for i in range(100):
            x_pred = np.zeros((1, maxlen, len(chars)))
            for t, char in enumerate(seed):
                x_pred[0, t, char_to_idx[char]] = 1.

            preds = modelLSTM.predict(x_pred, verbose=0)[0]
            next_index = sample(preds, diversity)
            next_char = idx_to_char[next_index]

            seed = seed[1:] + next_char

            generatedText += next_char

    return generatedText

@app.route("/seed/new",methods=["POST"])
def index():
    if request.data:
        seedDict = json.loads(request.data)
        textGen = textGenerator(seedDict['seed'])

        return jsonify({'text-generated':textGen}), 200
    else:
        return "400 BAD REGQUEST", 400

if __name__ == '__main__':
    app.debug = True
    app.run()


import numpy as np
import pandas as pd
from scipy.spatial import cKDTree
import itertools
from operator import itemgetter
import geopandas as gpd
import math
from .coordinates import getdistance

def ckdnearest(dfA_origin,
               dfB_origin,
               Aname=['lon', 'lat'],
               Bname=['lon', 'lat']):
    '''
    Search the nearest points in dfB_origin for dfA_origin,
    and calculate the distance

    Parameters
    -------
    dfA_origin : DataFrame
        DataFrame A
    dfB_origin : DataFrame
        DataFrame B
    Aname : List
        The column of lng and lat in DataFrame A
    Bname : List
        The column of lng and lat in DataFrame A

    Returns
    -------
    gdf : DataFrame
        The output DataFrame
    '''
    if len(dfA_origin) == 0:
        raise Exception('The input DataFrame dfA is empty')   # pragma: no cover
    if len(dfB_origin) == 0:
        raise Exception('The input DataFrame dfB is empty')   # pragma: no cover
    gdA = dfA_origin.copy()
    gdB = dfB_origin.copy()
    from scipy.spatial import cKDTree
    btree = cKDTree(gdB[Bname].values)
    dist, idx = btree.query(gdA[Aname].values, k=1)
    gdA['index'] = idx
    gdB['index'] = range(len(gdB))
    gdf = pd.merge(gdA, gdB, on='index')
    if (Aname[0] == Bname[0]) & (Aname[1] == Bname[1]):
        gdf['dist'] = getdistance(   # pragma: no cover
            gdf[Aname[0]+'_x'],   # pragma: no cover
            gdf[Aname[1]+'_y'],   # pragma: no cover
            gdf[Bname[0]+'_x'],   # pragma: no cover
            gdf[Bname[1]+'_y'])   # pragma: no cover
    else:
        gdf['dist'] = getdistance(
            gdf[Aname[0]],
            gdf[Aname[1]],
            gdf[Bname[0]],
            gdf[Bname[1]])
    return gdf

def ckdnearest_point(gdA, gdB):
    '''
    This method will match the nearest points in gdfB to gdfA,
    and add a new column called dist

    Parameters
    -------
    gdA : GeoDataFrame
        GeoDataFrame A, point geometry

    gdB : GeoDataFrame
        GeoDataFrame B, point geometry

    Returns
    -------
    gdf : DataFrame
        The output DataFrame
    '''
    if len(gdA) == 0:
        raise Exception('The input GeoDataFrame gdfA is empty')   # pragma: no cover
    if len(gdB) == 0:
        raise Exception('The input GeoDataFrame gdfB is empty')   # pragma: no cover
    nA = np.array(list(gdA.geometry.apply(lambda x: (x.x, x.y))))
    nB = np.array(list(gdB.geometry.apply(lambda x: (x.x, x.y))))
    btree = cKDTree(nB)
    dist, idx = btree.query(nA, k=1)
    gdA['dist'] = dist
    gdA['index'] = idx
    gdB['index'] = range(len(gdB))
    gdf = pd.merge(gdA, gdB, on='index')
    return gdf

def ckdnearest_line(gdfA, gdfB):
    '''
    This method will seach from gdfB to find the nearest line to
    the point in gdfA.

    Parameters
    -------
    gdA : GeoDataFrame
        GeoDataFrame A, point geometry

    gdB : GeoDataFrame
        GeoDataFrame B, linestring geometry

    Returns
    -------
    gdf : DataFrame
        Searching the nearset linestring in gdfB for the point in gdfA
    '''
    if len(gdfA) == 0:
        raise Exception('The input GeoDataFrame gdfA is empty')   # pragma: no cover
    if len(gdfB) == 0:
        raise Exception('The input GeoDataFrame gdfB is empty')   # pragma: no cover
    A = np.concatenate(
        [np.array(geom.coords) for geom in gdfA.geometry.to_list()])
    B = [np.array(geom.coords) for geom in gdfB.geometry.to_list()]
    B_ix = tuple(itertools.chain.from_iterable(
        [itertools.repeat(i, x) for i, x in enumerate(list(map(len, B)))]))
    B = np.concatenate(B)
    ckd_tree = cKDTree(B)
    dist, idx = ckd_tree.query(A, k=1)
    idx = itemgetter(*idx)(B_ix)
    gdfA['dist'] = dist
    gdfA['index'] = idx
    gdfB['index'] = range(len(gdfB))
    gdf = pd.merge(gdfA, gdfB, on='index')
    return gdf

def splitline_with_length(Centerline, maxlength=100):
    '''
    The intput is the linestring GeoDataFrame. The splited line’s
    length wull be no longer than maxlength

    Parameters
    -------
    Centerline : GeoDataFrame
        Linestring geometry
    maxlength : number
        The maximum length of the splited line

    Returns
    -------
    splitedline : GeoDataFrame
        Splited line
    '''
    def splitline(route, maxlength):
        routelength = route.length
        from shapely.geometry import LineString
        lss = []
        for k in range(int(routelength/maxlength)+1):
            if k == int(routelength/maxlength):
                lm = routelength
            else:
                lm = (k+1)*maxlength   # pragma: no cover
            a = np.linspace(k*maxlength, lm, 10)
            ls = []
            for line in a:
                ls.append(route.interpolate(line))
            lss.append(LineString(ls))
        lss = gpd.GeoDataFrame(lss, columns=['geometry'])
        return lss
    lsss = []
    for i in range(len(Centerline)):
        route = Centerline['geometry'].iloc[i]
        lss = splitline(route, maxlength)
        lss['id'] = i
        lsss.append(lss)
    lsss = pd.concat(lsss)
    lsss['length'] = lsss.length
    splitedline = lsss
    return splitedline

def merge_polygon(data, col):
    '''
    The input is the GeoDataFrame of polygon geometry, and the col
    name. This function will merge the polygon based on the category
    in the mentioned column

    Parameters
    -------
    data : GeoDataFrame
        The polygon geometry
    col : str
        The column name for indicating category

    Returns
    -------
    data1 : GeoDataFrame
        The merged polygon
    '''
    groupnames = []
    geometries = []
    for i in data[col].drop_duplicates():
        groupnames.append(i)
        geometries.append(data[data[col] == i].unary_union)
    data1 = gpd.GeoDataFrame()
    data1['geometry'] = geometries
    data1[col] = groupnames
    return data1

def polyon_exterior(data, minarea=0):
    '''
    The input is the GeoDataFrame of the polygon geometry. The method
    will construct new polygon by extending the outer boundary of the
    ploygon

    Parameters
    -------
    data : GeoDataFrame
        The polygon geometry
    minarea : number
        The minimum area. Polygon of less area will be removed

    Returns
    -------
    data1 : GeoDataFrame
        The processed polygon
    '''
    data1 = data.copy()

    def polyexterior(p):
        from shapely.geometry import Polygon, MultiPolygon
        if type(p) == MultiPolygon:
            geometries = []
            for i in p.geoms:
                poly = Polygon(i.exterior)
                if minarea > 0:
                    if poly.area > minarea:
                        geometries.append(poly)
                else:
                    geometries.append(poly) # pragma: no cover
            return MultiPolygon(geometries)
        if type(p) == Polygon:
            return Polygon(p.exterior)
    data1['geometry'] = data1['geometry'].apply(polyexterior)
    data1 = data1[-data1['geometry'].is_empty]
    return data1


# -*- coding: gbk -*-
##################################################################
#   农信银系统 trcbka_dict 字典到 ztcbka_dict 字典映射函数
#
#   作    者：  关彬捷
#   程序文件:   rccpsMap8529Dtrcbka_dict2Dztcbka_dict.py
#   修改时间:   Wed Jul  2 11:35:49 2008
##################################################################
import AfaLoggerFunc,TradeContext
from types import *
def map(from_dict,to_dict):

    if from_dict.has_key('BJEDTE'):
        to_dict['BJEDTE'] = from_dict['BJEDTE']
        AfaLoggerFunc.tradeDebug('ztcbka_dict[BJEDTE] = ' + str(to_dict['BJEDTE']))
    else:
        AfaLoggerFunc.tradeDebug("trcbka_dict['BJEDTE']不存在")

    if from_dict.has_key('BSPSQN'):
        to_dict['BSPSQN'] = from_dict['BSPSQN']
        AfaLoggerFunc.tradeDebug('ztcbka_dict[BSPSQN] = ' + str(to_dict['BSPSQN']))
    else:
        AfaLoggerFunc.tradeDebug("trcbka_dict['BSPSQN']不存在")

    if from_dict.has_key('BRSFLG'):
        to_dict['BRSFLG'] = from_dict['BRSFLG']
        AfaLoggerFunc.tradeDebug('ztcbka_dict[BRSFLG] = ' + str(to_dict['BRSFLG']))
    else:
        AfaLoggerFunc.tradeDebug("trcbka_dict['BRSFLG']不存在")

    if from_dict.has_key('BESBNO'):
        to_dict['BESBNO'] = from_dict['BESBNO']
        AfaLoggerFunc.tradeDebug('ztcbka_dict[BESBNO] = ' + str(to_dict['BESBNO']))
    else:
        AfaLoggerFunc.tradeDebug("trcbka_dict['BESBNO']不存在")

    if from_dict.has_key('BEACSB'):
        to_dict['BEACSB'] = from_dict['BEACSB']
        AfaLoggerFunc.tradeDebug('ztcbka_dict[BEACSB] = ' + str(to_dict['BEACSB']))
    else:
        AfaLoggerFunc.tradeDebug("trcbka_dict['BEACSB']不存在")

    if from_dict.has_key('BETELR'):
        to_dict['BETELR'] = from_dict['BETELR']
        AfaLoggerFunc.tradeDebug('ztcbka_dict[BETELR] = ' + str(to_dict['BETELR']))
    else:
        AfaLoggerFunc.tradeDebug("trcbka_dict['BETELR']不存在")

    if from_dict.has_key('BEAUUS'):
        to_dict['BEAUUS'] = from_dict['BEAUUS']
        AfaLoggerFunc.tradeDebug('ztcbka_dict[BEAUUS] = ' + str(to_dict['BEAUUS']))
    else:
        AfaLoggerFunc.tradeDebug("trcbka_dict['BEAUUS']不存在")

    if from_dict.has_key('NCCWKDAT'):
        to_dict['NCCWKDAT'] = from_dict['NCCWKDAT']
        AfaLoggerFunc.tradeDebug('ztcbka_dict[NCCWKDAT] = ' + str(to_dict['NCCWKDAT']))
    else:
        AfaLoggerFunc.tradeDebug("trcbka_dict['NCCWKDAT']不存在")

    if from_dict.has_key('TRCCO'):
        to_dict['TRCCO'] = from_dict['TRCCO']
        AfaLoggerFunc.tradeDebug('ztcbka_dict[TRCCO] = ' + str(to_dict['TRCCO']))
    else:
        AfaLoggerFunc.tradeDebug("trcbka_dict['TRCCO']不存在")

    if from_dict.has_key('TRCDAT'):
        to_dict['TRCDAT'] = from_dict['TRCDAT']
        AfaLoggerFunc.tradeDebug('ztcbka_dict[TRCDAT] = ' + str(to_dict['TRCDAT']))
    else:
        AfaLoggerFunc.tradeDebug("trcbka_dict['TRCDAT']不存在")

    if from_dict.has_key('TRCNO'):
        to_dict['TRCNO'] = from_dict['TRCNO']
        AfaLoggerFunc.tradeDebug('ztcbka_dict[TRCNO] = ' + str(to_dict['TRCNO']))
    else:
        AfaLoggerFunc.tradeDebug("trcbka_dict['TRCNO']不存在")

    if from_dict.has_key('SNDBNKCO'):
        to_dict['SNDBNKCO'] = from_dict['SNDBNKCO']
        AfaLoggerFunc.tradeDebug('ztcbka_dict[SNDBNKCO] = ' + str(to_dict['SNDBNKCO']))
    else:
        AfaLoggerFunc.tradeDebug("trcbka_dict['SNDBNKCO']不存在")

    if from_dict.has_key('SNDBNKNM'):
        to_dict['SNDBNKNM'] = from_dict['SNDBNKNM']
        AfaLoggerFunc.tradeDebug('ztcbka_dict[SNDBNKNM] = ' + str(to_dict['SNDBNKNM']))
    else:
        AfaLoggerFunc.tradeDebug("trcbka_dict['SNDBNKNM']不存在")

    if from_dict.has_key('RCVBNKCO'):
        to_dict['RCVBNKCO'] = from_dict['RCVBNKCO']
        AfaLoggerFunc.tradeDebug('ztcbka_dict[RCVBNKCO] = ' + str(to_dict['RCVBNKCO']))
    else:
        AfaLoggerFunc.tradeDebug("trcbka_dict['RCVBNKCO']不存在")

    if from_dict.has_key('RCVBNKNM'):
        to_dict['RCVBNKNM'] = from_dict['RCVBNKNM']
        AfaLoggerFunc.tradeDebug('ztcbka_dict[RCVBNKNM] = ' + str(to_dict['RCVBNKNM']))
    else:
        AfaLoggerFunc.tradeDebug("trcbka_dict['RCVBNKNM']不存在")

    if from_dict.has_key('BOJEDT'):
        to_dict['BOJEDT'] = from_dict['BOJEDT']
        AfaLoggerFunc.tradeDebug('ztcbka_dict[BOJEDT] = ' + str(to_dict['BOJEDT']))
    else:
        AfaLoggerFunc.tradeDebug("trcbka_dict['BOJEDT']不存在")

    if from_dict.has_key('BOSPSQ'):
        to_dict['BOSPSQ'] = from_dict['BOSPSQ']
        AfaLoggerFunc.tradeDebug('ztcbka_dict[BOSPSQ] = ' + str(to_dict['BOSPSQ']))
    else:
        AfaLoggerFunc.tradeDebug("trcbka_dict['BOSPSQ']不存在")

    if from_dict.has_key('ORTRCCO'):
        to_dict['ORTRCCO'] = from_dict['ORTRCCO']
        AfaLoggerFunc.tradeDebug('ztcbka_dict[ORTRCCO] = ' + str(to_dict['ORTRCCO']))
    else:
        AfaLoggerFunc.tradeDebug("trcbka_dict['ORTRCCO']不存在")

    if from_dict.has_key('CUR'):
        to_dict['CUR'] = from_dict['CUR']
        AfaLoggerFunc.tradeDebug('ztcbka_dict[CUR] = ' + str(to_dict['CUR']))
    else:
        AfaLoggerFunc.tradeDebug("trcbka_dict['CUR']不存在")

    if from_dict.has_key('OCCAMT'):
        to_dict['OCCAMT'] = from_dict['OCCAMT']
        AfaLoggerFunc.tradeDebug('ztcbka_dict[OCCAMT] = ' + str(to_dict['OCCAMT']))
    else:
        AfaLoggerFunc.tradeDebug("trcbka_dict['OCCAMT']不存在")

    if from_dict.has_key('NOTE1'):
        to_dict['NOTE1'] = from_dict['NOTE1']
        AfaLoggerFunc.tradeDebug('ztcbka_dict[NOTE1] = ' + str(to_dict['NOTE1']))
    else:
        AfaLoggerFunc.tradeDebug("trcbka_dict['NOTE1']不存在")

    if from_dict.has_key('NOTE2'):
        to_dict['NOTE2'] = from_dict['NOTE2']
        AfaLoggerFunc.tradeDebug('ztcbka_dict[NOTE2] = ' + str(to_dict['NOTE2']))
    else:
        AfaLoggerFunc.tradeDebug("trcbka_dict['NOTE2']不存在")

    if from_dict.has_key('NOTE3'):
        to_dict['NOTE3'] = from_dict['NOTE3']
        AfaLoggerFunc.tradeDebug('ztcbka_dict[NOTE3] = ' + str(to_dict['NOTE3']))
    else:
        AfaLoggerFunc.tradeDebug("trcbka_dict['NOTE3']不存在")

    if from_dict.has_key('NOTE4'):
        to_dict['NOTE4'] = from_dict['NOTE4']
        AfaLoggerFunc.tradeDebug('ztcbka_dict[NOTE4] = ' + str(to_dict['NOTE4']))
    else:
        AfaLoggerFunc.tradeDebug("trcbka_dict['NOTE4']不存在")

    return True


import logging
from xmlrpc.server import SimpleXMLRPCServer
from util import estimate_pi

def local_estimate_pi(n, *args):
    logging.debug(f'Call received: estimate_pi({n!r})')
    return estimate_pi(n)

if __name__ == "__main__":
    logging.basicConfig(level=logging.DEBUG)
    server = SimpleXMLRPCServer(('localhost', 9000), logRequests=True)
    # Register the function we are serving
    server.register_function(local_estimate_pi, 'estimate_pi')
    try:
        print("Use Control-C to exit")
        # Start serving our functions
        server.serve_forever()
    except KeyboardInterrupt:
        print("Exiting")

31380 Intelligent Systems/python projects/Week2D1/comm_exercise/rpc/rpc_sync_pi_slave.py


import argparse
import os

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("--datasets",type=str)
    parser.add_argument("--mode",type=str)
    parser.add_argument("--mergeversion",type=str)
    parser.add_argument("--desversion",type=str)

    args = parser.parse_args()
    return args

def merge(args):
    index = 0
    all_edgelist = []
    datasets = args.datasets.split("+")
    for dataset in datasets:
        with open("data/"+dataset+"/"+dataset+".edgelist",'r') as f:
            L = f.readlines()
            num_node = int(L[0].strip())
            L = L[1:]
            edge_list = [[int(x)+index for x in e.strip().split()] for e in L]
        all_edgelist.extend(edge_list)
        index += num_node

    if not os.path.exists("data/merge"+args.mergeversion):
        os.mkdir("data/merge"+args.mergeversion)
    with open("data/merge"+args.mergeversion+"/merge"+args.mergeversion+".edgelist",'w') as f:
        f.write(str(index)+"\n")
        for i in range(len(all_edgelist)):
            f.write(str(all_edgelist[i][0])+" "+str(all_edgelist[i][1])+"\n")
    print("all nodes {} all edges {}".format(index,len(all_edgelist)))

def split(args):
    indexlist = []
    index = 0
    datasets = args.datasets.split("+")
    splitedvec= ['\n']*len(datasets)
    for dataset in datasets:
        with open("data/"+dataset+"/"+dataset+".edgelist",'r') as f:
            L = f.readlines()
            num_node = int(L[0].strip())
        indexlist.append(index)
        index += num_node

    with open("emb/merge"+args.mergeversion+"/merge.struc2vec."+args.mergeversion+args.desversion+".embs32",'r') as f:
        L=f.readlines()[1:]
        for i in range(len(L)):
            row = L[i].strip().split()
            id = int(row[0])
            vec = " ".join(row[1:])
            datasetid,offset = 0,0
            for d,j in enumerate(indexlist):
                if j<=id:
                    datasetid,offset = d,j
                else:
                    break
            true_id = id - offset
            true_row = str(true_id)+' '+vec+"\n"
            splitedvec[datasetid]+=true_row

    for id,name in enumerate(datasets):
        if not os.path.exists("emb/"+name):
            os.mkdir("emb/"+name)
        with open(os.path.join("emb",name,name+".struc2vec."+str(id)+args.mergeversion+args.desversion+".embs32"),'w') as f:
            f.write(splitedvec[id])

if __name__=="__main__":
    args = parse_args()
    if args.mode == "merge":
        merge(args)
    elif args.mode == "split":
        split(args)


import sys
from mainInterface import *
from PyQt5 import QtCore, QtGui, QtWidgets
import xml.etree.cElementTree as ET
import xml.dom.minidom as minidom

def showDialog():
    text, ok = QtWidgets.QInputDialog.getText(None, 'Input',
                                                'Enter:',)
    if ok:
        return text
    else:
        return None

class mark_up_rect():

    pos = QtCore.QPoint()
    width = 0
    height = 0
    text = "test"
    r = 0
    g = 0
    b = 0
    a = 0

class MyGraphicsScene(QtWidgets.QGraphicsScene):

    rects = []
    added_rects = []
    added_text = []

    def __init__(self, *args, **kwargs):
        self.state = 0
        self.closeCheck = False
        super().__init__(*args, **kwargs)

    def mousePressEvent(self, event):

        if self.state == 4:
            print('haha')
            print(self.index)
            print(self.changing_rect)
            self.rects.remove(self.changing_rect)
            i = self.changing_rect
            i.width += (i.pos.x() - event.scenePos().x())
            i.height += (i.pos.y() - event.scenePos().y())
            i.pos = event.scenePos()
            print('pop0')
            self.removeItem(self.added_rects[self.index])
            self.added_rects.remove(self.added_rects[self.index])
            print('pop0.5')
            self.removeItem(self.added_text[self.index])
            self.added_text.remove(self.added_text[self.index])
            print('pop1')
            color = QtGui.QColor()
            color.setRgb(i.r,i.g,i.b,i.a)
            pen = QtGui.QPen(color)
            rect = self.addRect(i.pos.x(), i.pos.y(),
                                              i.width, i.height, pen)
            print('pop2')

            text = self.addText(i.text)
            print('pop1')
            self.removeItem(text)
            print('pop1')
            text.setDefaultTextColor(color)
            print('pop1')
            text.setPos(i.pos.x() + i.width + 5,i.pos.y())
            self.addItem(text)
            print('pop1')
            self.added_rects.append(rect)
            self.added_text.append(text)
            self.rects.append(i)
            print('pop3')

            self.state = 6

        if self.state == 5:
            self.rects.remove(self.changing_rect)
            i = self.changing_rect
            i.width = abs(i.pos.x() - event.scenePos().x())
            i.height = abs(i.pos.y() - event.scenePos().y())
            print('pop0')
            self.removeItem(self.added_rects[self.index])
            self.added_rects.remove(self.added_rects[self.index])
            print('pop0.5')
            self.removeItem(self.added_text[self.index])
            self.added_text.remove(self.added_text[self.index])
            print('pop1')
            color = QtGui.QColor()
            color.setRgb(i.r,i.g,i.b,i.a)
            pen = QtGui.QPen(color)
            rect = self.addRect(i.pos.x(), i.pos.y(),
                                              i.width, i.height, pen)
            print('pop2')

            text = self.addText(i.text)
            print('pop1')
            self.removeItem(text)
            print('pop1')
            text.setDefaultTextColor(color)
            print('pop1')
            text.setPos(i.pos.x() + i.width + 5,i.pos.y())
            self.addItem(text)
            print('pop1')
            self.added_rects.append(rect)
            self.added_text.append(text)
            self.rects.append(i)
            print('pop3')
            self.state = 6

        if self.state == 1:
            if event.scenePos().x() < self.pos.x():
                self.x = event.scenePos().x()
            else:
                self.x = self.pos.x()
            if event.scenePos().y() < self.pos.y():
                self.y = event.scenePos().y()
            else:
                self.y = self.pos.y()
            self._width = abs(event.scenePos().x() - self.pos.x())
            self._height = abs(event.scenePos().y() - self.pos.y())
            self.state = 3
            rect = self.addRect(self.x,  self.y,
                             self._width, self._height, self.pen)
            self.text = showDialog()
            if self.text:
                add_text = self.addText(self.text)
                self.removeItem(add_text)
                add_text.setDefaultTextColor(self.pen.color())
                add_text.setPos(self.x + self._width + 5,self.y)
                self.addItem(add_text)
                self.added_text.append(add_text)
            self.added_rects.append(rect)
            print(self._width, self._height)

        if self.state == 0:
            self.closeCheck = True
            self.pos = event.scenePos()
            self.changing_rect = mark_up_rect()
            self.index = 0
            print('here')
            for rect in self.rects:
                print('here1')
                if (abs(self.pos.x() - rect.pos.x()) < 5) and (abs(self.pos.y() - rect.pos.y()) < 5):
                    print('here2')
                    self.changing_rect = rect
                    self.state = 4
                    break;
                else:
                    print('here3')
                    self.index += 1
            print(self.state)
            if self.state == 0:
                self.index = 0
                for rect in self.rects:
                    if (abs(self.pos.x() - rect.width - rect.pos.x()) < 5) and (abs(self.pos.y() - rect.height - rect.pos.y()) < 5):
                        self.changing_rect = rect
                        self.state = 5
                        break;
                    else:
                        self.index += 1
            print(self.state)
            if self.state == 0:
                self.state = 1

        if self.state == 3:
            print('1')
            self.pos = QtCore.QPoint(self.x, self.y)
            print('2')
            mark_up = mark_up_rect()
            print('2')
            mark_up.pos = self.pos
            mark_up.width = self._width
            mark_up.height = self._height
            color = self.pen.color()
            mark_up.r,mark_up.g,mark_up.b,mark_up.a = color.getRgb()

            #print('#2.5')

            mark_up.text = self.text
            #print(mark_up.text)

            print('2')
            #print(mark_up.pos, mark_up._width, mark_up._height)
            self.rects.append(mark_up)
            print('3')
            for i in self.rects:
                print (i.pos, i.width, i.height)
            self.state = 0

        if self.state == 6:
            self.state = 0

class Window(QtWidgets.QMainWindow):
    def __init__(self, parent=None):
        QtWidgets.QWidget.__init__(self, parent)
        self.ui = Ui_MainWindow()
        self.ui.setupUi(self)
        self.closeCheck = False
        self.ui.scene = MyGraphicsScene()
        self.ui.scene.pen = QtGui.QPen()

        self.setGeometry(50, 50, 550, 420)
        self.setWindowTitle('Interface')
        self.setWindowIcon(QtGui.QIcon('network.png'))

        self.ui.actionLoad.triggered.connect(self.openFile)
        self.ui.actionSave.triggered.connect(self.saveFile)
        self.ui.actionChoose_color.triggered.connect(self.colorDialog)
        #self.ui.label.setText("")

    def colorDialog(self):
        col = QtWidgets.QColorDialog.getColor()
        self.ui.scene.pen = QtGui.QPen(col)
        print(self.ui.scene.pen)

    def openFile(self):
        options = QtWidgets.QFileDialog.Options()
        self.file,_=QtWidgets.QFileDialog.getOpenFileName(None,'OpenFile','',
                                                           "Image file(*.png *.jpg)",
                                                           options=options)
        if self.file:
            self.image = QtGui.QPixmap(self.file)
            #self.ui.label.setPixmap(self.image)
            self.resize(self.image.size())

            self.ui.scene.addPixmap(self.image)
            self.ui.graphicsView.setScene(self.ui.scene)
            #self.ui.graphicsView.resize(self.image.width() + 2, self.image.height() + 2)

            #открытие xml
            self.xml_name = self.file[:len(self.file)-3] + 'xml'
            print(self.xml_name)
            xml = open(self.xml_name, 'a')
            xml.close()
            xml = open(self.xml_name, 'r')
            print('#1')
            if xml:
                print('#1')
                read = xml.read()
                print('#2')
                if len(read) > 1:
                    print('#3')
                    xml.close()
                    xml = minidom.parse(self.xml_name)
                    collection = xml.documentElement
                    squares = collection.getElementsByTagName('square')
                    for square in squares:
                        print('c')
                        mark_up = mark_up_rect()
                        x = int(square.getAttribute('x'))
                        y = int(square.getAttribute('y'))
                        pos = QtCore.QPoint(x, y)
                        mark_up.pos = pos
                        mark_up.width = float(square.getAttribute('width'))
                        mark_up.height = float(square.getAttribute('height'))
                        mark_up.r = int(square.getAttribute('r'))
                        mark_up.g = int(square.getAttribute('g'))
                        mark_up.b = int(square.getAttribute('b'))
                        mark_up.a = int(square.getAttribute('a'))
                        if len(square.childNodes) != 0:
                            mark_up.text = square.childNodes[0].data
                        #print(square.childNodes)
                        self.ui.scene.rects.append(mark_up)
                    for i in self.ui.scene.rects:
                        print(self.ui.scene.pen)
                        color = QtGui.QColor()
                        print(color)
                        color.setRgb(i.r,i.g,i.b,i.a)
                        pen = QtGui.QPen(color)
                        rect = self.ui.scene.addRect(i.pos.x(), i.pos.y(),
                                              i.width, i.height, pen)
                        text = self.ui.scene.addText(i.text)
                        self.ui.scene.removeItem(text)
                        text.setDefaultTextColor(color)
                        text.setPos(i.pos.x() + i.width + 5,i.pos.y())
                        self.ui.scene.addItem(text)
                        self.ui.scene.added_rects.append(rect)
                        self.ui.scene.added_text.append(text)

    def saveFile(self):
        for i in self.ui.scene.added_rects:
            self.ui.scene.removeItem(i)
        for i in self.ui.scene.added_text:
            self.ui.scene.removeItem(i)
        options = QtWidgets.QFileDialog.Options()
        self.fileName, _ = QtWidgets.QFileDialog.getSaveFileName(self, "Save to file",
                                              "", "Image file(*.png *.jpg)",
                                              options=options)
        if self.fileName:
            self.image = QtGui.QImage(self.ui.scene.sceneRect().size().toSize(),
                                      QtGui.QImage.Format_ARGB32)
            self.image.fill(QtGui.QColor.fromRgb(255, 255, 255, 255));
            painter = QtGui.QPainter(self.image)
            self.ui.scene.render(painter)
            if self.image.save(self.fileName):
                self.ui.statusbar.showMessage('Saved to %s' %self.fileName)
                self.ui.scene.closeCheck = False
            else:
                self.ui.statusbar.showMessage('!!! File is NOT saved !!!')

            self.xml_name = self.fileName[:len(self.fileName)-3] + 'xml'
            print(self.xml_name)
            root = ET.Element('root')

            for i in self.ui.scene.rects:
                print('in cycle')
                color = QtGui.QColor()
                color.setRgb(i.r,i.g,i.b,i.a)
                pen = QtGui.QPen(color)
                rect = self.ui.scene.addRect(i.pos.x(), i.pos.y(),
                                              i.width, i.height, pen)
                text = self.ui.scene.addText(i.text)
                self.ui.scene.removeItem(text)
                text.setDefaultTextColor(color)
                text.setPos(i.pos.x() + i.width + 5,i.pos.y())
                self.ui.scene.addItem(text)
                self.ui.scene.added_text.append(text)
                self.ui.scene.added_rects.append(rect)
                ET.SubElement(root, 'square',
                              x=str(i.pos.x()),y=str(i.pos.y()),
                              width = str(i.width),
                              height = str(i.height),
                              r = str(i.r),
                              g = str(i.g),
                              b = str(i.b),
                              a = str(i.a)).text = i.text
            tree = ET.ElementTree(root)
            print(tree)
            tree.write(self.xml_name)
            print('written')

    def closeEvent(self, event):
        if self.ui.scene.closeCheck:
            result = QtWidgets.QMessageBox.question(self, "Confirm action",
                                                    "Save before quit?",
                                                    QtWidgets.QMessageBox.Yes |
                                                    QtWidgets.QMessageBox.No |
                                                    QtWidgets.QMessageBox.Cancel,
                                                    QtWidgets.QMessageBox.Cancel)
            if result == QtWidgets.QMessageBox.Yes:
                self.saveFile()
                event.accept()
            elif result == QtWidgets.QMessageBox.No:
                event.accept()
            else:
                event.ignore()
        else:
            event.accept()

if __name__ == "__main__":
    app = QtWidgets.QApplication(sys.argv)
    myapp = Window()
    myapp.show()
    sys.exit(app.exec_())


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models
import django.utils.timezone
import model_utils.fields
import ckeditor_uploader.fields

class Migration(migrations.Migration):

    dependencies = [
        ('aristotle_mdr', '0012_better_workflows'),
    ]

    operations = [
        migrations.CreateModel(
            name='AdministrationMode',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, verbose_name='created', editable=False)),
                ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, verbose_name='modified', editable=False)),
                ('name', models.TextField(help_text='The primary name used for human identification purposes.')),
                ('definition', ckeditor_uploader.fields.RichTextUploadingField(help_text='Representation of a concept by a descriptive statement which serves to differentiate it from related concepts', verbose_name='definition')),
            ],
            options={
                'abstract': False,
            },
        ),
        migrations.CreateModel(
            name='Question',
            fields=[
                ('_concept_ptr', models.OneToOneField(parent_link=True, auto_created=True, primary_key=True, serialize=False, to='aristotle_mdr._concept')),
                ('short_name', models.CharField(max_length=100, blank=True)),
                ('version', models.CharField(max_length=20, blank=True)),
                ('synonyms', models.CharField(max_length=200, blank=True)),
                ('references', ckeditor_uploader.fields.RichTextUploadingField(blank=True)),
                ('origin_URI', models.URLField(help_text='If imported, the original location of the item', blank=True)),
                ('comments', ckeditor_uploader.fields.RichTextUploadingField(help_text='Descriptive comments about the metadata item.', blank=True)),
                ('submitting_organisation', models.CharField(max_length=256, blank=True)),
                ('responsible_organisation', models.CharField(max_length=256, blank=True)),
                ('question_text', ckeditor_uploader.fields.RichTextUploadingField(blank=True)),
                ('instruction_text', ckeditor_uploader.fields.RichTextUploadingField(blank=True)),
                ('estimated_seconds_response_time', models.PositiveIntegerField(help_text='he estimated amount of time required to answer a question expressed in seconds.', null=True, blank=True)),
                ('collected_data_element', models.ForeignKey(related_name='questions', blank=True, to='aristotle_mdr.DataElement', null=True)),
                ('response_domain', models.ManyToManyField(to='aristotle_mdr.ValueDomain', null=True, verbose_name='Value Domain', blank=True)),
                ('superseded_by', models.ForeignKey(related_name='supersedes', blank=True, to='mallard_qr.Question', null=True)),
            ],
            options={
                'abstract': False,
            },
            bases=('aristotle_mdr._concept',),
        ),
        migrations.CreateModel(
            name='ResponseDomain',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('maximum_occurances', models.PositiveIntegerField(default=1, help_text='The maximum number of times a response can be included in a question')),
                ('minimum_occurances', models.PositiveIntegerField(default=1, help_text='The minimum number of times a response can be included in a question')),
                ('blank_is_missing_value', models.BooleanField(default=False, help_text='When value is true a blank or empty variable content should be treated as a missing value.')),
                ('question', models.ForeignKey(to='mallard_qr.Question')),
                ('value_domain', models.ForeignKey(to='aristotle_mdr.ValueDomain')),
            ],
            options={
                'abstract': False,
            },
        ),
    ]


import requests
import geopandas as gpd
import pandas as pd
import json

def API_to_df(dataset_id,format="json"):
    response_API = requests.get(f"https://data.economie.gouv.fr/api/v2/catalog/datasets/{dataset_id}/exports/{format}")
    data = response_API.json()
    if format == "geojson":
        with open(f"data/{dataset_id}.{format}","w") as outfile:
            json.dump(data, outfile)
        gdf = gpd.read_file(f"data/{dataset_id}.{format}")
        return gdf
    else:
        with open(f"data/{dataset_id}.json","w") as outfile:
            json.dump(data, outfile)
        df = pd.read_json(f"data/{dataset_id}.json")
        return df

if __name__ == "__main__":
    print(API_to_df("coordonnees-des-structures-dgfip","geojson").head())


import datetime
import functools
import json
import warnings
from decimal import Decimal
from enum import Enum, IntEnum
from typing import TYPE_CHECKING, Any, Callable, Optional, Type, TypeVar, Union
from uuid import UUID, uuid4

from pypika import functions
from pypika.enums import SqlTypes
from pypika.terms import Term

from tortoise import timezone
from tortoise.exceptions import ConfigurationError, FieldError
from tortoise.fields.base import Field
from tortoise.timezone import get_default_timezone, get_timezone, get_use_tz, localtime
from tortoise.validators import MaxLengthValidator

try:
    from ciso8601 import parse_datetime
except ImportError:  # pragma: nocoverage
    from iso8601 import parse_date

    parse_datetime = functools.partial(parse_date, default_timezone=None)

if TYPE_CHECKING:  # pragma: nocoverage
    from tortoise.models import Model

__all__ = (
    "BigIntField",
    "BinaryField",
    "BooleanField",
    "CharEnumField",
    "CharField",
    "DateField",
    "DatetimeField",
    "DecimalField",
    "FloatField",
    "IntEnumField",
    "IntField",
    "JSONField",
    "SmallIntField",
    "TextField",
    "TimeDeltaField",
    "UUIDField",
)

# Doing this we can replace json dumps/loads with different implementations
JsonDumpsFunc = Callable[[Any], str]
JsonLoadsFunc = Callable[[Union[str, bytes]], Any]
JSON_DUMPS: JsonDumpsFunc = functools.partial(json.dumps, separators=(",", ":"))
JSON_LOADS: JsonLoadsFunc = json.loads

try:
    # Use orjson as an optional accelerator
    import orjson

    JSON_DUMPS = lambda x: orjson.dumps(x).decode()  # noqa: E731
    JSON_LOADS = orjson.loads
except ImportError:  # pragma: nocoverage
    pass

class IntField(Field[int], int):
    """
    Integer field. (32-bit signed)

    ``pk`` (bool):
        True if field is Primary Key.
    """

    SQL_TYPE = "INT"
    allows_generated = True

    def __init__(self, pk: bool = False, **kwargs: Any) -> None:
        if pk:
            kwargs["generated"] = bool(kwargs.get("generated", True))
        super().__init__(pk=pk, **kwargs)

    @property
    def constraints(self) -> dict:
        return {
            "ge": 1 if self.generated or self.reference else -2147483648,
            "le": 2147483647,
        }

    class _db_postgres:
        GENERATED_SQL = "SERIAL NOT NULL PRIMARY KEY"

    class _db_sqlite:
        GENERATED_SQL = "INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL"

    class _db_mysql:
        GENERATED_SQL = "INT NOT NULL PRIMARY KEY AUTO_INCREMENT"

    class _db_mssql:
        GENERATED_SQL = "INT IDENTITY(1,1) NOT NULL PRIMARY KEY"

    class _db_oracle:
        GENERATED_SQL = "INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY NOT NULL"

class BigIntField(Field[int], int):
    """
    Big integer field. (64-bit signed)

    ``pk`` (bool):
        True if field is Primary Key.
    """

    SQL_TYPE = "BIGINT"
    allows_generated = True

    def __init__(self, pk: bool = False, **kwargs: Any) -> None:
        if pk:
            kwargs["generated"] = bool(kwargs.get("generated", True))
        super().__init__(pk=pk, **kwargs)

    @property
    def constraints(self) -> dict:
        return {
            "ge": 1 if self.generated or self.reference else -9223372036854775808,
            "le": 9223372036854775807,
        }

    class _db_postgres:
        GENERATED_SQL = "BIGSERIAL NOT NULL PRIMARY KEY"

    class _db_sqlite:
        GENERATED_SQL = "INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL"

    class _db_mysql:
        GENERATED_SQL = "BIGINT NOT NULL PRIMARY KEY AUTO_INCREMENT"

    class _db_mssql:
        GENERATED_SQL = "BIGINT IDENTITY(1,1) NOT NULL PRIMARY KEY"

    class _db_oracle:
        SQL_TYPE = "INT"
        GENERATED_SQL = "INT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY NOT NULL"

class SmallIntField(Field[int], int):
    """
    Small integer field. (16-bit signed)

    ``pk`` (bool):
        True if field is Primary Key.
    """

    SQL_TYPE = "SMALLINT"
    allows_generated = True

    def __init__(self, pk: bool = False, **kwargs: Any) -> None:
        if pk:
            kwargs["generated"] = bool(kwargs.get("generated", True))
        super().__init__(pk=pk, **kwargs)

    @property
    def constraints(self) -> dict:
        return {
            "ge": 1 if self.generated or self.reference else -32768,
            "le": 32767,
        }

    class _db_postgres:
        GENERATED_SQL = "SMALLSERIAL NOT NULL PRIMARY KEY"

    class _db_sqlite:
        GENERATED_SQL = "INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL"

    class _db_mysql:
        GENERATED_SQL = "SMALLINT NOT NULL PRIMARY KEY AUTO_INCREMENT"

    class _db_mssql:
        GENERATED_SQL = "SMALLINT IDENTITY(1,1) NOT NULL PRIMARY KEY"

    class _db_oracle:
        GENERATED_SQL = "SMALLINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY NOT NULL"

class CharField(Field[str]):
    """
    Character field.

    You must provide the following:

    ``max_length`` (int):
        Maximum length of the field in characters.
    """

    field_type = str

    def __init__(self, max_length: int, **kwargs: Any) -> None:
        if int(max_length) < 1:
            raise ConfigurationError("'max_length' must be >= 1")
        self.max_length = int(max_length)
        super().__init__(**kwargs)
        self.validators.append(MaxLengthValidator(self.max_length))

    @property
    def constraints(self) -> dict:
        return {
            "max_length": self.max_length,
        }

    @property
    def SQL_TYPE(self) -> str:  # type: ignore
        return f"VARCHAR({self.max_length})"

    class _db_oracle:
        def __init__(self, field: "CharField") -> None:
            self.field = field

        @property
        def SQL_TYPE(self) -> str:
            return f"NVARCHAR2({self.field.max_length})"

class TextField(Field[str], str):  # type: ignore
    """
    Large Text field.
    """

    indexable = False
    SQL_TYPE = "TEXT"

    def __init__(
        self, pk: bool = False, unique: bool = False, index: bool = False, **kwargs: Any
    ) -> None:
        if pk:
            warnings.warn(
                "TextField as a PrimaryKey is Deprecated, use CharField instead",
                DeprecationWarning,
                stacklevel=2,
            )
        if unique:
            raise ConfigurationError(
                "TextField doesn't support unique indexes, consider CharField or another strategy"
            )
        if index:
            raise ConfigurationError("TextField can't be indexed, consider CharField")

        super().__init__(pk=pk, **kwargs)

    class _db_mysql:
        SQL_TYPE = "LONGTEXT"

    class _db_mssql:
        SQL_TYPE = "NVARCHAR(MAX)"

    class _db_oracle:
        SQL_TYPE = "NCLOB"

class BooleanField(Field[bool]):
    """
    Boolean field.
    """

    # Bool is not subclassable, so we specify type here
    field_type = bool
    SQL_TYPE = "BOOL"

    class _db_sqlite:
        SQL_TYPE = "INT"

    class _db_mssql:
        SQL_TYPE = "BIT"

    class _db_oracle:
        SQL_TYPE = "NUMBER(1)"

class DecimalField(Field[Decimal], Decimal):
    """
    Accurate decimal field.

    You must provide the following:

    ``max_digits`` (int):
        Max digits of significance of the decimal field.
    ``decimal_places`` (int):
        How many of those significant digits is after the decimal point.
    """

    skip_to_python_if_native = True

    def __init__(self, max_digits: int, decimal_places: int, **kwargs: Any) -> None:
        if int(max_digits) < 1:
            raise ConfigurationError("'max_digits' must be >= 1")
        if int(decimal_places) < 0:
            raise ConfigurationError("'decimal_places' must be >= 0")
        super().__init__(**kwargs)
        self.max_digits = max_digits
        self.decimal_places = decimal_places
        self.quant = Decimal("1" if decimal_places == 0 else f"1.{('0' * decimal_places)}")

    def to_python_value(self, value: Any) -> Optional[Decimal]:
        if value is None:
            value = None
        else:
            value = Decimal(value).quantize(self.quant).normalize()
        self.validate(value)
        return value

    @property
    def SQL_TYPE(self) -> str:  # type: ignore
        return f"DECIMAL({self.max_digits},{self.decimal_places})"

    class _db_sqlite:
        SQL_TYPE = "VARCHAR(40)"

        def function_cast(self, term: Term) -> Term:
            return functions.Cast(term, SqlTypes.NUMERIC)

class DatetimeField(Field[datetime.datetime], datetime.datetime):
    """
    Datetime field.

    ``auto_now`` and ``auto_now_add`` is exclusive.
    You can opt to set neither or only ONE of them.

    ``auto_now`` (bool):
        Always set to ``datetime.utcnow()`` on save.
    ``auto_now_add`` (bool):
        Set to ``datetime.utcnow()`` on first save only.
    """

    SQL_TYPE = "TIMESTAMP"

    class _db_mysql:
        SQL_TYPE = "DATETIME(6)"

    class _db_postgres:
        SQL_TYPE = "TIMESTAMPTZ"

    class _db_mssql:
        SQL_TYPE = "DATETIME2"

    class _db_oracle:
        SQL_TYPE = "TIMESTAMP WITH TIME ZONE"

    def __init__(self, auto_now: bool = False, auto_now_add: bool = False, **kwargs: Any) -> None:
        if auto_now_add and auto_now:
            raise ConfigurationError("You can choose only 'auto_now' or 'auto_now_add'")
        super().__init__(**kwargs)
        self.auto_now = auto_now
        self.auto_now_add = auto_now | auto_now_add

    def to_python_value(self, value: Any) -> Optional[datetime.datetime]:
        if value is None:
            value = None
        else:
            if isinstance(value, datetime.datetime):
                value = value
            elif isinstance(value, int):
                value = datetime.datetime.fromtimestamp(value)
            else:
                value = parse_datetime(value)
            if timezone.is_naive(value):
                value = timezone.make_aware(value, get_timezone())
            else:
                value = localtime(value)
        self.validate(value)
        return value

    def to_db_value(
        self, value: Optional[datetime.datetime], instance: "Union[Type[Model], Model]"
    ) -> Optional[datetime.datetime]:
        # Only do this if it is a Model instance, not class. Test for guaranteed instance var
        if hasattr(instance, "_saved_in_db") and (
            self.auto_now
            or (self.auto_now_add and getattr(instance, self.model_field_name) is None)
        ):
            value = timezone.now()
            setattr(instance, self.model_field_name, value)
            return value
        if value is not None:
            if get_use_tz():
                if timezone.is_naive(value):
                    warnings.warn(
                        "DateTimeField %s received a naive datetime (%s)"
                        " while time zone support is active." % (self.model_field_name, value),
                        RuntimeWarning,
                    )
                    value = timezone.make_aware(value, "UTC")
        self.validate(value)
        return value

    @property
    def constraints(self) -> dict:
        data = {}
        if self.auto_now_add:
            data["readOnly"] = True
        return data

    def describe(self, serializable: bool) -> dict:
        desc = super().describe(serializable)
        desc["auto_now_add"] = self.auto_now_add
        desc["auto_now"] = self.auto_now
        return desc

class DateField(Field[datetime.date], datetime.date):
    """
    Date field.
    """

    skip_to_python_if_native = True
    SQL_TYPE = "DATE"

    def to_python_value(self, value: Any) -> Optional[datetime.date]:
        if value is not None and not isinstance(value, datetime.date):
            value = parse_datetime(value).date()
        self.validate(value)
        return value

    def to_db_value(
        self, value: Optional[Union[datetime.date, str]], instance: "Union[Type[Model], Model]"
    ) -> Optional[datetime.date]:
        if value is not None and not isinstance(value, datetime.date):
            value = parse_datetime(value).date()
        self.validate(value)
        return value

class TimeField(Field[datetime.time], datetime.time):
    """
    Time field.
    """

    skip_to_python_if_native = True
    SQL_TYPE = "TIME"

    class _db_oracle:
        SQL_TYPE = "NVARCHAR2(8)"

    def __init__(self, auto_now: bool = False, auto_now_add: bool = False, **kwargs: Any) -> None:
        if auto_now_add and auto_now:
            raise ConfigurationError("You can choose only 'auto_now' or 'auto_now_add'")
        super().__init__(**kwargs)
        self.auto_now = auto_now
        self.auto_now_add = auto_now | auto_now_add

    def to_python_value(self, value: Any) -> Optional[Union[datetime.time, datetime.timedelta]]:
        if value is not None:
            if isinstance(value, str):
                value = datetime.time.fromisoformat(value)
            if isinstance(value, datetime.timedelta):
                return value
            if timezone.is_naive(value):
                value = value.replace(tzinfo=get_default_timezone())
        self.validate(value)
        return value

    def to_db_value(
        self,
        value: Optional[Union[datetime.time, datetime.timedelta]],
        instance: "Union[Type[Model], Model]",
    ) -> Optional[Union[datetime.time, datetime.timedelta]]:
        # Only do this if it is a Model instance, not class. Test for guaranteed instance var
        if hasattr(instance, "_saved_in_db") and (
            self.auto_now
            or (self.auto_now_add and getattr(instance, self.model_field_name) is None)
        ):
            now = timezone.now().time()
            setattr(instance, self.model_field_name, now)
            return now
        if value is not None:
            if isinstance(value, datetime.timedelta):
                return value
            if get_use_tz():
                if timezone.is_naive(value):
                    warnings.warn(
                        "TimeField %s received a naive time (%s)"
                        " while time zone support is active." % (self.model_field_name, value),
                        RuntimeWarning,
                    )
                    value = value.replace(tzinfo=get_default_timezone())
        self.validate(value)
        return value

    class _db_mysql:
        SQL_TYPE = "TIME(6)"

    class _db_postgres:
        SQL_TYPE = "TIMETZ"

class TimeDeltaField(Field[datetime.timedelta]):
    """
    A field for storing time differences.
    """

    SQL_TYPE = "BIGINT"

    class _db_oracle:
        SQL_TYPE = "NUMBER(19)"

    def to_python_value(self, value: Any) -> Optional[datetime.timedelta]:
        self.validate(value)

        if value is None or isinstance(value, datetime.timedelta):
            return value
        return datetime.timedelta(microseconds=value)

    def to_db_value(
        self, value: Optional[datetime.timedelta], instance: "Union[Type[Model], Model]"
    ) -> Optional[int]:
        self.validate(value)

        if value is None:
            return None
        return (value.days * 86400000000) + (value.seconds * 1000000) + value.microseconds

class FloatField(Field[float], float):
    """
    Float (double) field.
    """

    SQL_TYPE = "DOUBLE PRECISION"

    class _db_sqlite:
        SQL_TYPE = "REAL"

    class _db_mysql:
        SQL_TYPE = "DOUBLE"

class JSONField(Field[Union[dict, list]], dict, list):  # type: ignore
    """
    JSON field.

    This field can store dictionaries or lists of any JSON-compliant structure.

    You can specify your own custom JSON encoder/decoder, leaving at the default should work well.
    If you have ``orjson`` installed, we default to using that,
    else the default ``json`` module will be used.

    ``encoder``:
        The custom JSON encoder.
    ``decoder``:
        The custom JSON decoder.

    """

    SQL_TYPE = "JSON"
    indexable = False

    class _db_postgres:
        SQL_TYPE = "JSONB"

    class _db_mssql:
        SQL_TYPE = "NVARCHAR(MAX)"

    class _db_oracle:
        SQL_TYPE = "NCLOB"

    def __init__(
        self,
        encoder: JsonDumpsFunc = JSON_DUMPS,
        decoder: JsonLoadsFunc = JSON_LOADS,
        **kwargs: Any,
    ) -> None:
        super().__init__(**kwargs)
        self.encoder = encoder
        self.decoder = decoder

    def to_db_value(
        self, value: Optional[Union[dict, list, str]], instance: "Union[Type[Model], Model]"
    ) -> Optional[str]:
        self.validate(value)

        if isinstance(value, (str, bytes)):
            try:
                self.decoder(value)
            except Exception:
                raise FieldError(f"Value {value} is invalid json value.")
            return value
        return None if value is None else self.encoder(value)

    def to_python_value(
        self, value: Optional[Union[str, bytes, dict, list]]
    ) -> Optional[Union[dict, list]]:
        if isinstance(value, (str, bytes)):
            try:
                return self.decoder(value)
            except Exception:
                raise FieldError(
                    f"Value {value if isinstance(value, str) else value.decode()} is invalid json value."
                )

        self.validate(value)
        return value

class UUIDField(Field[UUID], UUID):
    """
    UUID Field

    This field can store uuid value.

    If used as a primary key, it will auto-generate a UUID4 by default.
    """

    SQL_TYPE = "CHAR(36)"

    class _db_postgres:
        SQL_TYPE = "UUID"

    def __init__(self, **kwargs: Any) -> None:
        if kwargs.get("pk", False) and "default" not in kwargs:
            kwargs["default"] = uuid4
        super().__init__(**kwargs)

    def to_db_value(self, value: Any, instance: "Union[Type[Model], Model]") -> Optional[str]:
        return value and str(value)

    def to_python_value(self, value: Any) -> Optional[UUID]:
        if value is None or isinstance(value, UUID):
            return value
        return UUID(value)

class BinaryField(Field[bytes], bytes):  # type: ignore
    """
    Binary field.

    This is for storing ``bytes`` objects.
    Note that filter or queryset-update operations are not supported.
    """

    indexable = False
    SQL_TYPE = "BLOB"

    class _db_postgres:
        SQL_TYPE = "BYTEA"

    class _db_mysql:
        SQL_TYPE = "LONGBLOB"

    class _db_mssql:
        SQL_TYPE = "VARBINARY(MAX)"

class IntEnumFieldInstance(SmallIntField):
    def __init__(
        self,
        enum_type: Type[IntEnum],
        description: Optional[str] = None,
        generated: bool = False,
        **kwargs: Any,
    ) -> None:
        # Validate values
        minimum = 1 if generated else -32768
        for item in enum_type:
            try:
                value = int(item.value)
            except ValueError:
                raise ConfigurationError("IntEnumField only supports integer enums!")
            if not minimum <= value < 32768:
                raise ConfigurationError(
                    "The valid range of IntEnumField's values is {}..32767!".format(minimum)
                )

        # Automatic description for the field if not specified by the user
        if description is None:
            description = "\n".join([f"{e.name}: {int(e.value)}" for e in enum_type])[:2048]

        super().__init__(description=description, **kwargs)
        self.enum_type = enum_type

    def to_python_value(self, value: Union[int, None]) -> Union[IntEnum, None]:
        value = self.enum_type(value) if value is not None else None
        self.validate(value)
        return value

    def to_db_value(
        self, value: Union[IntEnum, None, int], instance: "Union[Type[Model], Model]"
    ) -> Union[int, None]:
        if isinstance(value, IntEnum):
            value = int(value.value)
        if isinstance(value, int):
            value = int(self.enum_type(value))
        self.validate(value)
        return value

IntEnumType = TypeVar("IntEnumType", bound=IntEnum)

def IntEnumField(
    enum_type: Type[IntEnumType],
    description: Optional[str] = None,
    **kwargs: Any,
) -> IntEnumType:
    """
    Enum Field

    A field representing an integer enumeration.

    The description of the field is set automatically if not specified to a multiline list of
    "name: value" pairs.

    **Note**: Valid int value of ``enum_type`` is acceptable.

    ``enum_type``:
        The enum class
    ``description``:
        The description of the field. It is set automatically if not specified to a multiline list
        of "name: value" pairs.

    """
    return IntEnumFieldInstance(enum_type, description, **kwargs)  # type: ignore

class CharEnumFieldInstance(CharField):
    def __init__(
        self,
        enum_type: Type[Enum],
        description: Optional[str] = None,
        max_length: int = 0,
        **kwargs: Any,
    ) -> None:
        # Automatic description for the field if not specified by the user
        if description is None:
            description = "\n".join([f"{e.name}: {str(e.value)}" for e in enum_type])[:2048]

        # Automatic CharField max_length
        if max_length == 0:
            for item in enum_type:
                item_len = len(str(item.value))
                if item_len > max_length:
                    max_length = item_len

        super().__init__(description=description, max_length=max_length, **kwargs)
        self.enum_type = enum_type

    def to_python_value(self, value: Union[str, None]) -> Union[Enum, None]:
        self.validate(value)

        return self.enum_type(value) if value is not None else None

    def to_db_value(
        self, value: Union[Enum, None, str], instance: "Union[Type[Model], Model]"
    ) -> Union[str, None]:
        self.validate(value)
        if isinstance(value, Enum):
            return str(value.value)
        if isinstance(value, str):
            return str(self.enum_type(value).value)
        return value

CharEnumType = TypeVar("CharEnumType", bound=Enum)

def CharEnumField(
    enum_type: Type[CharEnumType],
    description: Optional[str] = None,
    max_length: int = 0,
    **kwargs: Any,
) -> CharEnumType:
    """
    Char Enum Field

    A field representing a character enumeration.

    **Warning**: If ``max_length`` is not specified or equals to zero, the size of represented
    char fields is automatically detected. So if later you update the enum, you need to update your
    table schema as well.

    **Note**: Valid str value of ``enum_type`` is acceptable.

    ``enum_type``:
        The enum class
    ``description``:
        The description of the field. It is set automatically if not specified to a multiline list
        of "name: value" pairs.
    ``max_length``:
        The length of the created CharField. If it is zero it is automatically detected from
        enum_type.

    """

    return CharEnumFieldInstance(enum_type, description, max_length, **kwargs)  # type: ignore


import requests
import os
from bs4 import BeautifulSoup
import time
import random

resource_path = r'./mygonews'
if not os.path.exists(resource_path):
    os.mkdir(resource_path)
for p in range(346, 3875):
    url = 'http://www.mygonews.com/news/searchlistimgtxt?param=param&class=searchnews&page=' + str(p) + '&kw='

    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
                             '(KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36'}

    res = requests.get(url, headers=headers)
    res.encoding = 'utf-8'
    soup = BeautifulSoup(res.text.replace('
', '').replace('
', ''), 'html.parser')
    # print(soup)
    article_url = soup.select('div[class="phot litImgSize"] a')
    # time.sleep(random.uniform(3, 5))
    for i in range(len(article_url)):
        url = article_url[i]['href']
        # print(url)

        title = article_url[i]['href'].split('/')[-1]
        # print(title)

        newstime = soup.select('ul[class="list"] div[class="date"]')[0].text
        # print(time)

        res = requests.get(url, headers=headers)
        res.encoding = 'utf-8'
        text_soup = BeautifulSoup(res.text.replace('
', '').replace('
', ''), 'html.parser')
        [s.extract() for s in text_soup('strong')]
        # print(text_soup)
        article_text = text_soup.select('div[class="editbox"]')
        # print(article_text)
        news_json = {'媒體': 'MYGONEWS', '標題': title, '時間': newstime, '網址': url, '內容': article_text}
        # print(news_json)
        with open(r'%s/%s.txt' % (resource_path, newstime + title), 'w', encoding='utf-8') as w:
            w.write(str(news_json))
            w.close()
        print(title + '\n' + '完成儲存')
        print('====================接續下一篇====================')
        # break
        time.sleep(random.uniform(6, 11))
    # break


import discord
from db import db
from utils import utils
from logs.log import print
# This file contains helper functions for updating the channels

validOperands = (
    "and",
    "nand",
    "or",
    "nor",
    "xor",
    "nxor"
)

async def calculateChannels(member, mode, ctx, guild):
    if mode == "startup":
        print(
            f"Performing startup updates for channels in guild: {guild.name}")
    if mode == "member left":
        print(f"{member} left guild: {guild.name}!")

    elif mode == "member joined":
        print(f"{member} joined guild: {guild.name}!")

    elif mode == "forced update":
        print(f"Forced update commencing in guild: {guild.name}!")

    elif mode == "role changed":
        print(f"{member} updated a role in guild: {guild.name}")
    print("updating channels...")
    sumOfRoles = {}
    sumOfRoles["norole"] = 0
    for role in guild.roles:
        sumOfRoles[str(role.id)] = 0
    for member in guild.members:
        numberOfRoles = 0
        for role in member.roles:
            numberOfRoles += 1
            sumOfRoles[str(role.id)] += 1
        if numberOfRoles == 1:
            # Every member has at least one role: @everyone
            sumOfRoles["norole"] += 1

    channelRoles = db.getChannelTypes(guild.id)
    if channelRoles is not None:
        for channelId in channelRoles:
            role = channelRoles.get(channelId)
            roleNumber = sumOfRoles.get(role)
            targetChannel = None
            for channel in guild.voice_channels:
                if channel.id == channelId:
                    targetChannel = channel

            await updateChannel(targetChannel, roleNumber, guild)

    print(f"Channels updated in guild {guild}!")
    if (mode == "forced update"):
        await ctx.send("Channels updated!")

async def updateChannel(channel, roleNumber, guild):
    firstNumber = None
    words = channel.name.split()
    for i, word in enumerate(words):
        if word.isdigit():
            firstNumber = word
            words[i] = str(roleNumber)
            break
    words = " ".join(words)
    print(words)
    output = None
    if firstNumber is None:
        output = channel.name + str(roleNumber)
    else:
        output = words
    previousName = channel.name
    try:
        await channel.edit(name=output)
        print(f"channel {previousName} renamed to {channel.name} in guild {guild}")
    except discord.errors.Forbidden as e:
        print(f"discord Forbidden exception raised while trying to rename channel {previousName} in guild {guild.name}")

def cleanUpType(ctx, type):
    global validOperands
    type = str(type)
    type = type.lower()
    words = type.split()
    for i, word in enumerate(words):
        word, openingParenthesis, closingParenthesis = utils.removeParenthesis(word)
        isEveryone = False
        if word == "@everyone":
            word = ctx.guild.default_role.id
            isEveryone = True
        word = word.lower()
        isRole = False
        try:
            if isEveryone is True:
                raise Exception()
                #So we don't waste time checking if it's a role
                #I also just relized this try except blocks eats keyboard interupts, but it shouldn't run for long enough for that to really be a problem.
            maybeRole = word[3:-1]
            for role in ctx.guild.roles:
                roleId = str(role.id)
                if roleId == maybeRole:
                    isRole = True
                    break
            if isRole is True:
                word = maybeRole
        except:
            pass
        word = utils.addParenthesis(word, openingParenthesis, closingParenthesis)
        words[i] = word
    type = " ".join(words)
    return type

def roleValidityChecker(ctx, input):
    #DEPRECATED. Used to be in use when types only consisted of a role for the bot to track
    if isinstance(input, str) is False:
        return False
    words = input.split()
    if len(words) != 1:
        return False
    isRole = False
    for role in ctx.guild.roles:
        roleId = str(role.id)
        if roleId == input:
            isRole = True
            break
    if isRole:
        return True
    if (input == "norole"):
        return True
    if (input == "@everyone"):
        return True
    return False

def typeValidityChecker(ctx, type):
    if isinstance(type, str) is False:
        return False
    if utils.checkParenthesis(type) is False:
        return False

    words = type.split()
    for i, word in enumerate(words):
        word, openingParenthesis, closingParenthesis = utils.removeParenthesis(word)


def factors(n):
    lfactors = []
    for i in range(1, n+1):
        if n % i == 0:
            lfactors.append(i)
    return lfactors

def main():
    print("Factors for %s is %s " % (26,factors(26)))

if __name__ == "__main__":
    main()


from app.models import Category, Post
from app import db

# db.drop_all()
db.create_all()

anime = Category(name='Anime')
hentai = Category(name='Hentai')
naruto = Post(title='naruto', url='123', category=anime)
madoka = Post(title='madoka', url='123', category=anime)
boku = Post(title='boky no piko', url='432', category=hentai)
hentai.posts.append(boku)
anime.posts.append(naruto)
anime.posts.append(madoka)
db.session.add(anime)
db.session.add(hentai)
db.session.commit()

#
#
# print(anime.posts)
# print(hentai.posts)
# print(hentai.__dict__)