#
# (c) Copyright RIFT.io, 2013-2016, All Rights Reserved
#

import collections
import os
import tarfile
import tempfile
import threading
import uuid

import tornado
import tornado.escape
import tornado.ioloop
import tornado.web
import tornado.httputil
import requests

# disable unsigned certificate warning
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

import gi
gi.require_version('RwLaunchpadYang', '1.0')
gi.require_version('NsdYang', '1.0')
gi.require_version('VnfdYang', '1.0')

from gi.repository import (
        NsdYang,
        VnfdYang,
        )
import rift.mano.cloud

import rift.package.charm
import rift.package.checksums
import rift.package.config
import rift.package.convert
import rift.package.icon
import rift.package.package
import rift.package.script
import rift.package.store

from . import (
        export,
        extract,
        image,
        message,
        onboard,
        state,
        )

from .message import (
        MessageException,

        # Onboard Error Messages
        OnboardChecksumMismatch,
        OnboardDescriptorError,
        OnboardDescriptorExistsError,
        OnboardDescriptorFormatError,
        OnboardError,
        OnboardExtractionError,
        OnboardImageUploadError,
        OnboardMissingContentBoundary,
        OnboardMissingContentType,
        OnboardMissingTerminalBoundary,
        OnboardUnreadableHeaders,
        OnboardUnreadablePackage,
        OnboardUnsupportedMediaType,

        # Onboard Status Messages
        OnboardDescriptorOnboard,
        OnboardFailure,
        OnboardImageUpload,
        OnboardPackageUpload,
        OnboardPackageValidation,
        OnboardStart,
        OnboardSuccess,

        # Update Error Messages
        UpdateChecksumMismatch,
        UpdateDescriptorError,
        UpdateDescriptorFormatError,
        UpdateError,
        UpdateExtractionError,
        UpdateImageUploadError,
        UpdateMissingContentBoundary,
        UpdateMissingContentType,
        UpdatePackageNotFoundError,
        UpdateUnreadableHeaders,
        UpdateUnreadablePackage,
        UpdateUnsupportedMediaType,

        # Update Status Messages
        UpdateDescriptorUpdate,
        UpdateDescriptorUpdated,
        UpdatePackageUpload,
        UpdateStart,
        UpdateSuccess,
        UpdateFailure,
        )

from .tosca import ExportTosca

class HttpMessageError(Exception):
    def __init__(self, code, msg):
        self.code = code
        self.msg = msg

class RequestHandler(tornado.web.RequestHandler):
    def options(self, *args, **kargs):
        pass

    def set_default_headers(self):
        self.set_header('Access-Control-Allow-Origin', '*')
        self.set_header('Access-Control-Allow-Headers',
                        'Content-Type, Cache-Control, Accept, X-Requested-With, Authorization')
        self.set_header('Access-Control-Allow-Methods', 'POST, GET, PUT, DELETE')

@tornado.web.stream_request_body
class StreamingUploadHandler(RequestHandler):
    def initialize(self, log, loop):
        """Initialize the handler

        Arguments:
            log  - the logger that this handler should use
            loop - the tasklets ioloop

        """
        self.transaction_id = str(uuid.uuid4())

        self.loop = loop
        self.log = self.application.get_logger(self.transaction_id)

        self.package_name = None
        self.package_fp = None
        self.boundary = None

        self.log.debug('created handler (transaction_id = {})'.format(self.transaction_id))

    def msg_missing_content_type(self):
        raise NotImplementedError()

    def msg_unsupported_media_type(self):
        raise NotImplementedError()

    def msg_missing_content_boundary(self):
        raise NotImplementedError()

    def msg_start(self):
        raise NotImplementedError()

    def msg_success(self):
        raise NotImplementedError()

    def msg_failure(self):
        raise NotImplementedError()

    def msg_package_upload(self):
        raise NotImplementedError()

    @tornado.gen.coroutine
    def prepare(self):
        """Prepare the handler for a request

        The prepare function is the first part of a request transaction. It
        creates a temporary file that uploaded data can be written to.

        """
        if self.request.method != "POST":
            return

        self.log.message(self.msg_start())

        try:
            # Retrieve the content type and parameters from the request
            content_type = self.request.headers.get('content-type', None)
            if content_type is None:
                raise HttpMessageError(400, self.msg_missing_content_type())

            content_type, params = tornado.httputil._parse_header(content_type)

            if "multipart/form-data" != content_type.lower():
                raise HttpMessageError(415, self.msg_unsupported_media_type())

            if "boundary" not in params:
                raise HttpMessageError(400, self.msg_missing_content_boundary())

            self.boundary = params["boundary"]
            self.package_fp = tempfile.NamedTemporaryFile(
                    prefix="pkg-",
                    delete=False,
                    )

            self.package_name = self.package_fp.name

            self.log.debug('writing to {}'.format(self.package_name))

        except HttpMessageError as e:
            self.log.message(e.msg)
            self.log.message(self.msg_failure())

            raise tornado.web.HTTPError(e.code, e.msg.name)

        except Exception as e:
            self.log.exception(e)
            self.log.message(self.msg_failure())

    @tornado.gen.coroutine
    def data_received(self, data):
        """Write data to the current file

        Arguments:
            data - a chunk of data to write to file

        """
        self.package_fp.write(data)

    def post(self):
        """Handle a post request

        The function is called after any data associated with the body of the
        request has been received.

        """
        self.package_fp.close()
        self.log.message(self.msg_package_upload())

class UploadHandler(StreamingUploadHandler):
    """
    This handler is used to upload archives that contain VNFDs, NSDs, and PNFDs
    to the launchpad. This is a streaming handler that writes uploaded archives
    to disk without loading them all into memory.
    """

    def msg_missing_content_type(self):
        return OnboardMissingContentType()

    def msg_unsupported_media_type(self):
        return OnboardUnsupportedMediaType()

    def msg_missing_content_boundary(self):
        return OnboardMissingContentBoundary()

    def msg_start(self):
        return OnboardStart()

    def msg_success(self):
        return OnboardSuccess()

    def msg_failure(self):
        return OnboardFailure()

    def msg_package_upload(self):
        return OnboardPackageUpload()

    def post(self):
        """Handle a post request

        The function is called after any data associated with the body of the
        request has been received.

        """
        super().post()

        filesize = os.stat(self.package_name).st_size
        self.log.debug('wrote {} bytes to {}'.format(filesize, self.package_name))

        self.application.onboard(
                self.package_name,
                self.boundary,
                self.transaction_id,
                auth=self.request.headers.get('authorization', None),
                )

        self.set_status(200)
        self.write(tornado.escape.json_encode({
            "transaction_id": self.transaction_id,
                }))

class UpdateHandler(StreamingUploadHandler):
    def msg_missing_content_type(self):
        return UpdateMissingContentType()

    def msg_unsupported_media_type(self):
        return UpdateUnsupportedMediaType()

    def msg_missing_content_boundary(self):
        return UpdateMissingContentBoundary()

    def msg_start(self):
        return UpdateStart()

    def msg_success(self):
        return UpdateSuccess()

    def msg_failure(self):
        return UpdateFailure()

    def msg_package_upload(self):
        return UpdatePackageUpload()

    def post(self):
        """Handle a post request

        The function is called after any data associated with the body of the
        request has been received.

        """
        super().post()

        filesize = os.stat(self.package_name).st_size
        self.log.debug('wrote {} bytes to {}'.format(filesize, self.package_name))

        self.application.update(
                self.package_name,
                self.boundary,
                self.transaction_id,
                auth=self.request.headers.get('authorization', None),
                )

        self.set_status(200)
        self.write(tornado.escape.json_encode({
            "transaction_id": self.transaction_id,
                }))

class UploadStateHandler(state.StateHandler):
    STARTED = OnboardStart
    SUCCESS = OnboardSuccess
    FAILURE = OnboardFailure

class UpdateStateHandler(state.StateHandler):
    STARTED = UpdateStart
    SUCCESS = UpdateSuccess
    FAILURE = UpdateFailure

class UpdatePackage(threading.Thread):
    def __init__(self, log, filename, boundary, pkg_id, auth,
                 onboarder, uploader, package_store_map):
        super().__init__()
        self.log = log
        self.filename = filename
        self.boundary = boundary
        self.pkg_id = pkg_id
        self.auth = auth
        self.onboarder = onboarder
        self.uploader = uploader
        self.package_store_map = package_store_map

        self.io_loop = tornado.ioloop.IOLoop.current()

    def _update_package(self):
        # Extract package could return multiple packages if
        # the package is converted
        for pkg in self.extract_package():
            with pkg as temp_package:
                self.validate_package(temp_package)
                stored_package = self.update_package(temp_package)

            try:
                self.extract_charms(stored_package)
                self.extract_scripts(stored_package)
                self.extract_configs(stored_package)
                self.extract_icons(stored_package)

                self.upload_images(stored_package)
                self.update_descriptors(stored_package)

            except Exception:
                self.delete_stored_package(stored_package)
                raise

    def run(self):
        try:
            self._update_package()
            self.log.message(UpdateSuccess())

        except MessageException as e:
            self.log.message(e.msg)
            self.log.message(UpdateFailure())

        except Exception as e:
            self.log.exception(e)
            if str(e):
                self.log.message(UpdateError(str(e)))
            self.log.message(UpdateFailure())

    def extract_package(self):
        """Extract multipart message from tarball"""
        extractor = extract.UploadPackageExtractor(self.log)
        file_backed_package = extractor.extract_package_from_upload(
                self.filename, self.boundary, self.pkg_id
                )

        return file_backed_package

    def get_package_store(self, package):
        return self.package_store_map[package.descriptor_type]

    def update_package(self, package):
        store = self.get_package_store(package)

        try:
            store.update_package(package)
        except rift.package.store.PackageNotFoundError as e:
            raise MessageException(UpdatePackageNotFoundError(package.descriptor_id)) from e

        stored_package = store.get_package(package.descriptor_id)

        return stored_package

    def delete_stored_package(self, package):
        store = self.get_package_store(package)
        try:
            store.delete_package(package)
        except Exception as e:
            self.log.warning("Failed to delete package from store: %s", str(e))

    def upload_images(self, package):
        image_file_map = rift.package.image.get_package_image_files(package)
        name_hdl_map = {name: package.open(image_file_map[name]) for name in image_file_map}
        try:
            self.uploader.upload_images(name_hdl_map)

        except image.ImageUploadError as e:
            raise MessageException(UpdateImageUploadError()) from e

        finally:
            for hdl in name_hdl_map.values():
                hdl.close()

    def extract_charms(self, package):
        try:
            charm_extractor = rift.package.charm.PackageCharmExtractor(self.log)
            charm_extractor.extract_charms(package)
        except rift.package.charm.CharmExtractionError as e:
            raise MessageException(UpdateExtractionError()) from e

    def extract_scripts(self, package):
        try:
            script_extractor = rift.package.script.PackageScriptExtractor(self.log)
            script_extractor.extract_scripts(package)
        except rift.package.script.ScriptExtractionError as e:
            raise MessageException(UpdateExtractionError()) from e

    def extract_configs(self, package):
        try:
            config_extractor = rift.package.config.PackageConfigExtractor(self.log)
            config_extractor.extract_configs(package)
        except rift.package.config.ConfigExtractionError as e:
            raise MessageException(UpdateExtractionError()) from e

    def extract_icons(self, package):
        try:
            icon_extractor = rift.package.icon.PackageIconExtractor(self.log)
            icon_extractor.extract_icons(package)
        except rift.package.icon.IconExtractionError as e:
            raise MessageException(UpdateExtractionError()) from e

    def validate_package(self, package):
        checksum_validator = rift.package.package.PackageChecksumValidator(self.log)

        try:
            checksum_validator.validate(package)
        except rift.package.package.PackageFileChecksumError as e:
            raise MessageException(UpdateChecksumMismatch(e.filename)) from e
        except rift.package.package.PackageValidationError as e:
            raise MessageException(UpdateUnreadablePackage()) from e

    def update_descriptors(self, package):
        descriptor_msg = package.descriptor_msg

        self.log.message(UpdateDescriptorUpdate())

        try:
            self.onboarder.update(descriptor_msg)
        except onboard.UpdateError as e:
            raise MessageException(UpdateDescriptorError(package.descriptor_file)) from e

class OnboardPackage(threading.Thread):
    def __init__(self, log, filename, boundary, pkg_id, auth,
                 onboarder, uploader, package_store_map):
        super().__init__()
        self.log = log
        self.filename = filename
        self.boundary = boundary
        self.pkg_id = pkg_id
        self.auth = auth
        self.onboarder = onboarder
        self.uploader = uploader
        self.package_store_map = package_store_map

        self.io_loop = tornado.ioloop.IOLoop.current()

    def _onboard_package(self):
        # Extract package could return multiple packages if
        # the package is converted
        for pkg in self.extract_package():
            with pkg as temp_package:
                self.validate_package(temp_package)
                stored_package = self.store_package(temp_package)

                try:
                    self.extract_charms(stored_package)
                    self.extract_scripts(stored_package)
                    self.extract_configs(stored_package)
                    self.extract_icons(stored_package)

                    self.upload_images(stored_package)
                    self.onboard_descriptors(stored_package)

                except Exception:
                    self.delete_stored_package(stored_package)
                    raise

    def run(self):
        try:
            self._onboard_package()
            self.log.message(OnboardSuccess())

        except MessageException as e:
            self.log.message(e.msg)
            self.log.message(OnboardFailure())

        except Exception as e:
            self.log.exception(e)
            if str(e):
                self.log.message(OnboardError(str(e)))
            self.log.message(OnboardFailure())

    def extract_package(self):
        """Extract multipart message from tarball"""
        extractor = extract.UploadPackageExtractor(self.log)
        file_backed_packages = extractor.extract_package_from_upload(
                self.filename, self.boundary, self.pkg_id
                )

        return file_backed_packages

    def get_package_store(self, package):
        return self.package_store_map[package.descriptor_type]

    def store_package(self, package):
        store = self.get_package_store(package)

        try:
            store.store_package(package)
        except rift.package.store.PackageExistsError as e:
            #TODO: The package store needs to stay in sync with configured
            #descriptors (descriptor delete -> stored package deletion).  Until
            #that happens, we should just update the stored package.
            #raise MessageException(OnboardDescriptorExistsError(package.descriptor_id)) from e
            store.update_package(package)

        stored_package = store.get_package(package.descriptor_id)

        return stored_package

    def delete_stored_package(self, package):
        store = self.get_package_store(package)
        try:
            store.delete_package(package)
        except Exception as e:
            self.log.warning("Failed to delete package from store: %s", str(e))

    def upload_images(self, package):
        image_names = rift.package.image.get_package_image_files(package)
        image_file_map = rift.package.image.get_package_image_files(package)
        name_hdl_map = {name: package.open(image_file_map[name]) for name in image_file_map}
        try:
            self.uploader.upload_images(name_hdl_map)

        except image.ImageUploadError as e:
            raise MessageException(OnboardImageUploadError()) from e

        finally:
            for hdl in name_hdl_map.values():
                hdl.close()

    def extract_charms(self, package):
        try:
            charm_extractor = rift.package.charm.PackageCharmExtractor(self.log)
            charm_extractor.extract_charms(package)
        except rift.package.charm.CharmExtractionError as e:
            raise MessageException(OnboardExtractionError()) from e

    def extract_scripts(self, package):
        try:
            script_extractor = rift.package.script.PackageScriptExtractor(self.log)
            script_extractor.extract_scripts(package)
        except rift.package.script.ScriptExtractionError as e:
            raise MessageException(OnboardExtractionError()) from e

    def extract_configs(self, package):
        try:
            config_extractor = rift.package.config.PackageConfigExtractor(self.log)
            config_extractor.extract_configs(package)
        except rift.package.config.ConfigExtractionError as e:
            raise MessageException(OnboardExtractionError()) from e

    def extract_icons(self, package):
        try:
            icon_extractor = rift.package.icon.PackageIconExtractor(self.log)
            icon_extractor.extract_icons(package)
        except rift.package.icon.IconExtractionError as e:
            raise MessageException(OnboardExtractionError()) from e

    def validate_package(self, package):
        checksum_validator = rift.package.package.PackageChecksumValidator(self.log)

        try:
            checksum_validator.validate(package)
        except rift.package.package.PackageFileChecksumError as e:
            raise MessageException(OnboardChecksumMismatch(e.filename)) from e
        except rift.package.package.PackageValidationError as e:
            raise MessageException(OnboardUnreadablePackage()) from e

    def onboard_descriptors(self, package):
        descriptor_msg = package.descriptor_msg

        self.log.message(OnboardDescriptorOnboard())

        try:
            self.onboarder.onboard(descriptor_msg)
        except onboard.OnboardError as e:
            raise MessageException(OnboardDescriptorError(package.descriptor_file)) from e

class UploaderApplication(tornado.web.Application):
    def __init__(self, tasklet):
        self.tasklet = tasklet
        self.accounts = []
        self.messages = collections.defaultdict(list)
        self.export_dir = os.path.join(os.environ['RIFT_ARTIFACTS'], 'launchpad/exports')

        manifest = tasklet.tasklet_info.get_pb_manifest()
        self.use_ssl = manifest.bootstrap_phase.rwsecurity.use_ssl
        self.ssl_cert = manifest.bootstrap_phase.rwsecurity.cert
        self.ssl_key = manifest.bootstrap_phase.rwsecurity.key

        self.uploader = image.ImageUploader(self.log, self.accounts)
        self.onboarder = onboard.DescriptorOnboarder(
                self.log, "127.0.0.1", 8008, self.use_ssl, self.ssl_cert, self.ssl_key
                )
        self.package_store_map = {
                "vnfd": rift.package.store.VnfdPackageFilesystemStore(self.log),
                "nsd": rift.package.store.NsdPackageFilesystemStore(self.log),
                }

        self.exporter = export.DescriptorPackageArchiveExporter(self.log)
        self.loop.create_task(export.periodic_export_cleanup(self.log, self.loop, self.export_dir))

        attrs = dict(log=self.log, loop=self.loop)

        export_attrs = attrs.copy()
        export_attrs.update({
            "store_map": self.package_store_map,
            "exporter": self.exporter,
            "catalog_map": {
                "vnfd": self.vnfd_catalog,
                "nsd": self.nsd_catalog
                }
            })

        super(UploaderApplication, self).__init__([
            (r"/api/update", UpdateHandler, attrs),
            (r"/api/upload", UploadHandler, attrs),

            (r"/api/upload/([^/]+)/state", UploadStateHandler, attrs),
            (r"/api/update/([^/]+)/state", UpdateStateHandler, attrs),
            (r"/api/export/([^/]+)/state", export.ExportStateHandler, attrs),

            (r"/api/export/(nsd|vnfd)$", export.ExportHandler, export_attrs),
            (r"/api/export/([^/]+.tar.gz)", tornado.web.StaticFileHandler, {
                "path": self.export_dir,
                }),
            (r"/api/export/([^/]+.zip)", tornado.web.StaticFileHandler, {
                "path": self.export_dir,
                }),
            ])

    @property
    def log(self):
        return self.tasklet.log

    @property
    def loop(self):
        return self.tasklet.loop

    def get_logger(self, transaction_id):
        return message.Logger(self.log, self.messages[transaction_id])

    def onboard(self, package, boundary, transaction_id, auth=None):
        log = message.Logger(self.log, self.messages[transaction_id])

        pkg_id = str(uuid.uuid1())
        OnboardPackage(
                log,
                package,
                boundary,
                pkg_id,
                auth,
                self.onboarder,
                self.uploader,
                self.package_store_map,
                ).start()

    def update(self, package, boundary, transaction_id, auth=None):
        log = message.Logger(self.log, self.messages[transaction_id])

        pkg_id = str(uuid.uuid1())
        UpdatePackage(
                log,
                package,
                boundary,
                pkg_id,
                auth,
                self.onboarder,
                self.uploader,
                self.package_store_map,
                ).start()

    @property
    def vnfd_catalog(self):
        return self.tasklet.vnfd_catalog

    @property
    def nsd_catalog(self):
        return self.tasklet.nsd_catalog


# Algorithm to start Bayes programs
from MantidFramework import *
from mantidsimple import *

class Moments(PythonAlgorithm):

	def category(self):
		return "Workflow\\MIDAS;PythonAlgorithms"

	def PyInit(self):
		self.declareProperty(Name='InputType',DefaultValue='File',Validator=ListValidator(['File','Workspace']),Description = 'Origin of data input - File or Workspace')
		self.declareProperty(Name='Instrument',DefaultValue='iris',Validator=ListValidator(['irs','iris','osi','osiris']),Description = 'Instrument')
		self.declareProperty(Name='Analyser',DefaultValue='graphite002',Validator=ListValidator(['graphite002','graphite004']),Description = 'Analyser & reflection')
		self.declareProperty(Name='SamNumber',DefaultValue='',Validator=MandatoryValidator(),Description = 'Sample run number')
		self.declareProperty(Name='EnergyMin', DefaultValue=-0.5,Description = 'Minimum energy for fit. Default=-0.5')
		self.declareProperty(Name='EnergyMax', DefaultValue=0.5,Description = 'Maximum energy for fit. Default=0.5')
		self.declareProperty(Name='MultiplyBy', DefaultValue=1.0,Description = 'Scale factor to multiply S(Q,w). Default=1.0')
		self.declareProperty('Verbose',DefaultValue=True,Description = 'Switch Verbose Off/On')
		self.declareProperty('Plot',DefaultValue=True,Description = 'Switch Plot Off/On')
		self.declareProperty('Save',DefaultValue=False,Description = 'Switch Save result to nxs file Off/On')

	def PyExec(self):

		self.log().information('Moments calculation')
		inType = self.getPropertyValue('InputType')
		prefix = self.getPropertyValue('Instrument')
		ana = self.getPropertyValue('Analyser')
		sn = self.getPropertyValue('SamNumber')
		sam = prefix+sn+'_'+ana+'_sqw'
		emin = self.getPropertyValue('EnergyMin')
		emax = self.getPropertyValue('EnergyMax')
		erange = [float(emin), float(emax)]
		factor = self.getPropertyValue('MultiplyBy')
		factor = float(factor)

		verbOp = self.getProperty('Verbose')
		plotOp = self.getProperty('Plot')
		saveOp = self.getProperty('Save')
		workdir = config['defaultsave.directory']
		if inType == 'File':
			spath = os.path.join(workdir, sam+'.nxs')		# path name for sample nxs file
			self.log().notice('Input from File : '+spath)
			LoadNexus(Filename=spath, OutputWorkspace=sam)
		else:
			self.log().notice('Input from Workspace : '+sam)
		from IndirectEnergyConversion import SqwMoments
		SqwMoments(sam,erange,factor,verbOp,plotOp,saveOp)

mantid.registerPyAlgorithm(Moments())         # Register algorithm with Mantid


# System imports
import logging
import yaml
import json
import os.path as path
from jinja2 import Template
from sys import argv, exit

# Local imports
import generator_functions

logging.basicConfig(level=logging.INFO)

def _parse_config(filename):
    with open(filename, "r") as f:
        try:
            return yaml.safe_load(f)
        except Exception as e:
            logging.error("Could not load file {}: {}".format(filename, e))
            return None

def _read_source(config_path, config):
    if 'name' in config:
        source_filename = path.join(path.dirname(config_path), config['name'])
    else:
        source_filename = config_filename.replace('.yml', '.c')

    with open(source_filename, 'r') as source_file:
        return source_file.read()

def _generate_phase_source_from_template(source, config):
    template = Template(source)

    # Build a dictionary that maps all function names in generator_functions to
    # the actual functions, e.g. 'simple_int_range': simple_int_range. All of
    # these functions will then be available in the phase templates.
    functions = {}
    for name in dir(generator_functions):
        attr = getattr(generator_functions, name)
        if callable(attr) and attr.__module__ == 'generator_functions':
            functions[name] = attr

    return template.render(functions)

def generate_phase(config_path):
    config = _parse_config(config_path)
    if config is None:
        exit(1)

    source_template = _read_source(config_path, config)

    return _generate_phase_source_from_template(source_template, config)

def main():
    logging.info("Starting phase generation")
    if len(argv) == 3:
        generator_functions.static_value_map = json.loads(argv[2])
    elif len(argv) != 2:
        print("Invalid arguments. Please specify a phase configuration file")
        exit(1)

    config_path = argv[1]
    print(generate_phase(config_path))

if __name__ == '__main__':
    main()


#!/usr/bin/env python3

from flask import Flask
from flask_restful import Api, Resource, reqparse
import yaml
import os

configdir = os.curdir + '/config/'

class User(Resource):
    def get(self, name):
        files = os.listdir(configdir)
        for file in files:
            print('config file: ' + file)
            config = load_config(file)
            try:
                config.keys()
            except NameError:
                return 'ERROR: Could not parse yaml', 400
            for check in config.keys():
                try:
                    config[check]['command']
                except TypeError:
                    return 'ERROR: Could not find command in check definition', 400

                out = 'check: ' + check
                out += ' command: ' + ' '.join(config[check]['command'])
                print( out )

        return "User not found", 404

    def post(self, name):
        user = 'hi'
        return user, 201

    def put(self, name):
        user = 'hi'
        return user, 201

    def delete(self, name):
        global users
        users = [user for user in users if user["name"] != name]
        return "{} is deleted.".format(name), 200

def load_config( file ):
  with open(configdir + file, 'r') as stream:
    try:
      return yaml.safe_load(stream)
    except yaml.YAMLError as exc:
      print(exc)

app = Flask(__name__)
api = Api(app)
api.add_resource(User, "/user/<string:name>")

app.run(debug=True)


"""
Binary search trees are a data structure that enforce an ordering over
the data they store. That ordering in turn makes it a lot more efficient
at searching for a particular piece of data in the tree.

This part of the project comprises two days:
1. Implement the methods `insert`, `contains`, `get_max`, and `for_each`
   on the BSTNode class.
2. Implement the `in_order_print`, `bft_print`, and `dft_print` methods
   on the BSTNode class.
"""
import sys
sys.path.append('queue/')
from queue import Queue
class BSTNode:
    def __init__(self, value):
        self.value = value
        self.left = None
        self.right = None

    # Insert the given value into the tree
    def insert(self, value):
        if value < self.value:
            if not self.left:
                self.left = BSTNode(value)
            else:
                self.left.insert(value)
        else:
            if not self.right:
                self.right = BSTNode(value)
            else:
                self.right.insert(value)

    # Return True if the tree contains the value
    # False if it does not
    def contains(self, target):
        if self.value == target:
            return True

        if target < self.value:
            if not self.left:
                return False
            else:
                return self.left.contains(target)
        else:
            if not self.right:
                return False
            else:
                return self.right.contains(target)

    # Return the maximum value found in the tree
    def get_max(self):
        # recursive way
        if not self.right:
            return self.value
        return self.right.get_max()

        # iterative way
        # max_value = self.value

        # current_node = self

        # while current_node:
        #     if current_node.value > max_value:
        #         max_value = current_node.value
        #     current_node = current_node.right
        # return max_value

    # Call the function `fn` on the value of each node
    def for_each(self, fn):
        fn(self.value)

        if self.left:
            self.left.for_each(fn)

        if self.right:
            self.right.for_each(fn)

    # Part 2 -----------------------

    # Print all the values in order from low to high
    # Hint:  Use a recursive, depth first traversal
    def in_order_print(self):
        if self.value:
            if self.left:
                self.left.in_order_print()
            print(self.value)
            if self.right:
                self.right.in_order_print()

    # Print the value of every node, starting with the given node,
    # in an iterative breadth first traversal
    def bft_print(self):
        q = Queue()
        visited = []
        # print(dir(q))

        def rec(tree):
            if tree.value not in visited:
                q.enqueue(tree.value)
                visited.append(tree.value)
            if tree.left:
                q.enqueue(tree.left.value)
                visited.append(tree.left.value)
            if tree.right:
                q.enqueue(tree.right.value)
                visited.append(tree.right.value)
            if tree.left:
                rec(tree.left)
            if tree.right:
                rec(tree.right)
        rec(self)

        string = ''
        for x in range(q.__len__()):

            string += f'{q.dequeue()}\n'
            # print(value)

        print(string)
        return string
        pass

    # Print the value of every node, starting with the given node,
    # in an iterative depth first traversal
    def dft_print(self):
        pass

    # Stretch Goals -------------------------
    # Note: Research may be required

    # Print Pre-order recursive DFT
    def pre_order_dft(self):
        pass

    # Print Post-order recursive DFT
    def post_order_dft(self):
        pass

"""
This code is necessary for testing the `print` methods
"""
bst = BSTNode(1)

bst.insert(8)
bst.insert(5)
bst.insert(7)
bst.insert(6)
bst.insert(3)
bst.insert(4)
bst.insert(2)

# bst.in_order_print()
bst.bft_print()
# bst.dft_print()

# print("elegant methods")
# print("pre order")
# bst.pre_order_dft()
# print("in order")
# bst.in_order_dft()
# print("post order")
# bst.post_order_dft()

# test = Queue()
# print(test.__len__())
# print(test.enqueue(1))
# print(test.__len__())


# -*- coding: utf-8 -*-
import scrapy
from jianshu.items import ArticleItem

class JsmemberspiderSpider(scrapy.Spider):
    name = "jsMemberSpider"
    # allowed_domains = ["jianshu"]
    jianshu = 'http://www.jianshu.com'
    start_urls = ['http://www.jianshu.com/u/54b5900965ea?order_by=shared_at&page=1', ]

    article_all = []

    start_url = 'http://www.jianshu.com/u/54b5900965ea?order_by=shared_at&page={}'
    start_page = 1
    userID = '54b5900965ea'
    pageNumber = 1
    stop_url = 'http://www.jianshu.com/u/{USERID}?order_by=shared_at&page={pageNumber}'.format(USERID = userID, pageNumber = pageNumber)

    def parse(self, response):
        # 获取文章总数
        article_count = response.css(
            'body > div.container.person > div > div.col-xs-16.main > div.main-top > div.info > ul > li:nth-child(3) > div > a > p::text').extract_first()
        # 计算共分了多少页，每页9篇文章
        countPageNumber = int(int(article_count) / 9 + 0.5)
        # 获取文章列表
        articles = response.css('ul.note-list > li')
        for article in articles:
            articleItem = ArticleItem()
            # 获取作者名称
            articleItem['author_name'] = article.css('a.blue-link::text').extract_first()
            # author_name = article.css('a.blue-link::text').extract_first()
            # 获取作者的头像连接
            articleItem['author_image'] = 'http:' + article.css('div.author > a > img::attr(src)').extract_first()
            # author_image = 'http:' + article.css('div.author > a > img::attr(src)').extract_first()
            # 获取文章发布时间
            articleItem['article_release_time'] = article.css('div.name > span.time::attr(data-shared-at)').extract_first()
            article_release_time = article.css('div.name > span.time::attr(data-shared-at)').extract_first()
            # 获取标题
            articleItem['article_title'] = article.css('a.title::text').extract_first()
            # article_title = article.css('a.title::text').extract_first()
            # 获取文章描述
            articleItem['article_desc'] = article.css('p.abstract::text').extract_first().strip()
            # article_desc = article.css('p.abstract::text').extract_first().strip()
            # 获取文章链接
            articleItem['article_link'] = JsmemberspiderSpider.jianshu + article.css('div.content > a::attr(href)').extract_first()
            # article_link = JsmemberspiderSpider.jianshu + article.css('div.content > a::attr(href)').extract_first()
            # 获取阅读量，回复量，喜欢人数，赞赏人数
            articleItem['read_count'] = article.css('div.meta > a')[0].css('::text').extract()[-1].strip()
            articleItem['reply_count'] = article.css('div.meta > a')[1].css('::text').extract()[-1].strip()
            articleItem['likeit_count'] = article.css('div.meta > span')[0].css('::text').extract_first().strip()
            articleItem['payit_count'] = article.css('div.meta > span')[1].css('::text').extract_first().strip() if len(article.css('div.meta > span'))>=2 else 0
            # read_count = article.css('div.meta > a')[0].css('::text').extract()[-1].strip()
            # reply_count = article.css('div.meta > a')[1].css('::text').extract()[-1].strip()
            # likeit_count = article.css('div.meta > span')[0].css('::text').extract_first().strip()
            # payit_count = article.css('div.meta > span')[1].css('::text').extract_first().strip() if len(article.css('div.meta > span'))>=2 else 0
            # yield {
            #     'author_name': author_name,
            #     'author_image': author_image,
            #     'article_release_time': article_release_time,
            #     'article_title': article_title,
            #     'article_desc': article_desc,
            #     'article_link': article_link,
            #     'read_count': read_count,
            #     'reply_count': reply_count,
            #     'likeit_count': likeit_count,
            #     'payit_count': payit_count,
            # }
            JsmemberspiderSpider.article_all.append(articleItem)
            yield articleItem
            # pages = (i for i in range(2, countPageNumber + 1))
            current_page = int(response.url.split('page=')[1])
            next_page = JsmemberspiderSpider.start_url.format(current_page + 1)
            # 爬虫结束的条件，如果当前页是最后一页
            if current_page == countPageNumber:
                next_page = None
            if next_page is not None:
                next_page = response.urljoin(next_page)
                # yield {
                # '爬取中：': next_page,
                # }
                yield scrapy.Request(next_page, callback=self.parse)


# !/usr/bin/python3
import sys

num = int(sys.argv[1])

my = [0, 1, 1]

for i in range(num+1):
    if i < 3:
        continue
    else:
        cat = my[i-1] + my[i-2]
        my.append(cat)

print(my[num])


import logging
from django.core.exceptions import ObjectDoesNotExist
from rest_framework import status
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework.permissions import IsAuthenticated
from ecommerce_projec.ecommerce.services import CategoryService
from ecommerce_projec.ecommerce.serializers import CategorySerializer

class CategoryView(APIView):
    permission_classes = (IsAuthenticated,)

    def get(self, request, category_id):
        try:
            category = CategoryService.get_category(category_id)
            serializer = CategorySerializer(category)
            return Response(serializer.data, status.HTTP_200_OK)
        except (ObjectDoesNotExist, Exception) as err:
            logging.info(f"CategoryView::get Error: {err}")
            return Response(
                dict(success=False, message=f"{err}"),
                status.HTTP_400_BAD_REQUEST
            )

    def put(self, request, category_id):
        try:
            serializer = CategorySerializer(data=request.data)
            if not serializer.is_valid():
                raise Exception(serializer.errors)
            category = CategoryService.get_category(category_id)
            CategoryService.update(category, serializer.data)
            return Response(serializer.data, status.HTTP_200_OK)
        except (ObjectDoesNotExist, Exception) as err:
            logging.info(f"CategoryView::put Error: {err}")
            return Response(
                dict(success=False, message=f"{err}"),
                status.HTTP_400_BAD_REQUEST
            )

    def delete(self, request, category_id):
        try:
            CategoryService.delete(category_id)
            return Response(status.HTTP_200_OK)
        except (ObjectDoesNotExist, Exception) as err:
            logging.info(f"CategoryView::delete Error: {err}")
            return Response(
                dict(success=False, message=f"{err}"),
                status.HTTP_400_BAD_REQUEST
            )


from django.core.validators import MinValueValidator, MaxValueValidator
from django.db import models
from django.utils.translation import ugettext_lazy as _

from edc.audit.audit_trail import AuditTrail

from apps.bcpp.choices import YES_NO_DWTA, ALCOHOL_SEX

from .base_scheduled_visit_model import BaseScheduledVisitModel

class SexualBehaviour (BaseScheduledVisitModel):

    """CS002"""

    ever_sex = models.CharField(
        verbose_name=_("In your lifetime, have you ever had sex with anyone"
                      " (including your spouse, friends, or someone you have just met)?"),
        max_length=25,
        choices=YES_NO_DWTA,
        help_text="",
        )

    lifetime_sex_partners = models.IntegerField(
        verbose_name=_("In your lifetime, how many different people have you had"
                      " sex with?  Please remember to include casual and once-off partners"
                      " (prostitutes and truck drivers) as well as long-term partners"
                      " (spouses, boyfriends/girlfriends)[If you can't recall the exact "
                      "number, please give a best guess]"),
        max_length=3,
        null=True,
        blank=True,
        help_text="",
        )

    last_year_partners = models.IntegerField(
        verbose_name=_("In the past 12 months, how many different people have you had"
                      " sex with?  Please remember to include casual and once-off partners"
                      " (prostitutes and truck drivers) as well as long-term partners"
                      " (spouses, boyfriends/girlfriends)[If you can't recall the exact "
                      "number, please give a best guess]"),
        max_length=2,
        null=True,
        blank=True,
        help_text="Note:Leave blank if participant does not want to respond. ",
        )

    more_sex = models.CharField(
        verbose_name=_("In the past 12 months, did you have sex with somebody"
                      " living outside of the community?"),
        max_length=25,
        null=True,
        blank=True,
        choices=YES_NO_DWTA,
        help_text="",
        )

    first_sex = models.IntegerField(
        verbose_name=_("How old were you when you had sex for the first time?"
                      " [If you can't recall the exact age, please give a best guess]"),
        max_length=2,
        null=True,
        blank=True,
        validators=[MinValueValidator(10), MaxValueValidator(64)],
        help_text="Note:leave blank if participant does not want to respond.",
        )

    condom = models.CharField(
        verbose_name=_("During the last [most recent] time you had sex, did"
                      " you or your partner use a condom?"),
        max_length=25,
        null=True,
        blank=True,
        choices=YES_NO_DWTA,
        help_text="",
        )

    alcohol_sex = models.CharField(
        verbose_name=_("During the last [most recent] time you had sex, were"
                      " you or your partner drinking alcohol?"),
        max_length=25,
        null=True,
        blank=True,
        choices=ALCOHOL_SEX,
        help_text="",
        )

    history = AuditTrail()

    class Meta:
        app_label = 'bcpp_subject'
        verbose_name = "Sexual Behaviour"
        verbose_name_plural = "Sexual Behaviour"


# Linked List head is Node(-1)
class Node:
    def __init__(self, data):
        self.data = data
        self.nextElement = None

class LinkedList:
    def __init__(self):
        self.headNode = Node(-1)

    def isEmpty(self):
        if(self.headNode.nextElement == None):
            return True
        else:
            return False

    def getHead(self):
        return self.headNode

    def printList(self):
        if(self.isEmpty()):
            print("List is Empty")
            return False
        temp = self.headNode.nextElement
        while(temp.nextElement != None):
            print(temp.data, "->", end=" ")
            temp = temp.nextElement
        print(temp.data, "-> None")
        return True

    def insertAtHead(self, data):
        node = Node(data)
        node.nextElement = self.headNode.nextElement
        self.headNode.nextElement = node
        return self.headNode

def reverse2(list):
    if(list.isEmpty()):
        return LinkedList()
    list2 = LinkedList()
    current = list.getHead().nextElement
    while(current != None):
        list2.insertAtHead(current.data)
        current = current.nextElement
    return list2

# list = LinkedList()
# list.insertAtHead(6)
# list.insertAtHead(4)
# list.insertAtHead(9)
# list.insertAtHead(10)
# list.printList()

# list2 = reverse2(list)
# list2.printList()
#---------------------------------------------------------

def reverse(list):
  prev = None #To keep track of the previous element, will be used in swapping links
  current = list.getHead().nextElement #firstElement
  next = None

  #While Traversing the list, swap links
  while (current != None):
    next = current.nextElement
    current.nextElement = prev
    prev = current
    current = next

  #Linking Head Node with the new First Element
  list.headNode.nextElement = prev

list = LinkedList()
list.insertAtHead(6)
list.insertAtHead(4)
list.insertAtHead(9)
list.insertAtHead(10)
list.printList()

reverse(list)
list.printList()


# coding=utf8
from __future__ import unicode_literals

from django.db import models
from django.utils.encoding import python_2_unicode_compatible

from hidromed.empresas.models import Cliente, Acueducto

#modelos correspondientes a las polizas
@python_2_unicode_compatible
class Poliza(models.Model):
	numero = models.CharField(max_length=255)

	def __str__(self):
		return self.numero

@python_2_unicode_compatible
class Poliza_Cliente(models.Model):
	poliza = models.ForeignKey(Poliza, default=None, null=True, blank=True)
	cliente = models.ForeignKey(Cliente, default=None, null=True, blank=True)

	class Meta:
		verbose_name = 'Póliza vs Cliente'
		verbose_name_plural = 'Pólizas vs Clientes'

	def __str__(self):
		relacion = (u'%s' % self.poliza + ' - '
			u'%s' % self.cliente )
		return relacion

@python_2_unicode_compatible
class Poliza_Acueducto(models.Model):
	poliza = models.ForeignKey(Poliza, default=None, null=True, blank=True)
	acueducto = models.ForeignKey(Acueducto, default=None, null=True, blank=True)

	class Meta:
		verbose_name = 'Póliza vs Acueducto'
		verbose_name_plural = 'Pólizas vs Acueductos'

	def __str__(self):
		relacion = (u'%s' % self.poliza + ' - '
			u'%s' % self.acueducto )
		return relacion


from typing import List

import numpy as np
from keras.models import Model
from keras import backend as K

from ..abstract import RLAgentBase
from ..experience import Experience, ExperienceSampler
from ..utility import kerasic

class Actor(RLAgentBase):

    def __init__(self,
                 model: Model,
                 action_space,
                 memory: Experience=None,
                 discount_factor_gamma=0.99,
                 state_preprocessor=None):

        super().__init__(action_space, memory, discount_factor_gamma, state_preprocessor)
        self.model = model
        self.action_indices = np.arange(len(self.action_space))
        self.possible_actions_onehot = np.eye(len(self.action_space))

    def sample(self, state, reward, done):
        preprocessed_state = self.preprocess(state)[None, ...]
        probabilities = self.model.predict(preprocessed_state)[0]
        action = np.squeeze(np.random.choice(self.action_indices, p=probabilities, size=1))
        self._push_direct_experience(state, action, reward, done)
        return action

class A2C(RLAgentBase):

    history_keys = ["actor_utility", "actor_utility_std", "actor_entropy", "actor_loss",
                    "values", "advantages", "critic_loss"]

    def __init__(self,
                 actor: Model,
                 critic: Model,
                 action_space,
                 absolute_memory_limit=10000,
                 discount_factor_gamma=0.99,
                 state_preprocessor=None,
                 entropy_penalty_coef=0.,
                 copy_actor_models=False):

        super().__init__(action_space=action_space,
                         discount_factor_gamma=discount_factor_gamma,
                         state_preprocessor=state_preprocessor)

        self.actor_learner = actor
        self.critic_learner = critic
        self.num_actions = len(self.action_space)
        self.action_indices = np.arange(len(self.action_space))
        self.possible_actions_onehot = np.eye(len(self.action_space))
        self.entropy_penalty_coef = entropy_penalty_coef
        self._actor_train_function = self._make_actor_train_function()
        self.workers = []  # type: List[Actor]
        self.learning = False
        self.copy_actor_models = copy_actor_models
        self.absolute_memory_limit = absolute_memory_limit

    def set_learning_mode(self, switch: bool):
        pass

    def _update_memory_sampler(self):
        self.memory_sampler = ExperienceSampler([worker.memory for worker in self.workers])

    def create_worker(self, memory=None):
        if self.copy_actor_models:
            model = kerasic.copy_model(self.actor_learner)
        else:
            model = self.actor_learner
        if memory is None:
            memory = Experience(self.absolute_memory_limit)
        self.workers.append(
            Actor(model, self.action_space, memory, state_preprocessor=self.preprocess)
        )
        self._update_memory_sampler()
        return self.workers[-1]

    def dispatch_workers(self, n=1):
        if len(self.workers) > 0:
            raise RuntimeError("Workers already dispatched!")
        for i in range(n):
            self.create_worker(None)
        return self.workers

    def delete_workers(self):
        self.workers = []
        self.memory_sampler = None

    def _make_actor_train_function(self):
        advantages = K.placeholder(shape=(None,))
        softmaxes = self.actor_learner.output
        actions = K.argmax(softmaxes, axis=-1)
        action_masks = K.one_hot(actions, num_classes=len(self.action_space))
        action_masks = K.stop_gradient(action_masks)

        probabilities = K.sum(action_masks * softmaxes, axis=1)
        log_prob = K.log(probabilities)
        entropy = -K.mean(log_prob)
        utilities = -log_prob * advantages
        utility = K.mean(utilities)
        utility_std = K.std(utilities)

        loss = -entropy * self.entropy_penalty_coef + utility
        updates = self.actor_learner.optimizer.get_updates(loss, self.actor_learner.weights)

        return K.function(inputs=[self.actor_learner.input, advantages],
                          outputs=[utility, utility_std, entropy, loss],
                          updates=updates)

    def sample(self, state, reward, done):
        if self.learning:
            raise RuntimeError("A2C's sample may not be called in learning mode!")

        preprocessed_state = self.preprocess(state)[None, ...]
        probabilities = self.actor_learner.predict(preprocessed_state)[0]
        action = np.squeeze(np.random.choice(self.action_indices, p=probabilities, size=1))

        return action

    def push_experience(self, state, reward, done):
        raise RuntimeError("A2C does not learn directly. Please create_workers()!")

    def _sync_workers(self):
        if not self.copy_actor_models:
            return
        W = self.actor_learner.get_weights()
        for worker in self.workers:
            worker.model.set_weights(W)

    def fit(self, batch_size=-1, reset_memory=True):
        S, S_, A, R, F = self.memory_sampler.sample(batch_size)
        assert len(S)

        S_ = self.preprocess(S_)
        S = self.preprocess(S)

        value_next = self.critic_learner.predict(S_)[..., 0]
        bellman_target = value_next * self.gamma + R
        bellman_target[F] = R[F]
        mean_bellman_error = self.critic_learner.train_on_batch(S, bellman_target)

        value = self.critic_learner.predict(S)[..., 0]
        advantage = bellman_target - value

        utility, std, entropy, loss = self._actor_train_function([S, advantage])

        if reset_memory:
            for worker in self.workers:
                worker.memory.reset()

        self._sync_workers()

        return {"actor_utility": utility,
                "actor_utility_std": std,
                "actor_entropy": entropy,
                "actor_loss": loss,
                "values": value.mean(),
                "advantages": advantage.mean(),
                "critic_loss": mean_bellman_error}

    def get_savables(self):
        return {"A2C_actor": self.actor_learner, "A2C_critic": self.critic_learner}


W = 503
H = 376

F = 5
S = 5

def calc_min_padding(dim: int, filter_size: int, stride: int) -> (int, int):
    """
    Calculates the minimum padding for a particular dimension.

    Args:
        dim: The value to pad
        filter_size: The size of the Conv filter
        stride: The size of the stride.
    """
    min_padding = 0
    padded_output = (dim - filter_size + 2 * min_padding) / stride + 1
    while not float(padded_output).is_integer():
        min_padding += 1
        padded_output = (dim - filter_size + 2 * min_padding) / stride + 1
    return min_padding, padded_output

padding, padded_width = calc_min_padding(W, F, S)
print("WIDTH PADDING:")
print("Padding:", padding)
print("New width:", padded_width)

padding, padded_height = calc_min_padding(H, F, S)
print("HEIGHT PADDING:")
print("Padding:", padding)
print("New height:", padded_height)


####################################################################################################
##
##  Project:  Embedded Learning Library (ELL)
##  File:     tutorial.py
##  Authors:  Byron Changuion
##
##  Requires: Python 3.x
##
####################################################################################################

import sys
import os
import time
import numpy as np
import cv2

scriptPath = os.path.dirname(os.path.abspath(__file__))
sys.path.append(scriptPath)
sys.path.append(os.path.join(scriptPath, 'build'))
sys.path.append(os.path.join(scriptPath, 'build/Release')) # required for Windows

# Import the Python wrapper for the ELL model
import model

# Import helper functions
import tutorialHelpers as helpers

# Read an image from the camera
def get_image_from_camera(camera):
    if camera is not None:
        ret, frame = camera.read()
        if (not ret):
            raise Exception('your capture device is not returning images')
        return frame
    return None

def main():
    # Open the video camera. To use a different camera, change the camera index.
    camera = cv2.VideoCapture(0)

    # Read the category names
    categories = open('categories.txt', 'r').read().splitlines()

    # Get the model's input shape. We will use this information later to resize images appropriately.
    input_shape = model.get_default_input_shape()

    # Get the model's output shape and create an array to hold the model's output predictions
    output_shape = model.get_default_output_shape()
    predictions = model.FloatVector(output_shape.Size())

    # Declare a variable to hold the prediction times
    predictionTimes = []
    meanTimeToPredict = 0.0

    while ((cv2.waitKey(1) & 0xFF) == 0xFF):
        # Get an image from the camera.
        image = get_image_from_camera(camera)

        # Prepare an image for processing
        # - Resize and center-crop to the required width and height while preserving aspect ratio.
        # - OpenCV gives the image in BGR order. If needed, re-order the channels to RGB.
        # - Convert the OpenCV result to a std::vector<float>
        input = helpers.prepare_image_for_model(image, input_shape.columns, input_shape.rows)

        # Send the image to the compiled model and fill the predictions vector with scores, measure how long it takes
        start = time.time()
        model.predict(input, predictions)
        end = time.time()

        # Get the value of the top 5 predictions
        top5 = helpers.get_top_n(predictions, 5)

        # Generate header text that represents the top5 predictions
        headerText = ", ".join(["(" + str(int(element[1]*100)) + "%) " + categories[element[0]] for element in top5])
        helpers.draw_header(image, headerText)

        # Generate footer text that represents the mean evaluation time
        meanTimeToPredict = helpers.get_mean_duration(predictionTimes, end - start)
        footerText = '{:.0f}'.format(meanTimeToPredict * 1000) + 'ms/frame'
        helpers.draw_footer(image, footerText)

        # Display the image
        cv2.imshow('ELL model', image)

    print('Mean prediction time: ' + '{:.0f}'.format(meanTimeToPredict * 1000) + 'ms/frame')

if __name__ == "__main__":
    main()


###########################################################################
# Created by: CASIA IVA
# Email: jliu@nlpr.ia.ac.cn
# Copyright (c) 2018
###########################################################################
from __future__ import division
import os
import numpy as np
import torch
import torch.nn as nn
from torch.nn.functional import upsample, normalize
from ..nn import PAM_Module, topk_PAM_Module
from ..nn import CAM_Module , SE_CAM_Module, selective_aggregation_ASPP_Module, reduce_PAM_Module,SE_CAM_Module2, selective_aggregation_ASPP_Module2
from ..models import BaseNet
from ..nn import ASPP_Module

__all__ = ['GLCNet5_topkpam', 'get_glcnet5_topkpam']

class GLCNet5_topkpam(BaseNet):
    r"""Fully Convolutional Networks for Semantic Segmentation

    Parameters
    ----------
    nclass : int
        Number of categories for the training dataset.
    backbone : string
        Pre-trained dilated backbone network type (default:'resnet50'; 'resnet50',
        'resnet101' or 'resnet152').
    norm_layer : object
        Normalization layer used in backbone network (default: :class:`mxnet.gluon.nn.BatchNorm`;

    Reference:

        Long, Jonathan, Evan Shelhamer, and Trevor Darrell. "Fully convolutional networks
        for semantic segmentation." *CVPR*, 2015

    """

    def __init__(self, nclass, backbone, aux=False, se_loss=False, norm_layer=nn.BatchNorm2d, **kwargs):
        super(GLCNet5_topkpam, self).__init__(nclass, backbone, aux, se_loss, norm_layer=norm_layer, **kwargs)
        self.head = GLCNet5_topkpamHead(2048, nclass, norm_layer)

        self.dsn = nn.Sequential(
            nn.Conv2d(1024, 512, kernel_size=3, stride=1, padding=1),
            nn.BatchNorm2d(512), nn.ReLU(inplace=True),
            nn.Dropout2d(0.1),
            nn.Conv2d(512, nclass, kernel_size=1, stride=1, padding=0, bias=True)
        )
    def forward(self, x):
        imsize = x.size()[2:]
        _, _, c3, c4 = self.base_forward(x)
        # x_dsn = self.dsn(c3)
        x = self.head(c4)
        x = list(x)
        x[0] = upsample(x[0], imsize, **self._up_kwargs)
        # x[1] = upsample(x[1], imsize, **self._up_kwargs)
        # x[2] = upsample(x[2], imsize, **self._up_kwargs)
        # x[3] = upsample(x[3], imsize, **self._up_kwargs)
        # x_dsn = upsample(x_dsn, imsize, **self._up_kwargs)

        outputs = [x[0]]
        # outputs.append(x[1])
        # outputs.append(x[2])
        # outputs.append(x[3])
        # outputs.append(x_dsn)

        return tuple(outputs)

class GLCNet5_topkpamHead(nn.Module):
    def __init__(self, in_channels, out_channels, norm_layer):
        super(GLCNet5_topkpamHead, self).__init__()
        inter_channels = in_channels // 4

        self.conv5a = nn.Sequential(nn.Conv2d(in_channels, inter_channels, 3, padding=1, bias=False),
                                    norm_layer(inter_channels),
                                    nn.ReLU())
        self.conv5as = nn.Sequential(nn.Conv2d(in_channels, inter_channels, 3, padding=1, bias=False),
                                    norm_layer(inter_channels),
                                    nn.ReLU())
        self.sa = topk_PAM_Module(inter_channels, 256, inter_channels, 10)
        self.conv51 = nn.Sequential(nn.Conv2d(inter_channels, inter_channels, 3, padding=1, bias=False),
                                    norm_layer(inter_channels),
                                    nn.ReLU())

        self.conv5s = nn.Sequential(nn.Conv2d(in_channels, inter_channels, 1, padding=0, bias=False),
                                    norm_layer(inter_channels),
                                    nn.ReLU())
        self.aspp = selective_aggregation_ASPP_Module2(inter_channels, inner_features=256, out_features=256,
                                                      dilations=(12, 24, 36))
        self.conv52 = nn.Sequential(nn.Conv2d(inter_channels, inter_channels, 3, padding=1, bias=False),
                                    norm_layer(inter_channels),
                                    nn.ReLU())

        self.conv5c = nn.Sequential(nn.Conv2d(in_channels, inter_channels, 3, padding=1, bias=False),
                                    norm_layer(inter_channels),
                                    nn.ReLU())
        self.sc = SE_CAM_Module2(inter_channels)
        self.conv53 = nn.Sequential(nn.Conv2d(inter_channels, inter_channels, 3, padding=1, bias=False),
                                    norm_layer(inter_channels),
                                    nn.ReLU())

        self.conv5 = nn.Sequential(nn.Dropout2d(0.1), nn.Conv2d(512, out_channels, 1))
        self.conv6 = nn.Sequential(nn.Dropout2d(0.1), nn.Conv2d(512, out_channels, 1))
        self.conv7 = nn.Sequential(nn.Dropout2d(0.1), nn.Conv2d(512, out_channels, 1))
        self.conv8 = nn.Sequential(nn.Dropout2d(0.1), nn.Conv2d(512, out_channels, 1))

    def forward(self, x):
        #ssa
        feat1 = self.conv5a(x)
        sa_feat = self.sa(feat1)
        sa_conv = self.conv51(sa_feat)

        sa_output = self.conv5(sa_conv)

        output =[sa_output]
        #aaspp
        #
        # feat2 = self.conv5s(x)
        # feat_fuse = sa_conv + feat2

        # feat_as = self.conv5as(x)
        # aspp_feat,_ = self.aspp(feat_as)
        # aspp_conv = self.conv52(aspp_feat)
        #
        # aspp_output = self.conv6(aspp_conv)
        #
        # #sec
        # feat2 = self.conv5c(x)
        # sc_feat = self.sc(feat2)
        # sc_conv = self.conv53(sc_feat)
        #
        # sc_output = self.conv7(sc_conv)
        #
        # #fuse
        # feat_sum = aspp_conv + sc_conv + sa_conv
        # sasc_output = self.conv8(feat_sum)

        # output = [sasc_output]
        # output.append(sa_output)
        # output.append(sc_output)
        # output.append(aspp_output)

        return tuple(output)

def get_glcnet5_topkpam(dataset='pascal_voc', backbone='resnet50', pretrained=False,
              root='./pretrain_models', **kwargs):
    r"""DANet model from the paper `"Dual Attention Network for Scene Segmentation"
    <https://arxiv.org/abs/1809.02983.pdf>`
    """
    acronyms = {
        'pascal_voc': 'voc',
        'pascal_aug': 'voc',
        'pcontext': 'pcontext',
        'ade20k': 'ade',
        'cityscapes': 'cityscapes',
    }
    # infer number of classes
    from ..datasets import datasets, VOCSegmentation, VOCAugSegmentation, ADE20KSegmentation
    model = GLCNet5_topkpam(datasets[dataset.lower()].NUM_CLASS, backbone=backbone, root=root, **kwargs)
    if pretrained:
        from .model_store import get_model_file
        model.load_state_dict(torch.load(
            get_model_file('fcn_%s_%s' % (backbone, acronyms[dataset]), root=root)),
            strict=False)
    return model


# Set path
file = 'C:/Users/Bkcox/Desktop/Rosalind/Rosalind/rosalind_ini6.txt'
# Create empty dictionary
d = {}
# Opens file and closes after execution
with open(file) as f:
    # converts string in file into a string that can be read
    string = f.read()
    # for loop that reads every word split by whitespace;
    # split() default automatically separates words by whitespace
    for word in string.split():
        # conditional statement that assigns an initial value of 0 to a word
        # not in keys
        if word not in d.keys():
            d[word] = 0
        # Adds to the value assigned to a particular key; if the value has not been
        # entered before, the previous line of code will have started the value at 0, which
        # means the word will always start with a value of one and will go up with every
        # future hit.
        d[word] += 1
    # prints each key with its corresponding value for easier viewing
    for key, value in d.items():
        print(key)
        print(value)


import pandas as pd
import csv

f = open('log_b.txt',mode='r')
new_log = open('log_a.csv','a')
writer = csv.writer(new_log,dialect= 'unix')
writer.writerow(['Ip','Time','Request_Type','Serverside_Status','num_of_bytes','URL'])
for line in f:

    info = line.split('"')
    i = info[1]

    ip = i.split(' ')
    final_ip = ip[0]
    time = ip[3]

    codes = info[5]
    code_split = codes.split(' ')

    request_type = info[3]
    serverside_status = code_split[1]
    num_of_bytes= code_split[2]
    url = info[7]

    collate = [final_ip,time,request_type,serverside_status,num_of_bytes,url]
    writer.writerow(collate)


"""
prtesting.tests.app

modul: test_app_models

 Шаблон для тестовых процедур
    --------------------------------------------------------------

    from AnyMixin.files import writeDict_into_JSON, conv_strPath

    s_json = Type_value.init_formatTempl(s_path, 'jsn')
    js_arg = loadJSON_withTempl(s_json)

    try:
        res = ['test## ']

        return res
    except Exception as ex:
        return str(ex)

"""

# file: test_sprstatus_serv
def test_simpl_proc():
    from advuser.serv_sprstatus import Com_proc_sprstatus as serv_status

    try:
        res = ['test## ']

        get_statusID_user = serv_status.get_statusID_user

        lst = ('suadm','podg-vl','kisl','')

        res.append('testing get_statusID_user: ')
        for item in lst:
            res_proc = get_statusID_user(item)
            res.append('user:{1} statusID:{0} ##'.format(res_proc, item or 'empty' ))

        # ---------- тестирование процедур без параметров ----------------

        get_status_qust_simp = serv_status.get_status_qust_simp
        res_status = get_status_qust_simp()
        if res_status:
            res.append( '##result get_status_qust_simp -> {0} %%%%%'.format(res_status.PR_strStatus) )
        else:
            res.append('get_status_qust_simp нет данных')

        get_status_qust_regs = serv_status.get_status_qust_regs
        res_status = get_status_qust_regs()
        if res_status:
            res.append( '##result get_status_qust_regs -> {0}'.format(res_status.PR_strStatus) )
        else:
            res.append('get_status_qust_regs нет данных')

        get_status_header = serv_status.get_status_header
        res_status = get_status_header()
        if res_status:
            res.append( '##result get_status_header -> {0}'.format(res_status.PR_strStatus) )
        else:
            res.append('get_status_header нет данных')

        get_status_pradm = serv_status.get_status_pradm
        res_status = get_status_pradm()
        if res_status:
            res.append( '##result get_status_pradm -> {0}'.format(res_status.PR_strStatus) )
        else:
            res.append('get_status_pradm нет данных')

        get_status_suadm = serv_status.get_status_suadm
        res_status = get_status_suadm()
        if res_status:
            res.append( '##result get_status_suadm -> {0}'.format(res_status.PR_strStatus) )
        else:
            res.append('get_status_suadm нет данных')

        get_status_notstatus = serv_status.get_status_notstatus
        res_status = get_status_notstatus()
        if res_status:
            res.append( '##result get_status_notstatus -> {0}'.format(res_status.PR_strStatus) )
        else:
            res.append('get_status_notstatus нет данных')

        return res
    except Exception as ex:
        return str(ex)

# Тестирование modul advuser.serv_sprstatus
# file: tests/advuser/modl_serv/test_sprstatus_serv.json
def test_getStatus_or_None():
    from advuser.serv_sprstatus import Com_proc_sprstatus, serv_SprStatus, getStatus_or_None
    from app import getUser

    try:
        res = ['Testing Com_proc_sprstatus## ']

        user = getUser('podg-vl')
        getStatus = Com_proc_sprstatus.getStatus_or_None

        res_status = getStatus_or_None(user)
        if res_status:
            res.append(str(res_status))
        else:
            res.append('Статус не определен')

        # ---------- тестирование через процедуру-диспетчер ------------

        # тестирование через arg_status
        res_serv = serv_SprStatus('getStatus_or_None', user)
        if res_serv:
            res.append(res_serv.PR_strStatus)
        else:
            res.append('serv_SprStatus: статус не определен')

        # тестирование через модель AdvUser
        res_serv = serv_SprStatus('getStatus_or_None', getUser('basam').advuser)
        if res_serv:
            res.append('##' + res_serv.PR_strStatus+ '##')
        else:
            res.append('serv_SprStatus: статус не определен')

        # тестирование через аргумент dict
        arg = dict(username='suadm')
        res_serv = serv_SprStatus('getStatus_or_None', **arg)
        if res_serv:
            res.append(res_serv.PR_strStatus)
        else:
            res.append('serv_SprStatus: статус не определен')

        # тестирование через строковый идентификатор status_id
        res_serv = serv_SprStatus('getStatus_or_None', 'proj-sadm')
        if res_serv:
            res.append(res_serv.PR_strStatus)
        else:
            res.append('serv_SprStatus: статус не определен')

        # тестирование через строковый идентификатор status_id
        arg = dict(statusID='proj-memb')
        res_serv = serv_SprStatus('getStatus_or_None', **arg)
        if res_serv:
            res.append(res_serv.PR_strStatus)
        else:
            res.append('serv_SprStatus: статус не определен')

        # тестирование выброса исключения и записи в файл *.log
        arg = dict(statusID='proj-memb', username='pradm_')
        res_serv = serv_SprStatus('getStatus_or_None', user, **arg)
        if res_serv:
            res.append(res_serv.PR_strStatus)
        else:
            res.append('serv_SprStatus: статус не определен')

        ## тестирование приоритетности
        #arg = dict(statusID='proj-memb')
        #res_serv = serv_SprStatus('getStatus_or_None', 'qust-simp', **arg)
        #if res_serv:
        #    res.append(str(res_serv))
        #else:
        #    res.append('serv_SprStatus: статус не определен')

        # тестирование приоритетности
        arg = dict(statusID='proj-memb')
        res_serv = serv_SprStatus('getStatus_or_None', 'qust-simp', **dict(statusID='proj-memb'))
        if res_serv:
            res.append(res_serv.PR_strStatus)
        else:
            res.append('serv_SprStatus: статус не определен')

        # тестирование выбороса исключения через параметр statusID
        res_serv = serv_SprStatus('getStatus_or_None', 'qust-simp_')
        if res_serv:
            res.append(res_serv.PR_strStatus)
        else:
            res.append('serv_SprStatus: статус не определен ')

        #res_serv = getStatus_or_None(**dict(statusID='proj-memb'))
        #if res_serv:
        #    res.append(str(res_serv))
        #else:
        #    res.append('serv_SprStatus: статус не определен')

        res_serv = getStatus_or_None('proj-memb')
        if res_serv:
            res.append('## ' + res_serv.PR_strStatus)
        else:
            res.append('serv_SprStatus: статус не определен')

        return res
    except Exception as ex:
        return str(ex)

def test_get_permModf_prof():
    from advuser.serv_sprstatus import get_permModf_prof
    from app import getUser

    try:
        res = []

        user = 'colch'

        # список доступных значений из SprStatus.pk
        lst_status = ('notstatus','qust-simp','qust-regs','proj-sadm','proj-memb','proj-head','subheader','headerexp')

        for item in lst_status:
            res_prof = get_permModf_prof(user, 'upd_{0}'.format( item) )
            if res_prof:
                res.append('# {0} ##'.format( str(res_prof.res_obj)))
            else:
                res.append('{0} {1}: права не определены'.format(user, item))

        return res
    except Exception as ex:
        return str(ex)


from matplotlib import pyplot as plt
import genetic
import numpy as np

'''
an example of a genetic algorithm optimizing the parameters of a linear equation to find the
line of best fit for given points

there are only two parameters that are being optimized, but is easily scaleable;
    slope(flaot): between -10 and 10 --- this may need to be changed if the best fit line has a more extreme slope

    y-int(flaot): between -10 and 10 --- may also need to be changed depending on the location of the data

feel free to modify/add points or number of generations evolved

ethan gilmore
10/17/19
'''

x_points = [1, 1.5, 1.9, 2.1, 2.75, 4]
y_points = [5, 4.3, 2.4, 3.7, 2.5, 1]

'''
this is the structure for defining a set of parameters, or "gene_structure"

your parameters will be represented inside a dictionary with the keys named what the values represent
the value for each pair will be a 3-tuple

if you want your value to be a
    float: (float, min, max)
        float defines the type, but replace the min and the max fo set the bounds for the variable

    int: (int, min, max)
        int defines the type, replace min and max to set bounds

    list: (list, length, type)
        a list is a little different. the first element will be list to define it as a list
        second is the length, or number of things in the list
        third is the type of variable in the list - a float or an int

        example:
            {"my_list": (list, 10, (int, 1, 10))}
            this will create a paramter called my_list which is a list of length 10 filled with ints between 1 and 10
'''
parameters = {"slope": (float, -10, 10), "y-int": (float, -10, 10)}

'''
this is the structure for a fitness function

a fitness function is what determines how good a set of parameters (reffered to as an organism) is.
your fitness function should consist of;

Args:
    parameters / genes(dict): the actual values of an organism, modeled to your predefined structure

Returns:
    (int or float): returns a value specifying how good a set of paramters performed

    keep in mind that organisms (sets of parameters) with the HIGHEST RETURN VALUE when passed through
    this function are the ones kept and chosen to reproduce
'''

def loss(parameters):
    mean_squared_error = 0
    #find the total error of the line and the points
    for i, x in enumerate(x_points):
        #find the error between the actual y and the y of the function built with slope and y-int
        error = y_points[i] - (x * parameters['slope'] + parameters['y-int'])
        #decrease the mean squared error, because i want the set of parameters with the least difference to be greater
        #i.e. those with a greater error will be have a more negative fitness score
        mean_squared_error -= error**2
    return mean_squared_error

#create a population of size 20 with the paremeters and loss function we created
pop = genetic.Population(20, parameters, loss)
[pop.evolve() for i in range(10000)]

#set the slope and y-int to the organism with the best fitness
slope = pop.organisms[-1]['slope']
y_int = pop.organisms[-1]['y-int']

#used to plot points and the corrosponding line of best fit
plt.plot(x_points, y_points, 'ro')
x = np.linspace(-5,5,100)
y = slope*x+y_int
plt.plot(x, y)
plt.title(f"y = {round(slope, 2)}x + {round(y_int, 2)}")
plt.show()


class Fruit:
    price = 0                                   # 类变量

    def __init__(self):
        self.__color = "red"                      # 定义私有变量

    def getColor(self):
        print(self.__color)                       # 打印私有变量

    @ staticmethod                              # 使用@staticmethod修饰器静态方法
    def getPrice():
        print(Fruit.price)

    def __getPrice():                            # 定义私有函数
        Fruit.price = Fruit.price + 10
        print(Fruit.price)

    count = staticmethod(__getPrice)                # 使用staticmethod方法定义静态方法

if __name__ == "__main__":
    apple = Fruit()                              # 实例化apple
    apple.getColor()                            # 使用实例调用静态方法
    Fruit.count()                                # 使用类名调用静态方法
    banana = Fruit()
    Fruit.count()
    Fruit.getPrice()


#!/usr/bin/env python3

from math import sqrt, sin, cos, pi

radius_x = 280/2
radius_y = 192/2

def f(t):
    return (sqrt(t)*cos(2*pi*sqrt(t)), 0.87*sqrt(t)*sin(2*pi*sqrt(t)))

coords = []
for i in range(1000000):
    any = False
    a, b = f(float(i)/10.0)
    x = round(radius_x+a)
    y = round(radius_y+b)
    if y % 2 != 0:
        continue
    if x < 0 or x >= radius_x or y < 0 or y >= radius_y:
        continue
    coords.append((x,y))

d = {}
unique_coords = []
for c in coords:
    if not d.get(c):
        unique_coords.append(c)
        d[c] = 1

unique_vals = []
for x, y in unique_coords:
    aval = "$" + hex(y)[2:].rjust(2, "0").upper()
    bval = "%" + \
        bin(x%7)[2:].rjust(3, "0") + \
        bin(x//7)[2:].rjust(5, "0")
    unique_vals.append((aval, bval))

with open("../../../src/fx/fx.hgr.iris.data.a", "w") as f:
    for aval, bval in unique_vals:
        f.write("         !byte %s,%s\n" % (aval, bval))

unique_vals.reverse()
with open("../../../src/fx/fx.hgr.iris.in.data.a", "w") as f:
    for aval, bval in unique_vals:
        f.write("         !byte %s,%s\n" % (aval, bval))

unique_vals.reverse()
ripple_vals = []
for i, j, k, l in zip(range(1680), range(1680,3360), range(3360,5040), range(5040,6720)):
    ripple_vals.append(unique_vals[i])
    ripple_vals.append(unique_vals[j])
    ripple_vals.append(unique_vals[k])
    ripple_vals.append(unique_vals[l])
with open("../../../src/fx/fx.hgr.ripple.data.a", "w") as f:
    for aval, bval in ripple_vals:
        f.write("         !byte %s,%s\n" % (aval, bval))

corner4_vals = []
for x, y in unique_coords:
    x = 139-x
    y = 95-y
    aval = "$" + hex(y)[2:].rjust(2, "0").upper()
    bval = "%" + \
        bin(x%7)[2:].rjust(3, "0") + \
        bin(x//7)[2:].rjust(5, "0")
    corner4_vals.append((aval, bval))
with open("../../../src/fx/fx.hgr.corner4.in.data.a", "w") as f:
    for aval, bval in corner4_vals:
        f.write("         !byte %s,%s\n" % (aval, bval))
corner4_vals.reverse()
with open("../../../src/fx/fx.hgr.corner4.out.data.a", "w") as f:
    for aval, bval in corner4_vals:
        f.write("         !byte %s,%s\n" % (aval, bval))


import random

class Player:
    """Keeps track of the last card displayed. Calculates score change and
    draws next card. Gets player input.

    Attributes:
        current_card (int): The last card displayed.
    """

    def __init__(self):
        """The class constructor.

        Args:
            self (Player): an instance of Player.
        """
        self.current_card = random.randint(1, 13)

    def get_points(self, player_guess):
        """Calculates the change in points and draws and returns next card.

        Args:
            self (Player): an instance of Player.
            player_guess (str): guess from player.
        """
        next_card = random.randint(1, 13)
        return_score = 0
        if self.current_card > next_card and player_guess == "l":
            return_score = 100
        elif self.current_card < next_card and player_guess == "h":
            return_score = 100
        elif self.current_card == next_card:
            return_score = 0
        else:
            return_score = -75
        self.current_card = next_card
        return return_score, next_card

    def get_guess(self):
        """Get a guess of high (h) or low (l) from the player

        Args:
            self (Player): an instance of Player.
        """
        guess = input ("Higher or lower? [h/l] ")
        return guess


import click
from six.moves import map, filter

from .gitlab import GitlabApi

SUPPORTED_INTEGRATION_LIST = ['gitlab']

def get_integration_list(ctx):
    global_cfg = ctx.obj['global']
    return map(lambda x: x.name, global_cfg.sections())

def get_integration(ctx, integration_type, name):
    return ctx.obj['global'].section("{}:{}".format(integration_type, name))

def add_integration(ctx, integration_type, integration_name=None):
    global_cfg = ctx.obj['global']
    enabled_integration_list = ctx.obj['base']['integrations']
    if integration_name:
        enabled_integration = '{}:{}'.format(integration_type, integration_name)
        if enabled_integration not in enabled_integration_list:
            click.secho('You don\'t configure integration {} for {}, please do it first'
                        .format(integration_name, integration_type),
                        fg=ctx.obj['base']['error_color'])
            exit(127)
    else:
        enabled_integration = next(filter(lambda x: x.startswith(integration_type), enabled_integration_list), None)
        if not enabled_integration:
            click.secho('You don\'t configure any integration for {}, please do it first'
                        .format(integration_type),
                        fg=ctx.obj['base']['error_color'])
            exit(127)
    integration_obj = False
    integration_section = global_cfg.section(enabled_integration)
    if integration_type == 'gitlab':
        integration_obj = GitlabApi(integration_section.get('url'),
                                    integration_section.get('token'),
                                    integration_section.get('cert_ignore'))
    ctx.obj['integration'] = integration_obj

def create_gitlab_integration(ctx):
    name = None
    global_cfg = ctx.obj['global']
    while not name:
        name = click.prompt('Please, provide integration name')
        if global_cfg.section('gitlab:' + name):
            click.secho("This name already exists, please select another one", fg=ctx.obj['base']['error_color'])
            name = None
    url = click.prompt('Please, provide gitlab url')
    token = click.prompt('Please, provide gitlab token')
    cert_ignore = click.confirm('Certification check ignore')
    test_success = False
    while not test_success:
        gitlab_api = GitlabApi(url, token, cert_ignore=cert_ignore)
        test_success = gitlab_api.test()
        if not test_success:
            click.secho('Test wan\'t successful, maybe, some configuration error', fg=ctx.obj['base']['error_color'])
            test_success = click.confirm('Enable test ignore')
            if not test_success:
                url = click.prompt('Please, provide gitlab url', default=url)
                token = click.prompt('Please, provide gitlab token', default=token)
                if not cert_ignore:
                    cert_ignore = click.confirm('Certification check ignore')
    new_section = global_cfg.section('gitlab:' + name)
    new_section['url'] = url
    new_section['token'] = token
    new_section['cert_ignore'] = cert_ignore
    global_cfg.sync()
    click.secho('Gitlab integration named {} successful created'.format(name), fg=ctx.obj['base']['good_color'])

INTEGRATION_CREATE_DICT = {
    'gitlab': create_gitlab_integration
}


import os
from flask import (Flask, render_template)
from datetime import timedelta

def create_app(test_config=None):
    # create and configure the app
    app = Flask(__name__, instance_relative_config=True)
    app.config['SEND_FILE_MAX_AGE_DEFAULT'] = timedelta(seconds=1)

    app.config.from_mapping(
        SECRET_KEY='dev',
        DATABASE=os.path.join(app.instance_path, 'flask_r.sqlite'),
    )

    from flask_app.flaskr import auth, blog
    app.register_blueprint(auth.bp)
    app.register_blueprint(blog.bp)
    app.add_url_rule('/', endpoint='index')

    return app

if __name__ == '__main__':
    create_app().run()


import os
import configparser

class ReadIni:

    def get_value(self, section, key):
        curpath = os.path.abspath(os.path.join(os.getcwd(), "../config"))
        cfgpath = os.path.join(curpath, 'LocalElement.ini')
        # 创建管理对象
        conf = configparser.ConfigParser()

        # 读ini文件
        conf.read(cfgpath)

        # 获取所有的section
        sections = conf.sections()
        print(sections)

        # 获取指定section下面的key/value值
        # items = conf.items('LoginElement')
        value = conf[section][key]
        return value

ri = ReadIni()
ReadIni.get_value(ri, 'LoginElement', 'username')


import server_socket
import threading
import os
import time

class Steer(object):

    def __init__(self, client):
        self.client = client
        self.ultrasonic = 9999
        self.microphone = 'NONE'
        self.line = 'NONE'
        self.obj_list = []
        self.stopline = False
        self.state = 'NONE'
        self.speed = '0'
        self.light_signal = 'NONE'
        self.obs = 'NO'
        self.stoptime = 0
        self.sign = 0
        self.green = 0

    def Set_UltraSonic(self, ultra) :
        if int(ultra) > 1000 :
            return
        self.ultrasonic = int(ultra)

    def Set_Microphone(self, mic) :
        self.microphone = mic

    def Set_Line(self, line) :
        self.line = line

    def Set_Stopline(self, result):
        self.stopline = result

    def Set_ObjectDetection(self, obj) :
        self.obj_list = obj

    def ultrasonic_process(self, ultra) :
        if ultra < 30 :
            return 's'
        else :
            return 'w'

    # def mic_process(self, speech) :
    #     if (speech == '정지') or (speech == '멈춰') or (speech == '멈추라고') :
    #         return 's'
    #     elif (speech == '가') or (speech == '가라고') or (speech == '출발') :
    #         return 'w'
    #     else :
    #         return ''

    def Control(self) :
#        mic_comm = self.mic_process(self.microphone)
#        us_comm = self.ultrasonic_process(self.ultrasonic)

        #os.system('clear')
        print('속도 : ', self.speed)
        print('상태 : ', self.state)
        print('전방 거리 : ', self.ultrasonic)
        print('신호 명령 : ', self.light_signal)
        print('음성 명령 : ', self.microphone)
        print('정지 명령 : ', self.stopline)
        print('장애물 : ', self.obs)
# ------------------------------------------------------------------------------------
        # speed limit
        if ('speed_limit' in self.obj_list) and (self.sign == 0):
            if self.speed != '50' :
                self.client.send('30'.encode())
                self.speed = '30'
                self.sign = 1
                self.green = 0

        elif 'limit60' in self.obj_list :
            print('limit 60')
            if self.speed != '60' :
                self.client.send('60'.encode())
                self.speed = '60'

# ------------------------------------------------------------------------------------
        # stop
        # it's only for different sound..
        if (self.stopline == True) and self.stoptime == 0 :
            if self.state != 'STOP_LINE' :
                self.stoptime = time.time()
                self.client.send('ss'.encode())
                self.state = 'STOP_LINE'
                print('STOP LINE')
                time.sleep(2)
# ------------------------------------------------------------------------------------

        if self.stopline == True and (time.time() - self.stoptime) > 10 :
            self.stoptime = 0
# ------------------------------------------------------------------------------------
        if ('red_light' in self.obj_list) :
            if (self.light_signal != 'LIGHT_STOP') or (mic_comm == 'w') :
                self.client.send('ls'.encode())
                self.light_signal = 'LIGHT_STOP'
                self.state = 'STOP'
                print('RED LIGHT')
                self.sign = 0

        # elif (mic_comm == 'w') :
        #     if self.state == 'STOP' and self.obs != 'YES' :
        #         self.client.send('lw'.encode())
        #         self.state = 'DRIVE'
        #
        # elif (mic_comm == 's') :
        #     if self.state != 'STOP' :
        #         self.client.send('s'.encode())
        #         self.state = 'STOP'
        #         print('STOP')
        #
        # elif (us_comm == 's') :
        #     if self.obs != 'YES' :
        #         self.client.send('us'.encode())
        #         self.obs = 'YES'
        #         self.state = 'STOP'
        #         print('STOP')
        # elif (us_comm == 'w') :
        #     if self.light_signal != 'LIGHT_STOP' and self.obs != 'NO' :
        #         self.obs = 'NO'
        #         self.state = 'DRIVE'
        #         self.client.send('w'.encode())
# ------------------------------------------------------------------------------------
        if ('green_light' in self.obj_list) and self.green == 0 :
            self.light_signal = 'DRIVE'
            self.state = 'DRIVE'
            self.client.send('lw'.encode())
            self.sign = 0
            self.green = 1

# ------------------------------------------------------------------------------------
        # driving
        if (self.light_signal != 'LIGHT_STOP') :
            if ('green_light' in self.obj_list) and self.green == 0:
                self.green = 1
                self.client.send('lw'.encode())
                self.light_signal = "DRIVE"

            if self.state != 'STOP' :
                self.state = 'DRIVE'
                if 'turn_right_yes' in self.obj_list :
                    self.client.send('td'.encode())
                    print("RIGHT")
                elif self.line == 2:
                    self.client.send('w'.encode())
                    print("FORWARD")
                elif self.line == 0:
                    self.client.send('a'.encode())
                    print("LEFT")
                elif self.line == 1:
                    self.client.send('d'.encode())
                    print("RIGHT")
                else :
                    print("WAIT..")
# ------------------------------------------------------------------------------------
        self.microphone = ''
        print("Sign:",self.sign, "green", self.green, "state:", self.state)


import logging
import os
import subprocess
import zipfile
from abc import ABC

import requests
from django.conf import settings
from django.db import connections
from django.utils import timezone

# import psycopg2
from data_ocean.models import RegistryUpdaterModel, Dataset

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

class Downloader(ABC):
    auth = None
    url = None
    data = {}
    headers = {}
    local_path = settings.LOCAL_FOLDER
    chunk_size = 8 * 1024 * 1024
    stream = True
    reg_name = ''
    file_name = ''
    file_path = ''
    file_size = 0
    check_by_length = False
    zip_required_file_sign = ''
    unzip_required_file_sign = ''
    unzip_after_download = False
    log_obj = None
    start_time = None
    source_dataset_url = ''

    def __init__(self):
        assert self.reg_name
        assert self.local_path
        self.url = self.get_source_file_url()
        assert self.url
        self.file_name = self.get_source_file_name()
        assert self.file_name
        self.file_path = self.local_path + self.file_name

    def get_headers(self):
        return self.headers

    def is_zip_file(self):
        if not zipfile.is_zipfile(self.file_path):
            msg = f'Error! {self.file_path} is not a Zip file!'
            self.log_obj.unzip_message = msg
            self.log_obj.save()
            logger.exception(f'{self.reg_name}: {msg}')
            raise Exception('Unzip error!', msg)
        return True

    def test_zip_file(self):
        logger.info(f'{self.reg_name}: Test archive {self.file_path} start...')
        test_error = subprocess.run(['unzip', '-t', self.file_path]).returncode
        if test_error:
            msg = f'Error! Test archive {self.file_path} failed!'
            self.log_obj.unzip_message = msg
            self.log_obj.save()
            logger.exception(f'{self.reg_name}: {msg}')
            raise Exception('Unzip error!', msg)
        logger.info(f'{self.reg_name}: Test archive finished successfully.')
        return True

    def get_zip_filelist(self):
        file_list = zipfile.ZipFile(self.file_path).namelist()
        logger.info(f'{self.reg_name}: Archive file list: {file_list}')
        return file_list

    def unzip_result(self, i):
        logger.info(f'{self.reg_name}: Unzipping {self.file_path} ...')
        unzip_status = subprocess.run(['unzip', '-o', self.file_path, i, '-d', self.local_path]).returncode
        if not unzip_status:
            logger.info(f'{self.reg_name}: Unzipping {self.file_path} finished successfully.')
            self.remove_file()
            self.file_name = i
            self.file_path = self.local_path + self.file_name
            self.log_obj.unzip_file_name = self.file_name
            self.log_obj.unzip_status = True
            self.log_obj.save()
            return True

        msg = f'Error! Unzip failed for {self.file_path}! Return status: {unzip_status}!'
        self.log_obj.unzip_message = msg
        self.log_obj.save()
        logger.exception(f'{self.reg_name}: {msg}')
        raise Exception('Error!', msg)

    def no_req_sign(self):
        msg = f'Error! Unzip failed for {self.file_path}! File with required sign not found!'
        self.log_obj.unzip_message = msg
        self.log_obj.save()
        logger.exception(f'{self.reg_name}: {msg}')
        raise Exception('Error!', msg)

    def unzip_source_file(self):
        self.is_zip_file()
        self.test_zip_file()
        for i in self.get_zip_filelist():
            if self.unzip_required_file_sign in i:
                if self.unzip_result(i):
                    return
        self.no_req_sign()

    def file_size_is_correct(self):
        if os.path.isfile(self.file_path):
            return os.path.getsize(self.file_path) == self.file_size

    def remove_file(self):
        if os.path.isfile(self.file_path):
            os.remove(self.file_path)

    def log_init(self):
        self.log_obj = RegistryUpdaterModel.objects.create(registry_name=self.reg_name)

    def get_source_file_url(self):
        assert self.url
        return self.url

    def get_source_file_name(self):
        assert self.file_name
        return self.file_name

    def vacuum_analyze(self, table_list=None):
        query = 'select count(*) from'
        query_optimize = 'vacuum analyze'

        with connections['default'].cursor() as c:
            for table in table_list:
                start_time = timezone.now()
                c.execute('%s %s' % (query_optimize, table))
                logger.info(f'{self.reg_name}: {query_optimize} {table} at {timezone.now() - start_time}')
                start_time = timezone.now()
                c.execute('%s %s' % (query, table))
                logger.info(f'{self.reg_name}: {query} {table} at {timezone.now() - start_time}')

    def download(self):
        assert self.url
        assert self.file_path

        start_time = timezone.now()
        try:
            with requests.get(self.url, data=self.data, stream=self.stream,
                              auth=self.auth, headers=self.get_headers()) as r:
                r.raise_for_status()

                self.file_size = int(r.headers['Content-Length'])
                logger.info(f"{self.reg_name}: Start downloading: {self.file_path}, size: {self.file_size} ...")
                self.remove_file()
                with open(self.file_path, 'wb') as f:
                    for chunk in r.iter_content(chunk_size=self.chunk_size):
                        f.write(chunk)
                    f.flush()

                if not self.file_size_is_correct():
                    raise requests.exceptions.RequestException('Error! Bad file size after download.')

                logger.info(
                    f"{self.reg_name}: {self.file_path} downloaded successfully at {timezone.now() - start_time}.")

                self.log_obj.download_finish = timezone.now()
                self.log_obj.download_status = True
                self.log_obj.download_file_name = self.file_path
                self.log_obj.download_file_length = self.file_size
                self.log_obj.save()

        except requests.exceptions.RequestException as e:

            self.log_obj.download_finish = timezone.now()
            self.log_obj.download_status = False
            self.log_obj.download_message = f'{e}'[:255]
            self.log_obj.save()
            logger.exception(f'{self.reg_name}: {e}')
            raise Exception('Error!', e)

        if self.unzip_after_download:
            self.unzip_source_file()

    def update_total_records(self, dataset_name, total_records):
        dataset = Dataset.objects.get(name=dataset_name)
        dataset.total_records = total_records
        dataset.save(update_fields=['total_records'])


# split based on valid tlds
import os
import sys

valid_tld_set = set([l.strip().lower() for l in open(os.path.join(os.path.dirname(__file__), 'valid_tld.lst'), 'r')])

inf_name = sys.argv[1]

valid_file = open(inf_name+'.valid.tld', 'w')
invalid_file = open(inf_name+'.invalid.tld', 'w')

for l in open(inf_name, 'r'):
    e = l.strip()
    tld = e.split('@')[-1].lower()
    if tld in valid_tld_set:
        print >>valid_file, e
    else:
        print >> invalid_file, e

valid_file.close()
invalid_file.close()