class TreeNode(object):
    def __init__(self, x):
        self.val = x
        self.left = None
        self.right = None

class ParentTreeNode(object):
    def __init__(self, current, parent):
        self.current = current
        self.parent = parent

def lowestCommonAncestor(root, p, q):
    qu = [ParentTreeNode(root, None)]
    pp = ParentTreeNode(root, None)
    qp = ParentTreeNode(root, None)
    find_p = False
    find_q = False
    while qu and (not find_p or not find_q):
        t = qu.pop(0)
        left_node = t.current.left
        if left_node:
            lnp = ParentTreeNode(left_node, t)
            qu.append(lnp)

            if left_node.val == p.val:
                pp = lnp
                find_p = True

            if left_node.val == q.val:
                qp = lnp
                find_q = True

        right_node = t.current.right
        if right_node:
            rnp = ParentTreeNode(right_node, t)
            qu.append(rnp)

            if right_node.val == p.val:
                pp = rnp
                find_p = True

            if right_node.val == q.val:
                qp = rnp
                find_q = True

    parents = {p.val, q.val}
    pp = pp.parent
    qp = qp.parent

    while pp or qp:
        if pp:
            if pp.current.val in parents:
                return pp.current.val
            parents.add(pp.current.val)
            pp = pp.parent
        if qp:
            if qp.current.val in parents:
                return qp.current.val
            parents.add(qp.current.val)
            qp = qp.parent

def stringToTreeNode(input):
    input = input.strip()
    input = input[1:-1]
    if not input:
        return None

    inputValues = [s.strip() for s in input.split(',')]
    root = TreeNode(int(inputValues[0]))
    nodeQueue = [root]
    front = 0
    index = 1
    while index < len(inputValues):
        node = nodeQueue[front]
        front = front + 1

        item = inputValues[index]
        index = index + 1
        if item != "null":
            leftNumber = int(item)
            node.left = TreeNode(leftNumber)
            nodeQueue.append(node.left)

        if index >= len(inputValues):
            break

        item = inputValues[index]
        index = index + 1
        if item != "null":
            rightNumber = int(item)
            node.right = TreeNode(rightNumber)
            nodeQueue.append(node.right)
    return root

r = stringToTreeNode("[3,5,1,6,2,0,8,null,null,7,4]")

p = stringToTreeNode("[5]")
q = stringToTreeNode("[1]")
k = lowestCommonAncestor(r, p, q)
print(k)


from manager import Manager
from robot import Robot
def atTheWorkStaion(worker):
	print("At the Work Station")
	worker.startWork()
	worker.stopWork()

def atTheCafeteria (worker):
	print("At the Cafeteria")
	worker.startEating()
	worker.stopEating()
manager=Manager()
robot=Robot()
atTheWorkStaion(manager)
atTheWorkStaion(robot)

atTheCafeteria(manager)
atTheCafeteria(robot)


# feeds.py

import dbm.gnu
import collections
import json
import logging
import os
import sys
import time

import feedparser
import yaml

import tornado.gen
import tornado.httpclient
import tornado.options
import tornado.process

from modules.__common__ import shorten_url, strip_html

# Metadata

NAME     = 'feeds'
ENABLE   = True
TYPE     = 'timer'
TEMPLATE = 'From {color}{green}{feed}{color} feed: {bold}{title}{bold} by {color}{cyan}{author}{color} @ {color}{blue}{link}{color}'

# Timer

@tornado.gen.coroutine
def timer(bot):
    # Execute feeds script
    bot.logger.info('Executing %s', __file__)
    command = ['python3', __file__, '--config-dir={}'.format(bot.config_dir)]
    environ = dict(os.environ, **{'PYTHONPATH': os.path.join(__file__, '..', '..') + ':' + os.environ.get('PYTHONPATH', '')})
    process = tornado.process.Subprocess(command, stdout=tornado.process.Subprocess.STREAM, env=environ)
    results = yield tornado.gen.Task(process.stdout.read_until_close)

    # Read configuration
    config_path      = os.path.join(bot.config_dir, 'feeds.yaml')
    feeds_config     = yaml.safe_load(open(config_path))
    templates        = feeds_config.get('templates', {})
    default_template = templates.get('default', TEMPLATE)

    # Read and process results
    cache_path = os.path.join(bot.config_dir, 'feeds.cache')

    with dbm.open(cache_path, 'c') as feeds_cache:
        for feed, entries in json.loads(results).items():
            for entry in entries:
                title    = entry['title'].replace('\r', ' ').replace('\n', ' ')
                key      = entry['link'].encode('ascii', 'ignore')
                link     = yield shorten_url(entry['link'])
                author   = entry['author']
                channels = entry['channels']

                # Send each entry to the appropriate channel
                for channel in channels:
                    template = templates.get(channel, default_template)
                    message  = bot.format_text(template,
                            feed   = feed,
                            title  = title,
                            link   = link,
                            author = author
                    )
                    bot.send_message(message, channel=channel)

                # Mark entry as delivered
                bot.logger.info('Delivered %s from %s to %s', title, feed, ', '.join(channels))
                feeds_cache[key] = str(time.time())

# Register

def register(bot):
    config_path  = os.path.join(bot.config_dir, 'feeds.yaml')
    feeds_config = yaml.safe_load(open(config_path))
    timeout      = feeds_config.get('timeout', 5*60)

    return (
        (timeout, timer),
    )

# Script

def script(config_dir):
    config_path  = os.path.join(config_dir, 'feeds.yaml')
    feeds_config = yaml.safe_load(open(config_path))

    cache_path   = os.path.join(config_dir, 'feeds.cache')
    feeds_cache  = dbm.open(cache_path, 'c')

    entries      = collections.defaultdict(list)
    entries_limit= feeds_config.get('limit', 5)
    logger       = logging.getLogger()

    for feed_config in feeds_config['feeds']:
        feed_title    = feed_config['title']
        feed_url      = feed_config['url']
        feed_channels = feed_config['channels']
        feed_key      = feed_title.encode('ascii', 'ignore')

        logger.info('Fetching %s (%s)', feed_title, feed_url)

        client  = tornado.httpclient.HTTPClient()
        request = tornado.httpclient.HTTPRequest(feed_url, request_timeout=10)
        try:
            result = client.fetch(request)
        except Exception as e:
            logger.warning(e)
            continue

        logger.info('Parsing %s (%s)', feed_title, feed_url)

        for entry in feedparser.parse(result.body)['entries']:
            link   = entry.get('link', '')
            title  = strip_html(entry.get('title', ''))
            author = entry.get('author', 'Unknown')
            key    = link.encode('ascii','ignore')

            # If there is no key, then skip
            if not key:
                logger.debug('No key for %s', link)
                continue

            # If key is in cache and the timestamp is > 1.0, then skip
            if key in feeds_cache and float(feeds_cache[key]) > 1.0:
                logger.debug('Skipping %s (already %s)', link, feeds_cache[key].decode())
                continue

            # If feed is not in cache, then it is new, so mark all entries as skipped
            if feed_key not in feeds_cache:
                logger.debug('Skipping %s (new feed)', link)
                feeds_cache[key] = str(time.time())
                continue

            # If date published is too old, then mark and skip recording
            timestamp = entry.get('updated_parsed', entry.get('published_parsed', None))
            timestamp = time.mktime(timestamp) if timestamp else time.time()
            if time.time() - timestamp > 24*60*60:
                logger.debug('Skipping %s (too old)', link)
                feeds_cache[key] = str(time.time())
                continue

            # Record entry with a key of 1.0 and then add to list of items
            logger.info('Recording %s', link)
            feeds_cache[key] = str(1.0)
            entries[feed_title].append({
                'title'     : title,
                'author'    : author,
                'link'      : link,
                'channels'  : feed_channels,
                'timestamp' : timestamp,
            })

        # Mark feed in cache
        feeds_cache[feed_key] = str(time.time())

        # Only report up to rss_limit entries for this feed
        entries[feed_title] = entries[feed_title][0:entries_limit]

    for feed_title, feed_entries in entries.items():
        entries[feed_title] = list(sorted(feed_entries, key=lambda e: e['timestamp']))

    json.dump(entries, sys.stdout)

# Main Execution

if __name__ == '__main__':
    tornado.options.define('config_dir', default=None,  help='Configuration directory')
    tornado.options.parse_command_line()
    options = tornado.options.options.as_dict()

    script(os.path.expanduser(options.get('config_dir', '~/.config/bobbit')))

# vim: set sts=4 sw=4 ts=8 expandtab ft=python:


from django.test import TestCase
from apps.logistics.models import *
from apps.business.models import Department

# Create your tests here.
class CoffeStockAssertionErrorTestCase(TestCase):
    def setUp(self):
        Department.objects.create(name='planta_baja')
        CoffeType.objects.create(name='naranjada', current_price='0.15')
        # Ya se ha creado un CoffeStock de ambos.

    def test_coffe_stock_assertion_error(self):
        department = Department.objects.get(name='planta_baja')
        coffe_type = CoffeType.objects.get(name='naranjada')
        with self.assertRaises(AssertionError) as e:
            CoffeStock.objects.create(department=department, coffe_type=coffe_type)
        self.assertTrue('Ya existe un CoffeStock para ese par Deparament/CoffeType' in str(e.exception))

class CoffeTypeSaveTestCase(TestCase):
    def setUp(self):
        Department.objects.create(name='terraza')
        Department.objects.create(name='planta_baja')
        CoffeType.objects.create(name='cappuccino', current_price=1.5) # Aqui me tiene que crear los dos CoffeStocks.

    def test_coffe_stocks_are_created(self):
        coffe_type = CoffeType.objects.get(name='cappuccino')
        self.assertEqual(CoffeStock.objects.filter(coffe_type=coffe_type, department=Department.objects.get(name='terraza')).count(), 1)
        self.assertEqual(CoffeStock.objects.filter(coffe_type=coffe_type, department=Department.objects.get(name='planta_baja')).count(), 1)

class CoffeOrderAreCreatedWithStock0(TestCase):
    def setUp(self):
        CoffeType.objects.create(name='naranjada', current_price=1.2)
        Department.objects.create(name='terraza')
        #Aqui ya me ha creado un CoffeStock.

    def test_when_stock_is_created(self):
        coffe_stock = CoffeStock.objects.get(department=Department.objects.get(name='terraza'))
        self.assertTrue(CoffeOrder.objects.filter(coffe_stock=coffe_stock, unit_price=1.2, units=100, status='Pending').count()==1)

class CoffeOrderAreCreatedWithStock2(TestCase):
    def setUp(self):
        CoffeType.objects.create(name='naranjada', current_price=1.2)
        Department.objects.create(name='terraza')
        #Aqui ya me ha creado un CoffeStock.
        coffe_stock = CoffeStock.objects.get(department=Department.objects.get(name='terraza'))
        coffe_stock.current_units = 2
        coffe_stock.save()

    def test_when_stock_is_decreased(self):
        # Solo tenemos que tener una orden.
        coffe_stock = CoffeStock.objects.get(department=Department.objects.get(name='terraza'))
        self.assertTrue(CoffeOrder.objects.filter(coffe_stock=coffe_stock, unit_price=1.2, units=100, status='Pending').count()==1)

        # Al disminuir  el stock a 0, se tiene que crear otra orden.
        coffe_stock.current_units = 0
        coffe_stock.save()
        self.assertTrue(CoffeOrder.objects.filter(coffe_stock=coffe_stock, unit_price=1.2, units=100, status='Pending').count()==2)


from typing import Union

from hwt.hdl.operator import Operator
from hwt.hdl.operatorDefs import AllOps
from hwt.hdl.types.defs import BIT, INT
from hwt.hdl.value import Value
from hwt.serializer.exceptions import UnsupportedEventOpErr
from hwt.serializer.generic.context import SerializerCtx
from hwt.synthesizer.rtlLevel.rtlSignal import RtlSignal

class VerilogSerializer_ops():
    # http://www.asicguru.com/verilog/tutorial/operators/57/
    opPrecedence = {
        AllOps.RISING_EDGE: 17,
        AllOps.FALLING_EDGE: 17,
        AllOps.TERNARY: 16,
        AllOps.AND: 11,
        AllOps.XOR: 11,
        AllOps.OR: 11,
        AllOps.EQ: 10,
        AllOps.NEQ: 10,
        AllOps.GT: 9,
        AllOps.LT: 9,
        AllOps.GE: 9,
        AllOps.LE: 9,
        AllOps.CONCAT: 5,
        AllOps.ADD: 7,
        AllOps.SUB: 7,
        AllOps.DIV: 6,
        AllOps.NEG: 5,
        AllOps.MUL: 3,
        AllOps.NOT: 3,
        AllOps.DOWNTO: 2,
        AllOps.TO: 2,
        AllOps.CALL: 2,
        AllOps.INDEX: 1,
        # AllOps.SHIFTL:8,
        # AllOps.SHIFTR:8,
        AllOps.BitsAsSigned: 1,
        AllOps.BitsAsUnsigned: 1,
        AllOps.BitsAsVec: 1,
    }
    _unaryOps = {
        AllOps.NOT: "~%s",
        AllOps.BitsAsSigned: "$signed(%s)",
        AllOps.BitsAsUnsigned: "$unsigned(%s)",
        AllOps.BitsAsVec: "%s",
    }

    _binOps = {
        AllOps.AND: '%s & %s',
        AllOps.OR: '%s | %s',
        AllOps.XOR: '%s ^ %s',
        AllOps.CONCAT: "{%s, %s}",
        AllOps.DIV: '%s / %s',
        AllOps.DOWNTO: '%s:%s',
        AllOps.TO: '%s:%s',
        AllOps.EQ: '%s == %s',
        AllOps.GT: '%s > %s',
        AllOps.GE: '%s >= %s',
        AllOps.LE: '%s <= %s',
        AllOps.LT: '%s < %s',
        AllOps.SUB: '%s - %s',
        AllOps.MUL: '%s * %s',
        AllOps.NEQ: '%s != %s',
        AllOps.ADD: '%s + %s',
        AllOps.POW: '%s ** %s',
    }

    @classmethod
    def _operandIsAnotherOperand(cls, operand):
        if isinstance(operand, RtlSignal) and operand.hidden\
                and isinstance(operand.origin, Operator):
            return True

    @classmethod
    def _operand(cls, operand: Union[RtlSignal, Value], i: int,
                 operator: Operator,
                 expr_requires_parenthesis: bool,
                 cancel_parenthesis: bool,
                 ctx: SerializerCtx):

        # [TODO] if operand is concatenation and parent operator
        #        is not concatenation operand should be extracted
        #        as tmp variable
        #        * maybe flatten the concatenations
        if operator.operator != AllOps.CONCAT\
                and cls._operandIsAnotherOperand(operand)\
                and operand.origin.operator == AllOps.CONCAT:
            tmpVar = ctx.createTmpVarFn("tmp_concat_", operand._dtype)
            tmpVar.def_val = operand
            # Assignment(tmpVar, operand, virtual_only=True)
            operand = tmpVar

        oper = operator.operator
        width = None
        if oper not in [AllOps.BitsAsUnsigned, AllOps.BitsAsVec,
                        AllOps.BitsAsSigned] and\
                oper is not AllOps.INDEX and\
                operand._dtype == INT and\
                operator.result is not None and\
                not operator.result._dtype == INT:
            # has to lock width
            for o in operator.operands:
                try:
                    bl = o._dtype.bit_length
                except AttributeError:
                    bl = None
                if bl is not None:
                    width = bl()
                    break

            assert width is not None, (operator, operand)

        s = super()._operand(operand, i, operator,
                             width is not None or expr_requires_parenthesis,
                             width is None and cancel_parenthesis, ctx)
        if width is not None:
            return "%d'%s" % (width, s)
        else:
            return s

    @classmethod
    def Operator(cls, op: Operator, ctx):
        ops = op.operands
        o = op.operator

        op_str = cls._unaryOps.get(o, None)
        if op_str is not None:
            cancel_parenthesis = op_str.endswith(")")
            return op_str % (cls._operand(ops[0], 0, op, False, cancel_parenthesis, ctx))

        op_str = cls._binOps.get(o, None)
        if op_str is not None:
            return cls._bin_op(op, op_str, ctx, cancel_parenthesis=o == AllOps.CONCAT)
        if o == AllOps.CALL:
            return "%s(%s)" % (
                cls.FunctionContainer(ops[0]),
                # operand i does not matter as they are all in ()
                ", ".join(map(lambda _op: cls._operand(_op, 1, op, False, True, ctx), ops[1:])))
        elif o == AllOps.INDEX:
            return cls._operator_index(op, ctx)
        elif o == AllOps.TERNARY:
            zero, one = BIT.from_py(0), BIT.from_py(1)
            if ops[1] == one and ops[2] == zero:
                # ignore redundant x ? 1 : 0
                return cls.condAsHdl([ops[0]], True, ctx)
            else:
                op0 = cls.condAsHdl([ops[0]], True, ctx)
                op1 = cls._operand(ops[1], op, False, False, ctx)
                op2 = cls._operand(ops[2], op, False, False, ctx)
                return "%s ? %s : %s" % (op0, op1, op2)
        elif o == AllOps.RISING_EDGE or o == AllOps.FALLING_EDGE:
            raise UnsupportedEventOpErr()
        elif o in [AllOps.BitsAsUnsigned, AllOps.BitsAsVec]:
            op0, = ops
            do_cast = bool(op0._dtype.signed)
            op_str = cls._operand(op0, op, False, do_cast, ctx)
            if do_cast:
                return "$unsigned(%s)" % op_str
            else:
                return op_str
        else:
            raise NotImplementedError(
                "Do not know how to convert expression with operator %s to verilog" % (o))


"""BubbleSorting Algorithm"""

import numpy as np
import matplotlib.pyplot as  plt
import random as rn
import time as tm

def bubbleSortAscendingOrder(group):
    isSorted = False
    length = len(group)
    indexes = range(0,length-1)
    swaps = 0
    comparisons = 0

    while not isSorted:
        isSorted = True

        for i in indexes:
            temp = group[i]
            comparisons = comparisons +1

            if temp > group[i+1]:
                group[i] = group[i+1]
                group[i+1] = temp
                swaps = swaps +1
                isSorted = False

    return [group,comparisons,swaps]

def cuadratic(x):
    return x**2

size = list(range(1,1002,50))

rawResultsSwaps = []
resultsSwaps = []
rawResultsComps = []
resultsComps = []

model = []
raw = 0
"""startTime = tm.time()"""

for quant in size:
    model.append(cuadratic(quant))
    group = list(range(quant))

    rawResultsSwaps.append([])
    rawResultsComps.append([])
    repeats = 0
    while repeats < 50:
        rn.shuffle(group)
        info = bubbleSortAscendingOrder(group)

        comps = info[1]
        swaps = info[2]
        rawResultsComps[raw].append(comps)
        rawResultsSwaps[raw].append(swaps)
        repeats = repeats +1

    resultsSwaps.append(np.average(rawResultsSwaps[raw]))
    resultsComps.append(np.average(rawResultsComps[raw]))
    raw = raw +1

"""elapsedTime = tm.time() - startTime"""
"""print(tm.strftime("%H:%M:%S", tm.gmtime(elapsedTime)))"""

plt.figure()
plt.grid()
plt.plot(size,model,'b--',linewidth=2,label='Model of complexity')
plt.plot(size,resultsSwaps,'*g',markersize=4,label='Number of swaps')
plt.plot(size,resultsComps,'or',markersize=4,label='Number of comparisons')
plt.legend()
plt.xlim(0,1100)
plt.xlabel("n (number of distinct elements)")


class entity:
    arguments = {}
    # QUEUE VARS
    is_queue_form = False
    method_set, queue_set = [], {}
    def fast_get(self):
        for method_name in dir(self):
            if callable(getattr(self, method_name)):
                if method_name[0] != '_':
                    if getattr(self, method_name).__name__ == 'wrapper':
                        self.method_set.append(method_name)
        for method in self.method_set:
            self.start_function(method)
        temp = sorted(self.queue_set.items(), key=lambda x: x[1])
        _queue = [key for key, value in temp]
        self.is_queue_form = True
        for method in _queue:
            self.start_function(method)
    def start_function(self, name):
        f = getattr(self, name)
        try:
            if name in self.arguments:
                f(*self.arguments[name])
            else: f()
        except AttributeError: pass
    @classmethod
    def toFunName(cls,s):
        return s[0].upper() + s[1:]
    @classmethod
    def toVarName(cls,s):
        return s[0].lower() + s[1:]

# @ name - (sense) variable name
def add2queue(num, name):
    def real_decorator(fn):
        def wrapper(self, *args, **kwargs):
            fun_name = fn.__name__
            if not self.is_queue_form:
                self.queue_set[fun_name] = num
            else:
                if fun_name in self.arguments:
                    args = self.arguments[fun_name]
                value = fn(self, *args, **kwargs)
                setattr(self, name, value)
        return wrapper
    return real_decorator


#!/usr/bin/env python
# Software License Agreement (BSD License)
#
# Copyright (c) 2008, Willow Garage, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above
#    copyright notice, this list of conditions and the following
#    disclaimer in the documentation and/or other materials provided
#    with the distribution.
#  * Neither the name of Willow Garage, Inc. nor the names of its
#    contributors may be used to endorse or promote products derived
#    from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
# Revision $Id$

## Simple talker demo that published std_msgs/Strings messages
## to the 'chatter' topic

import rospy
from std_msgs.msg import String
from std_msgs.msg import Int16
from std_msgs.msg import UInt64MultiArray
from std_msgs.msg import Float64
import explorerhat

leftwheel = explorerhat.motor.one
rightwheel = explorerhat.motor.two
leftInc = explorerhat.input.one
righInc = explorerhat.input.two

leftCtr = 0
rightCtr = 0
setSpeed = 0

def countUpLeft(input):
    global leftCtr
    leftCtr = leftCtr + input.read()

def countUpRight(input):
    global rightCtr
    rightCtr = rightCtr + input.read()

def callbackLeftWheel(data):
    global leftwheel
    if data.data > 0:
        leftwheel.forwards(data.data)
    else:
        leftwheel.backwards(data.data*-1)
    #rospy.loginfo(rospy.get_caller_id() + 'I heard %d', data.data[0])

def callbackRightWheel(data):
    global rightwheel
    if data.data > 0:
        rightwheel.forwards(data.data)
    else:
        rightwheel.backwards(data.data*-1)
    #rospy.loginfo(rospy.get_caller_id() + 'I heard %d', data.data[1])

def callbackWheelLR(data):
    global leftwheel
    if data.data[0] > 0:
        leftwheel.forwards(data.data[0])
    else:
        leftwheel.backwards(data.data[0]*-1)
    if data.data[1] > 0:
        rightwheel.forwards(data.data[1])
    else:
        rightwheel.backwards(data.data[1]*-1)
    rospy.loginfo(rospy.get_caller_id() + 'I heard:\tLeft %d\tRight %d', data.data[0], data.data[1])

def callbackSetMotors(data):
    global leftwheel
    global rightwheel
    global setSpeed
    leftSpeed = setSpeed + data.data/2
    rightSpeed = setSpeed - data.data/2
    if setSpeed == 0:
        leftSpeed = 0
        rightSpeed = 0
    if leftSpeed > 0:
        if leftSpeed > 100:
            leftSpeed = 100
        leftwheel.forwards(leftSpeed)
    else:
        if leftSpeed < -100:
            leftSpeed = -100
        leftwheel.backwards(leftSpeed*-1)
    if rightSpeed > 0:
        if rightSpeed > 100:
            rightSpeed = 100
        rightwheel.forwards(rightSpeed)
    else:
        if rightSpeed < -100:
            rightSpeed = -100
        rightwheel.backwards(rightSpeed*-1)
    rospy.loginfo(rospy.get_caller_id() + 'From PID: %d', data.data)

def callbackSetVelocity(data):
    global setSpeed
    setSpeed = data.data

def talker():
    pub_Ink = rospy.Publisher('robot/inkLR', UInt64MultiArray, queue_size=10)
    #pub_RInk = rospy.Publisher('robot/rightInk', Int16, queue_size=10)

    rospy.Subscriber('robot/leftWheel', Int16, callbackLeftWheel)
    rospy.Subscriber('robot/rightWheel', Int16, callbackRightWheel)
    #rospy.Subscriber('robot/setMotors', Float64, callbackSetMotors)
    #rospy.Subscriber('robot/setVelocity', Int16, callbackSetVelocity)
    leftInc.on_high(countUpLeft)
    righInc.on_high(countUpRight)
    incLR = UInt64MultiArray()
    incLR.data = [0,0]
    rospy.init_node('robot', anonymous=True)
    rate = rospy.Rate(10) # 10hz
    while not rospy.is_shutdown():
        incLR.data = [leftCtr,rightCtr]
        pub_Ink.publish(incLR)
        #pub_RInk.publish(rightCtr)
        #pub_lInk.publish(leftCtr)
        rate.sleep()

if __name__ == '__main__':
    try:
        talker()
    except rospy.ROSInterruptException:
        pass


import sys
import os
import experiments.registration.split_tools as st
from dipy.fixes import argparse as arg

parser = arg.ArgumentParser(
    description=""
)

parser.add_argument(
    'mod1', action = 'store', metavar = 'mod1',
    help = '''Modality identifier. It will be added as suffix to the filename
    of each input file, e.g. "", "_t2", "_pd" ''')

parser.add_argument(
    'mod2', action = 'store', metavar = 'mod2',
    help = '''Modality identifier. It will be added as suffix to the filename
    of each input file, e.g. "", "_t2", "_pd" ''')

parser.add_argument(
    'action', action = 'store', metavar = 'action',
    help = '''Any of ["s", "s2", "c", "o", "u"] to split all pairs, split
    corresponding pairs, clean, collect or submit, respectively''')

parser.add_argument(
    'names_from', action = 'store', metavar = 'names_from',
    help = '''Text file containing the input file names''')

parser.add_argument(
    '-nt', '--names_to', action = 'store', metavar = 'names_to',
    help = '''Text file containing the input file names''')

if __name__=="__main__":
    req_dir = os.path.dirname(os.path.realpath(__file__))
    required_files = [req_dir+'/dipyreg.py',
                      req_dir+'/job_syn_cc.sh']

    params = parser.parse_args()
    params.mod1
    st.split_script(["", params.action, params.names_from, params.names_to], required_files, 'multi', params.mod1, params.mod2)


import html5lib

from urlparse import urlparse, urljoin

from django.conf import settings
from django.utils.encoding import force_unicode

from django_extensions.db.fields import json
from html5lib import treebuilders
from html5lib.serializer import htmlserializer

BUTTER_ASSETS = {
    'butter.js': 'src/',
    'butter.ui.css': 'css/',
    }

def get_butter_library(src):
    """Determines if the element linked is a valid ``butter`` library"""
    if not src:
        return None
    # Is it already a valid URL?
    url = urlparse(src)
    if url.netloc:
        return None
    # Is this part of butter?
    filename = src.split('/')[-1]
    if filename in BUTTER_ASSETS:
        return settings.STATIC_URL + BUTTER_ASSETS[filename] + filename
    return None

def _serialize_stream(document_tree):
    walker = html5lib.treewalkers.getTreeWalker('lxml')
    stream = walker(document_tree)
    serializer = htmlserializer.HTMLSerializer(omit_optional_tags=False,
                                               quote_attr_values=True)
    return unicode(serializer.render(stream))

def prepare_template_stream(stream, base_url):
    """Prepares the stream to be stored in the DB"""
    stream = force_unicode(stream) if stream else u''
    tree = treebuilders.getTreeBuilder('lxml')
    parser = html5lib.HTMLParser(tree=tree, namespaceHTMLElements=False)
    document_tree = parser.parse(stream)
    script_elements = document_tree.xpath('//script[@src]')
    for script in script_elements:
        src = script.get('src')
        butter_library = get_butter_library(src)
        if butter_library:
            script.set('src', butter_library)
    make_links_absolute(document_tree, base_url)
    return _serialize_stream(document_tree)

def remove_default_values(stream, base_url=None):
    data = json.loads(stream)
    for attr in ['baseDir', 'name', 'savedDataUrl']:
        if attr in data:
            del data[attr]
    return json.dumps(data)

URL_ATTRIBUTES = [
    ('a', 'href'),
    ('applet', 'codebase'),
    ('area', 'href'),
    ('blockquote', 'cite'),
    ('body', 'background'),
    ('del', 'cite'),
    ('form', 'action'),
    ('frame', 'longdesc'),
    ('frame', 'src'),
    ('iframe', 'longdesc'),
    ('iframe', 'src'),
    ('head', 'profile'),
    ('img', 'longdesc'),
    ('img', 'src'),
    ('img', 'usemap'),
    ('input', 'src'),
    ('input', 'usemap'),
    ('ins', 'cite'),
    ('link', 'href'),
    ('object', 'classid'),
    ('object', 'codebase'),
    ('object', 'data'),
    ('object', 'usemap'),
    ('q', 'cite'),
    ('script', 'src'),
    ('video', 'src'),
    ]

def make_links_absolute(document_tree, base_url):
    for tag, attr in URL_ATTRIBUTES:
        xpath = '//%s[@%s]' % (tag, attr)
        element_list = document_tree.xpath(xpath)
        for element in element_list:
            url = get_absolute_url(base_url, element.get(attr))
            element.set(attr, url)
    return document_tree

def get_absolute_url(base_url, path):
    url = urlparse(path)
    if url.netloc:
        return path
    return urljoin(base_url, path)


# To add a new cell, type '# %%'
# To add a new markdown cell, type '# %% [markdown]'
# %%
import os, sys, random
from pathlib import Path
from functools import partial
from datetime import datetime, timezone, timedelta
from IPython.core.debugger import set_trace as bk
import numpy as np
import torch
from torch import nn
import torch.nn.functional as F
import torch.tensor as T
import nlp
from transformers import ElectraConfig, ElectraTokenizerFast, ElectraForMaskedLM, ElectraForPreTraining
from fastai.text.all import *
from hugdatafast import *
from _utils.utils import *
from _utils.would_like_to_pr import *

# %% [markdown]
# # 1. Configuraton

# %%
c = MyConfig({
    'device': 'cuda:0',

    'base_run_name': 'vanilla', # run_name = {base_run_name}_{seed}
    'seed': 11081, # 11081 36 1188 76 1 # None/False to randomly choose seed from [0,99999]

    'adam_bias_correction': False,
    'schedule': 'original_linear',
    'sampling': 'fp32_gumbel',
    'electra_mask_style': True,
    'tie_gen_in_out_embedding': False,
    'gen_smooth_label': False,
    'disc_smooth_label': False,

    'size': 'small',
    'my_model': False, # False to use huggingface model architecture (untrained weights)
})

# only for my personal research purpose
hparam_update = {

}

""" Vanilla ELECTRA settings
'adam_bias_correction': False,
'schedule': 'original_linear',
'sampling': 'fp32_gumbel',
'electra_mask_style': True,
'tie_gen_in_out_embedding': False,
'gen_smooth_label': False,
'disc_smooth_label': False,
"""

# %%
# Check and Default
assert c.sampling in ['fp32_gumbel', 'fp16_gumbel', 'multinomial']
assert c.schedule in ['original_linear', 'separate_linear', 'one_cycle', 'adjusted_one_cycle']
if not c.base_run_name: c.base_run_name = str(datetime.now(timezone(timedelta(hours=+8))))[6:-13].replace(' ','').replace(':','').replace('-','')
if not c.seed: c.seed = random.randint(0, 99999)
c.run_name = f'{c.base_run_name}_{c.seed}'
if c.gen_smooth_label is True: c.gen_smooth_label = 0.1
if c.disc_smooth_label is True: c.disc_smooth_label = 0.1

# Setting of different sizes
i = ['small', 'base', 'large'].index(c.size)
c.mask_prob = [0.15, 0.15, 0.25][i]
c.lr = [5e-4, 2e-4, 2e-4][i]
c.bs = [128, 256, 2048][i]
c.steps = [10**6, 766*1000, 400*1000][i]
c.max_length = [128, 512, 512][i]
generator_size_divisor = [4, 3, 4][i]
disc_config = ElectraConfig.from_pretrained(f'google/electra-{c.size}-discriminator')
gen_config = ElectraConfig.from_pretrained(f'google/electra-{c.size}-generator')
# note that public electra-small model is actually small++ and don't scale down generator size
gen_config.hidden_size = int(disc_config.hidden_size/generator_size_divisor)
gen_config.num_attention_heads = int(disc_config.num_attention_heads/generator_size_divisor)
gen_config.intermediate_size = int(disc_config.intermediate_size/generator_size_divisor)
hf_tokenizer = ElectraTokenizerFast.from_pretrained(f"google/electra-{c.size}-generator")

# Path to data
Path('./datasets', exist_ok=True)
Path('./checkpoints/pretrain').mkdir(exist_ok=True, parents=True)
if c.size in ['small', 'base']:
  wiki_cache_dir = Path("./datasets/wikipedia/20200501.en/1.0.0")
  book_cache_dir = Path("./datasets/bookcorpus/plain_text/1.0.0")
  wbdl_cache_dir = Path("./datasets/wikibook_dl")
  wbdl_cache_dir.mkdir(exist_ok=True)

# Print info
print(f"process id: {os.getpid()}")
print(c)
print(hparam_update)

# %%
if c.my_model:
  sys.path.insert(0, os.path.abspath(".."))
  from modeling.model import ModelForGenerator,ModelForDiscriminator
  from hyperparameter import electra_hparam_from_hf
  gen_hparam = electra_hparam_from_hf(gen_config, hf_tokenizer)
  gen_hparam.update(hparam_update)
  disc_hparam = electra_hparam_from_hf(disc_config, hf_tokenizer)
  disc_hparam.update(hparam_update)

# %% [markdown]
# # 1. Load Data

# %%
if c.size in ['small', 'base']:

  # wiki
  if (wiki_cache_dir/f"wiki_electra_{c.max_length}.arrow").exists():
    print('loading the electra data (wiki)')
    wiki = nlp.Dataset.from_file(str(wiki_cache_dir/f"wiki_electra_{c.max_length}.arrow"))
  else:
    print('load/download wiki dataset')
    wiki = nlp.load_dataset('wikipedia', '20200501.en', cache_dir='./datasets')['train']

    print('creat data from wiki dataset for ELECTRA')
    wiki = ELECTRADataTransform(wiki, is_docs=True, text_col='text', max_length=c.max_length, hf_toker=hf_tokenizer).map(cache_file_name=str(wiki_cache_dir/f"wiki_electra_{c.max_length}.arrow"))

  # bookcorpus
  if (book_cache_dir/f"book_electra_{c.max_length}.arrow").exists():
    print('loading the electra data (BookCorpus)')
    book = nlp.Dataset.from_file(str(book_cache_dir/f"book_electra_{c.max_length}.arrow"))
  else:
    print('load/download BookCorpus dataset')
    book = nlp.load_dataset('bookcorpus', cache_dir='./datasets')['train']

    print('creat data from BookCorpus dataset for ELECTRA')
    book = ELECTRADataTransform(book, is_docs=False, text_col='text', max_length=c.max_length, hf_toker=hf_tokenizer).map(cache_file_name=str(book_cache_dir/f"book_electra_{c.max_length}.arrow"))

  wb_data = HF_MergedDataset(wiki, book)
  wb_dsets = HF_Datasets({'train': wb_data}, cols=['input_ids','sentA_lenth'], hf_toker=hf_tokenizer, n_inp=2)
  dls = wb_dsets.dataloaders(bs=c.bs, num_workers=6,
                             shuffle_train=True,
                             srtkey_fc=False,
                             cache_dir='./datasets/wikibook_dl', cache_name='dl_{split}.json')

else: # for large size
  raise NotImplementedError

# %% [markdown]
# # 2. Masked language model objective
# %% [markdown]
# ## 2.1 MLM objective callback

# %%
"""
Modified from HuggingFace/transformers (https://github.com/huggingface/transformers/blob/0a3d0e02c5af20bfe9091038c4fd11fb79175546/src/transformers/data/data_collator.py#L102).
It is a little bit faster cuz
- intead of a[b] a on gpu b on cpu, tensors here are all in the same device
- don't iterate the tensor when create special tokens mask
And
- doesn't require huggingface tokenizer
- cost you only 550 µs for a (128,128) tensor on gpu, so dynamic masking is cheap
"""
def mask_tokens(inputs, mask_token_index, vocab_size, special_token_indices, mlm_probability=0.15, replace_prob=0.1, orginal_prob=0.1, ignore_index=-100):
  """
  Prepare masked tokens inputs/labels for masked language modeling: (1-replace_prob-orginal_prob)% MASK, replace_prob% random, orginal_prob% original within mlm_probability% of tokens in the sentence.
  * ignore_index in nn.CrossEntropy is default to -100, so you don't need to specify ignore_index in loss
  """

  device = inputs.device
  labels = inputs.clone()

  # Get positions to apply mlm (mask/replace/not changed). (mlm_probability)
  probability_matrix = torch.full(labels.shape, mlm_probability, device=device)
  special_tokens_mask = torch.full(inputs.shape, False, dtype=torch.bool, device=device)
  for sp_id in special_token_indices:
    special_tokens_mask = special_tokens_mask | (inputs==sp_id)
  probability_matrix.masked_fill_(special_tokens_mask, value=0.0)
  mlm_mask = torch.bernoulli(probability_matrix).bool()
  labels[~mlm_mask] = ignore_index  # We only compute loss on mlm applied tokens

  # mask  (mlm_probability * (1-replace_prob-orginal_prob))
  mask_prob = 1 - replace_prob - orginal_prob
  mask_token_mask = torch.bernoulli(torch.full(labels.shape, 0.8, device=device)).bool() & mlm_mask
  inputs[mask_token_mask] = mask_token_index

  # replace with a random token (mlm_probability * replace_prob)
  if int(replace_prob)!=0:
    rep_prob = replace_prob/(replace_prob + orginal_prob)
    replace_token_mask = torch.bernoulli(torch.full(labels.shape, 0.5, device=device)).bool() & mlm_mask & ~mask_token_mask
    random_words = torch.randint(vocab_size, labels.shape, dtype=torch.long, device=device)
    inputs[replace_token_mask] = random_words[replace_token_mask]

  # do nothing (mlm_probability * orginal_prob)
  pass

  return inputs, labels, mlm_mask

class MaskedLMCallback(Callback):
  @delegates(mask_tokens)
  def __init__(self, mask_tok_id, special_tok_ids, vocab_size, ignore_index=-100, for_electra=False, **kwargs):
    self.ignore_index = ignore_index
    self.for_electra = for_electra
    self.mask_tokens = partial(mask_tokens,
                               mask_token_index=mask_tok_id,
                               special_token_indices=special_tok_ids,
                               vocab_size=vocab_size,
                               ignore_index=-100,
                               **kwargs)

  def before_batch(self):
    input_ids, sentA_lenths  = self.xb
    masked_inputs, labels, is_mlm_applied = self.mask_tokens(input_ids)
    if self.for_electra:
      self.learn.xb, self.learn.yb = (masked_inputs, sentA_lenths, is_mlm_applied, labels), (labels,)
    else:
      self.learn.xb, self.learn.yb = (masked_inputs, sentA_lenths), (labels,)

  @delegates(TfmdDL.show_batch)
  def show_batch(self, dl, idx_show_ignored, verbose=True, **kwargs):
    b = dl.one_batch()
    input_ids, sentA_lenths  = b
    masked_inputs, labels, is_mlm_applied = self.mask_tokens(input_ids.clone())
    # check
    assert torch.equal(is_mlm_applied, labels!=self.ignore_index)
    assert torch.equal((~is_mlm_applied *masked_inputs + is_mlm_applied * labels), input_ids)
    # change symbol to show the ignored position
    labels[labels==self.ignore_index] = idx_show_ignored
    # some notice to help understand the masking mechanism
    if verbose:
      print("We won't count loss from position where y is ignore index")
      print("Notice 1. Positions have label token in y will be either [Mask]/other token/orginal token in x")
      print("Notice 2. Special tokens (CLS, SEP) won't be masked.")
      print("Notice 3. Dynamic masking: every time you run gives you different results.")
    # show
    tfm_b =(masked_inputs, sentA_lenths, is_mlm_applied, labels) if self.for_electra else (masked_inputs, sentA_lenths, labels)
    dl.show_batch(b=tfm_b, **kwargs)

# %%
mlm_cb = MaskedLMCallback(mask_tok_id=hf_tokenizer.mask_token_id,
                          special_tok_ids=hf_tokenizer.all_special_ids,
                          vocab_size=hf_tokenizer.vocab_size,
                          mlm_probability=c.mask_prob,
                          replace_prob=0.0 if c.electra_mask_style else 0.1,
                          orginal_prob=0.15 if c.electra_mask_style else 0.1,
                          for_electra=True)
#mlm_cb.show_batch(dls[0], idx_show_ignored=hf_tokenizer.convert_tokens_to_ids(['#'])[0])

# %% [markdown]
# # 3. ELECTRA (replaced token detection objective)
# see details in paper [ELECTRA: Pre-training Text Encoders as Discriminators Rather Than Generators](https://arxiv.org/abs/2003.10555)

# %%
class ELECTRAModel(nn.Module):

  def __init__(self, generator, discriminator, hf_tokenizer):
    super().__init__()
    self.generator, self.discriminator = generator,discriminator
    self.gumbel_dist = torch.distributions.gumbel.Gumbel(0.,1.)
    self.hf_tokenizer = hf_tokenizer

  def to(self, *args, **kwargs):
    "Also set dtype and device of contained gumbel distribution if needed"
    super().to(*args, **kwargs)
    a_tensor = next(self.parameters())
    device, dtype = a_tensor.device, a_tensor.dtype
    if c.sampling=='fp32_gumbel': dtype = torch.float32
    self.gumbel_dist = torch.distributions.gumbel.Gumbel(torch.tensor(0., device=device, dtype=dtype), torch.tensor(1., device=device, dtype=dtype))

  def forward(self, masked_inputs, sentA_lenths, is_mlm_applied, labels):
    """
    masked_inputs (Tensor[int]): (B, L)
    sentA_lenths (Tensor[int]): (B, L)
    is_mlm_applied (Tensor[boolean]): (B, L), True for positions chosen by mlm probability
    labels (Tensor[int]): (B, L), -100 for positions where are not mlm applied
    """
    attention_mask, token_type_ids = self._get_pad_mask_and_token_type(masked_inputs, sentA_lenths)
    if c.my_model:
      gen_logits = self.generator(masked_inputs, attention_mask, token_type_ids, is_mlm_applied)[0]
      # already reduced before the mlm output layer, save more space and speed
      mlm_gen_logits = gen_logits # ( #mlm_positions, vocab_size)
    else:
      gen_logits = self.generator(masked_inputs, attention_mask, token_type_ids)[0] # (B, L, vocab size)
      # reduce size to save space and speed
      mlm_gen_logits = gen_logits[is_mlm_applied, :] # ( #mlm_positions, vocab_size)

    with torch.no_grad():
      # sampling
      pred_toks = self.sample(mlm_gen_logits) # ( #mlm_positions, )
      # produce inputs for discriminator
      generated = masked_inputs.clone() # (B,L)
      generated[is_mlm_applied] = pred_toks # (B,L)
      # produce labels for discriminator
      is_replaced = is_mlm_applied.clone() # (B,L)
      is_replaced[is_mlm_applied] = (pred_toks != labels[is_mlm_applied]) # (B,L)

    disc_logits = self.discriminator(generated, attention_mask, token_type_ids)[0] # (B, L)

    return mlm_gen_logits, generated, disc_logits, is_replaced, attention_mask, is_mlm_applied

  def _get_pad_mask_and_token_type(self, input_ids, sentA_lenths):
    """
    Only cost you about 500 µs for (128, 128) on GPU, but so that your dataset won't need to save attention_mask and token_type_ids and won't be unnecessarily large, thus, prevent cpu processes loading batches from consuming lots of cpu memory and slow down the machine.
    """
    attention_mask = input_ids != self.hf_tokenizer.pad_token_id
    seq_len = input_ids.shape[1]
    token_type_ids = torch.tensor([ ([0]*len + [1]*(seq_len-len)) for len in sentA_lenths.tolist()],
                                  device=input_ids.device)
    return attention_mask, token_type_ids

  def sample(self, logits):
    "Reimplement gumbel softmax cuz there is a bug in torch.nn.functional.gumbel_softmax when fp16 (https://github.com/pytorch/pytorch/issues/41663). Gumbel softmax is equal to what official ELECTRA code do, standard gumbel dist. = -ln(-ln(standard uniform dist.))"
    if c.sampling == 'fp32_gumbel':
      return (logits.float() + self.gumbel_dist.sample(logits.shape)).argmax(dim=-1)
    elif c.sampling == 'fp16_gumbel': # 5.06 ms
      return (logits + self.gumbel_dist.sample(logits.shape)).argmax(dim=-1)
    elif c.sampling == 'multinomial': # 2.X ms
      return torch.multinomial(F.softmax(logits, dim=-1), 1).squeeze()

class ELECTRALoss():
  def __init__(self, loss_weights=(1.0, 50.0), gen_label_smooth=False, disc_label_smooth=False):
    self.loss_weights = loss_weights
    self.gen_loss_fc = LabelSmoothingCrossEntropyFlat(eps=gen_label_smooth) if gen_label_smooth else CrossEntropyLossFlat()
    self.disc_loss_fc = nn.BCEWithLogitsLoss()
    self.disc_label_smooth = disc_label_smooth

  def __call__(self, pred, targ_ids):
    mlm_gen_logits, generated, disc_logits, is_replaced, non_pad, is_mlm_applied = pred
    gen_loss = self.gen_loss_fc(mlm_gen_logits.float(), targ_ids[is_mlm_applied])
    disc_logits = disc_logits.masked_select(non_pad) # -> 1d tensor
    is_replaced = is_replaced.masked_select(non_pad) # -> 1d tensor
    if self.disc_label_smooth:
      is_replaced = is_replaced.float().masked_fill(~is_replaced, disc_label_smooth)
    disc_loss = self.disc_loss_fc(disc_logits.float(), is_replaced.float())
    return gen_loss * self.loss_weights[0] + disc_loss * self.loss_weights[1]

  def decodes(self, pred):
    gen_logits, generated, disc_logits, is_replaced, non_pad = pred
    gen_pred = gen_logits.argmax(dim=-1)
    disc_pred = disc_logits > 0
    return gen_pred, generated, disc_pred, is_replaced

# %% [markdown]
# # 5. Train

# %%
# Generator and Discriminator
if c.my_model:
  gen_hparam['tie_in_out_embedding'] = c.tie_gen_in_out_embedding
  generator = ModelForGenerator(gen_hparam)
  discriminator = ModelForDiscriminator(disc_hparam)
  discriminator.electra.embedding = generator.electra.embedding
else:
  generator = ElectraForMaskedLM(gen_config)
  discriminator = ElectraForPreTraining(disc_config)
  discriminator.electra.embeddings = generator.electra.embeddings
  if c.tie_gen_in_out_embedding:
    generator.generator_predictions.dense.weight = generator.electra.embeddings.word_embeddings.weight

# ELECTRA training loop
electra_model = ELECTRAModel(generator, discriminator, hf_tokenizer)
electra_loss_func = ELECTRALoss(gen_label_smooth=c.gen_smooth_label, disc_label_smooth=c.disc_smooth_label)

# jit (Haven't fiqured out how to make it work)
# input_ids, sentA_lenths = dls.one_batch()
# masked_inputs, labels, is_mlm_applied = mlm_cb.mask_tokens(input_ids)
# electra_jit_model = torch.jit.trace(electra_model, (masked_inputs, sentA_lenths, is_mlm_applied, labels))

# Optimizer
if c.adam_bias_correction: opt_func = partial(Adam, eps=1e-6, mom=0.9, sqr_mom=0.999, wd=0.01)
else: opt_func = partial(Adam_no_bias_correction, eps=1e-6, mom=0.9, sqr_mom=0.999, wd=0.01)

# Learning rate shedule
if c.schedule.endswith('linear'):
  lr_shed_func = linear_warmup_and_then_decay if c.schedule=='separate_linear' else linear_warmup_and_decay
  lr_shedule = ParamScheduler({'lr': partial(linear_warmup_and_decay,
                                             lr_max=c.lr,
                                             warmup_steps=10000,
                                             total_steps=c.steps,)})

# Learner
dls.to(torch.device(c.device))
learn = Learner(dls, electra_model,
                loss_func=electra_loss_func,
                opt_func=opt_func ,
                path='./checkpoints',
                model_dir='pretrain',
                cbs=[mlm_cb,
                     RunSteps(c.steps, [0.0625, 0.125, 0.25, 0.5, 1.0], c.run_name+"_{percent}"),
                     ],
                )

# Mixed precison and Gradient clip
learn.to_native_fp16(init_scale=2.**11)
learn.add_cb(GradientClipping(1.))

# Seed
random.seed(c.seed)
np.random.seed(c.seed)
torch.manual_seed(c.seed)

# Run
print(f"{c.run_name} , starts at {datetime.now()}")
#bk()
if c.schedule == 'one_cycle': learn.fit_one_cycle(9999, lr_max=c.lr)
elif c.schedule == 'adjusted_one_cycle': learn.fit_one_cycle(9999, lr_max=c.lr, div=1e5, pct_start=10000/c.steps)
else: learn.fit(9999, cbs=[lr_shedule])


import re
import requests
import core
from Init import app
from exts import db
from models import BaseInfo,IPInfo,DomainInfo,BugList
from BugScan import BugScan
from Init import redispool
from SpiderGetUrl2 import SpiderGetUrl2
from BaseMessage import GetBaseMessage
from IPMessage import IPMessage
from DomainMessage import DomainMessage

Bugs=["SQLBugScan","XSSBugScan","ComInScan","FileIncludeScan"]

requests.packages.urllib3.disable_warnings()
'''
SZheConsole 碎遮扫描器的总控制代码
获取baseinfo ->MySQL
 ip->获取ipinfo->MySQL
 domain->获取domaininfo->MySQL
页面url深度遍历 ->从redis里读取->bugscan->MySQL
    未设置外键，用程序来保证逻辑的正确性
'''

def BugScanConsole(attackurl):
    '''
    动态调用类方法，减少冗余代码
    将存在bug的url存在buglist表中，同时根据漏洞类型的不同，指向bugtype表中对应的漏洞类型
    '''
    try:
        while redispool.scard(attackurl) != 0:
            url = redispool.spop(attackurl)
            Bug=BugScan(attackurl,url)
            with app.app_context():
                for value in Bugs:
                    vulnerable, payload,bugdetail=getattr(Bug, value)()
                    if vulnerable:
                            bug = BugList(oldurl=attackurl,bugurl=url,bugname=value,buggrade=redispool.hget('bugtype', value),payload=payload,bugdetail=bugdetail)
                            redispool.pfadd(redispool.hget('bugtype', value), url)
                            redispool.pfadd(value, url)
                            db.session.add(bug)
                db.session.commit()
            Bug.POCScan()
        # time.sleep(0.5)
    except Exception as e:
        print(e)
        pass

def inputfilter(url):
    '''
    入口过滤函数
    输入源的格式可多变:
    127.0.0.1
    http://127.0.0.1
    www.baidu.com
    https://www.baidu.com
    等
    返回格式为 ： return www.baidu.com,https://www.baidu.com,baidu.rep
    :param url:
    :return:
    '''
    rep,rep1,rep2=None,None,None
    if url.endswith("/"):
        url=url[:-1]
    if not url.startswith("http://") and not url.startswith("https://"):
        attackurl1="http://"+url
        attackurl2="https://"+url
        try:
            rep1=requests.get(attackurl1, headers=core.GetHeaders(), timeout=4, verify=False)
        except Exception as e:
            pass
        try:
            rep2=requests.get(attackurl2, headers=core.GetHeaders(), timeout=4, verify=False)
        except Exception as e:
            pass
        if rep1:
            return url,attackurl1,rep1
        elif rep2:
            return url,attackurl2,rep2
        else:
            print("None data")
            try:
                count=redispool.hget('targetscan', 'waitcount')
                if 'str' in str(type(count)):
                    waitcount=int(count)-1
                    redispool.hset("targetscan", "waitcount", str(waitcount))
                else:
                    redispool.hset("targetscan", "waitcount", "0")
                redispool.hdel("targetscan", "nowscan")
            except Exception as e:
                print(e)
                pass
            return None,None,None
    else:
        attackurl=url
        try:
            rep=requests.get(attackurl, headers=core.GetHeaders(), timeout=4, verify=False)
        except:
            pass
        if rep:
            if "http://" in url:
                return url.replace("http://",""),attackurl,rep
            else:
                return url.replace("https://",""),attackurl,rep
        else:
            print("{}访问超时".format(attackurl))
            return None,None,None

def SZheScan(url):
    try:
        #输入入口进行过滤
        url,attackurl,rep=inputfilter(url)
        #若过滤后无url，即url无效或响应时间过长，退出对该url的扫描
        if not url:
            print("Not Allow This URL")
            return
        redispool.hset("targetscan", "nowscan", attackurl)
        with app.app_context():
            # 对该url基础信息进行搜集,实例化GetBaseMessage对象
            baseinfo = GetBaseMessage(url, attackurl,rep)
            #正则表达式判断其为IP或是域名，并且实例化相应的深度信息搜集对象
            pattern = re.compile('^\d+\.\d+\.\d+\.\d+(:(\d+))?$')
            #判断IP是否存在端口
            if pattern.findall(url) and ":" in url:
                infourl=url.split(":")[0]
            else:
                infourl=url
            if pattern.findall(url):
                boolcheck = True
                ipinfo = IPMessage(infourl)
            else:
                boolcheck = False
                domaininfo = DomainMessage(url)
            info = BaseInfo(url=url, boolcheck=boolcheck, status=baseinfo.GetStatus(), title=baseinfo.GetTitle(),
                            date=baseinfo.GetDate(), responseheader=baseinfo.GetResponseHeader(),
                            Server=baseinfo.GetFinger(), portserver=baseinfo.PortScan(), sendir=baseinfo.SenDir())
            db.session.add(info)
            db.session.flush()
            infoid=info.id
            db.session.commit()
            baseinfo.WebLogicScan()
            baseinfo.AngelSwordMain()
            if boolcheck:
                redispool.pfadd("ip", infourl)
                ipinfo=IPInfo(baseinfoid=infoid, bindingdomain=ipinfo.GetBindingIP(), sitestation=ipinfo.GetSiteStation(),
                            CMessage=ipinfo.CScanConsole(),
                            ipaddr=ipinfo.FindIpAdd())
                db.session.add(ipinfo)
            else:
                redispool.pfadd("domain", infourl)
                domaininfo=DomainInfo(baseinfoid=infoid, subdomain=domaininfo.GetSubDomain(), whois=domaininfo.GetWhoisMessage(),
                                bindingip=domaininfo.GetBindingIP(),
                                sitestation=domaininfo.GetSiteStation(), recordinfo=domaininfo.GetRecordInfo(),
                                domainaddr=domaininfo.FindDomainAdd())
                db.session.add(domaininfo)
            db.session.commit()
            #默认url深度爬取为 2 ，避免大站链接过多，可在设置中进行修改
            redispool.append("runlog", "对{}页面进行深度爬取\n".format(attackurl))
            SpiderGetUrl2(attackurl,deepth=2)
            redispool.append("runlog", "对该网站{}爬取到的url进行常规漏扫 :D\n".format(attackurl))
            print("对该网站爬取到的url进行常规漏扫 :D")
            BugScanConsole(url)
            count=redispool.hget('targetscan', 'waitcount')
            if 'str' in str(type(count)):
                waitcount=int(count)-1
                redispool.hset("targetscan", "waitcount", str(waitcount))
            else:
                redispool.hset("targetscan", "waitcount", "0")
            redispool.hdel("targetscan", "nowscan")
            #漏洞列表中存在该url的漏洞，证明该url是受到影响的，将redis havebugpc受影响主机加一
            firstbugurl= BugList.query.order_by(BugList.id.desc()).first().oldurl
            if firstbugurl==url:
                redispool.pfadd("havebugpc", url)
            redispool.append("runlog", "{} scan end !\n".format(url))
            print("{} scan end !".format(url))
            # print(redispool.get('runlog'))
    except Exception as e:
        print(e)
        pass

if __name__=='__main__':
    print(inputfilter("https://www.cnblogs.com/"))
#     SZheConsole("testphp.vulnweb.com",redispool)
    # BugScanConsole("testphp.vulnweb.com",redispool)


def num_common_letters(goal_word, guess):
    goal_list = get_list(goal_word)
    guess_list = get_list(guess)
    already_seen = []
    i = 0
    while i <= len(goal_list) - 1:
        if guess_list[i] in goal_list and guess_list[i] not in already_seen:
             already_seen += [guess_list[i]]
        i += 1
    return len(already_seen)


import os
import pickle
from operator import add

import cv2
import keras
from keras.layers import concatenate, Conv2D, MaxPooling2D, UpSampling2D, Dropout
# os.environ["CUDA_VISIBLE_DEVICES"] = "0"
from keras.models import *
from keras.optimizers import *

import Metrics as met
import MyOculusCsvLib as mocl
import MyOculusImageLib as moil

class Models:

    @staticmethod
    def model_show_function(x):
        y = []
        for i in range(len(x) - 1):
            y.append(x[i + 1])
        moil.show(x[0], other_im=y)

    def __init__(self, rowDim, colDim, mode=0, channels=1, out_channels=1, modify_col=False, row_div_col=0,
                 weights_path="../weights/unet", var_filename="../weights/var.txt", show_function=None, read_func=None,
                 validate_path_provider_func=None, validate_start_path=None, preprocessFunc=lambda x: x,
                 constantVar=None):
        self.model = None
        self.constantVar = constantVar
        self.path = weights_path
        self.var_filename = var_filename
        self.rowDim = int(rowDim)
        self.colDim = int(colDim)
        self.mode = mode
        self.channels = channels
        self.out_channels = out_channels
        self.row_div_col = row_div_col
        self.show_function = show_function
        self.read_func = read_func
        self.validate_path_provider_func = validate_path_provider_func
        self.validate_start_path = validate_start_path
        self.preprocessFunc = preprocessFunc
        if row_div_col > 0:
            if modify_col:
                self.colDim = int(rowDim / row_div_col)
            else:
                self.rowDim = int(colDim * row_div_col)

    def save_weights(self, increase=1):
        self.model.save_weights(self.path + '_' + str(self.var_file(increase)))

    def load_weights(self):
        try:
            numb = self.var_file(True)
            if numb >= 0:
                self.model.load_weights(self.path + '_' + str(numb))
                return True
            return False
        except:
            print("Nie udało się wczytać wag sieci!")
            return False

    def centerDiffMetric(pred, x, y):
        return met.centerDiff(pred, x, y)

    def predict(self, im):
        w, h, c = moil.getWidthHeightChannels(im)

        if w != self.colDim or h != self.rowDim or c != self.channels:
            im = cv2.resize(im, (self.colDim, self.rowDim))[:, :]
        im = self.prepareImage(im)
        im = self.model.predict(im)
        im = moil.convertImageNetOutput(im)
        return im

    def prepareImage(self, im, retboth=False):
        img = self.preprocessFunc(im)
        imgX = img.reshape((1, self.rowDim, self.colDim, self.channels))
        imgX = imgX / 255
        if retboth:
            return imgX, img
        return imgX

    def readImage(self, name, path, extension=".jpg"):
        return self.read_func(name=name, extension=extension, path=path, target_size=(self.colDim, self.rowDim), mode=0)

    def validate(self, pathForce=None, validateMode=0, preprocessFunc=lambda x: x, draw=True, onlyWithMetric=False,
                 onlyWithoutMetric=False, sumTimes=None, metrics=['distance', 'youden', 'jaccard', 'dice'],
                 validTimes=1, weightsTimesValids=None, validName=''):
        avgs, globals = (0, 0)
        for i in range(validTimes):
            if weightsTimesValids is not None:
                self.constantVar = i * weightsTimesValids
                self.load_weights()
            sum = [0] * len(metrics)
            confusion_matrix = [0] * 4
            globalCount = False
            for metr in metrics:
                if 'global' in metr:
                    globalCount = True
            times = 0
            visited_path = {}
            while True:
                if pathForce is None:
                    path = self.validate_path_provider_func(self.validate_start_path, visited_path)
                    visited_path[path] = times

                else:
                    path = pathForce
                if path is None:
                    break
                if not os.path.exists(path):
                    continue
                images = os.listdir(path)
                for imp in images:  # len(os.listdir(path)) - 2):

                    true_path = path + 'mask/'
                    if not os.path.exists(os.path.join(path, imp)):
                        continue
                    if onlyWithMetric and not os.path.exists(os.path.join(true_path, imp)):
                        continue
                    else:
                        if onlyWithoutMetric and os.path.exists(os.path.join(true_path, imp)):
                            continue

                    im = self.read_func(name=imp, extension='', path=path, target_size=(self.colDim, self.rowDim),
                                        mode=0)
                    imgX, img = self.prepareImage(im, retboth=True)
                    pred = self.model.predict(imgX)

                    pred = moil.convertImageNetOutput(pred)

                    toDraw = im if draw else None

                    x = [im, pred, img]
                    if os.path.exists(os.path.join(true_path, imp)):
                        true = self.read_func(name=imp, extension='', path=true_path,
                                              target_size=(self.colDim, self.rowDim))

                        true = true.reshape((self.rowDim, self.colDim, self.out_channels))
                        x.append(true)
                        results = met.customMetric(pred, true, toDraw=toDraw, metrics=metrics, globalCount=globalCount)
                        sum = list(map(add, sum, results[0]))

                        confusion_matrix = list(map(add, confusion_matrix, results[1]))

                        times += 1
                        if sumTimes is not None and times >= sumTimes:
                            break
                    else:
                        met.draw(pred, toDraw)

                    if sumTimes is None:
                        self.show_function(x)

            avgs = [x / times for x in sum]
            strgSum = ''
            strgAvgs = ''
            for val in sum:
                strgSum += str(val) + ', '
            for val in avgs:
                strgAvgs += str(val) + ', '

            globals = []
            if globalCount:
                globals = met.globals(confusion_matrix)
                print("Global Jaccard: " + str(globals[0]) + ", Global Dice: " + str(globals[1]))
            print("Times: " + str(times) + ", sums: " + strgSum + "Average metrics: " + strgAvgs)
            self.validate_to_csv(metrics, avgs + globals, validName)
        return avgs + globals

    def validate_to_csv(self, metrics, values, name=''):
        path = self.path.split('/')[:-1]
        path = '/'.join(path)
        epoch = (self.var_file(True) % 100) * 100
        epoch = epoch + int(self.var_file(True) / 100) * 100
        mocl.writeToCsv(path + '/scores' + name + '.csv', ['name'] + metrics,
                        [self.path.split('/')[-1] + '_' + str(epoch)] + values)

    def check_performance(self, validate_generator, times=1, metrics=['distance', 'youden', 'jaccard', 'dice']):
        for i in range(times):
            pic = validate_generator.next()

            true = pic[1][0]

            pred = self.model.predict(pic[0][0].reshape(1, self.rowDim, self.colDim, self.channels))
            pred = moil.convertImageNetOutput(pred)
            true = moil.convertImageNetOutput(true)

            met.customMetric(pred, true, metrics=metrics)
            x = []

            x.append(pic[0][0].reshape((self.rowDim, self.colDim, self.channels)))
            x.append(true)
            x.append(pred)

            if self.show_function != None:
                self.show_function(x)

    def var_file(self, read=False, increase=1):
        if self.constantVar is not None:
            return self.constantVar
        numb = -1
        if not os.path.isfile(self.var_filename):
            fo = open(self.var_filename, "w")
            fo.write("-1")
            fo.close()
        else:
            fo = open(self.var_filename, "r")
            numb = int(next(fo))
            fo.close()

        if read:
            return numb
        if numb < 0:
            numb = 0
        else:
            numb += increase
        fo = open(self.var_filename, "w")
        fo.write(str(numb))
        fo.close()
        return numb

    def load_loss(self, epoch):
        try:
            with open(self.path + "_losses_" + str(epoch), 'rb') as handle:
                return pickle.load(handle)
        except:
            return None

    def plot_loss(self, epoch=None, start_ep = 0):
        b = None
        if epoch is None:
            epoch = self.var_file(read=True)
        b = self.load_loss(epoch)
        if b is None:
            return
        b=b[start_ep:]

    def get_model(self, filters=2, le=1e-04, decay=0):

        if self.model is not None:
            return self.model

        inputs = Input((self.rowDim, self.colDim, self.channels))
        conv1 = Conv2D(filters, 3, activation='relu', padding='same', kernel_initializer='he_normal')(inputs)

        print("conv1 shape:", conv1.shape)
        conv1 = Conv2D(filters, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv1)
        print("conv1 shape:", conv1.shape)
        pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
        print("pool1 shape:", pool1.shape)

        conv2 = Conv2D(filters * 2, 3, activation='relu', padding='same', kernel_initializer='he_normal')(pool1)
        print("conv2 shape:", conv2.shape)
        conv2 = Conv2D(filters * 2, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv2)
        print("conv2 shape:", conv2.shape)
        pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
        print("pool2 shape:", pool2.shape)

        conv3 = Conv2D(filters * 4, 3, activation='relu', padding='same', kernel_initializer='he_normal')(pool2)
        print("conv3 shape:", conv3.shape)
        conv3 = Conv2D(filters * 4, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv3)
        print("conv3 shape:", conv3.shape)
        pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
        print("pool3 shape:", pool3.shape)

        conv4 = Conv2D(filters * 8, 3, activation='relu', padding='same', kernel_initializer='he_normal')(pool3)
        conv4 = Conv2D(filters * 8, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv4)
        drop4 = Dropout(0.5)(conv4)
        pool4 = MaxPooling2D(pool_size=(2, 2))(drop4)

        conv5 = Conv2D(filters * 16, 3, activation='relu', padding='same', kernel_initializer='he_normal')(pool4)
        conv5 = Conv2D(filters * 16, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv5)
        drop5 = Dropout(0.5)(conv5)

        up6 = Conv2D(filters * 8, 2, activation='relu', padding='same', kernel_initializer='he_normal')(
            UpSampling2D(size=(2, 2))(drop5))
        merge6 = concatenate([drop4, up6], axis=3)
        conv6 = Conv2D(filters * 8, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge6)
        conv6 = Conv2D(filters * 8, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv6)

        up7 = Conv2D(filters * 4, 2, activation='relu', padding='same', kernel_initializer='he_normal')(
            UpSampling2D(size=(2, 2))(conv6))
        merge7 = concatenate([conv3, up7], axis=3)
        conv7 = Conv2D(filters * 4, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge7)
        conv7 = Conv2D(filters * 4, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv7)

        up8 = Conv2D(filters * 2, 2, activation='relu', padding='same', kernel_initializer='he_normal')(
            UpSampling2D(size=(2, 2))(conv7))
        merge8 = concatenate([conv2, up8], axis=3)
        conv8 = Conv2D(filters * 2, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge8)
        conv8 = Conv2D(filters * 2, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv8)

        up9 = Conv2D(filters, 2, activation='relu', padding='same', kernel_initializer='he_normal')(
            UpSampling2D(size=(2, 2))(conv8))
        merge9 = concatenate([conv1, up9], axis=3)
        conv9 = Conv2D(filters, 3, activation='relu', padding='same', kernel_initializer='he_normal')(merge9)
        conv9 = Conv2D(filters, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv9)
        conv9 = Conv2D(2, 3, activation='relu', padding='same', kernel_initializer='he_normal')(conv9)
        conv10 = Conv2D(1, 1, activation='sigmoid')(conv9)

        model = Model(input=inputs, output=conv10)
        model.compile(optimizer=Adam(lr=le, decay=decay), loss='binary_crossentropy', metrics=['accuracy'])

        self.model = model

        return self.model

class Callbacks(keras.callbacks.Callback):

    def __init__(self, ModelClass, save_modulo_epochs=None, collectLoss=False, printDecay=False, message=None):
        self.ModelClass = ModelClass
        self.save_modulo_epochs = save_modulo_epochs
        self.printDecay = printDecay
        self.collectLoss = collectLoss
        try:
            epoch = ModelClass.var_file(read=True)
            losses = ModelClass.load_loss(epoch)
        except:
            losses = []
        if losses is None:
            losses = []
        self.losses = losses
        self.message = message

    def on_epoch_end(self, epoch, logs={}):
        if self.message is not None:
            print(self.message)
        if self.printDecay:
            lr = self.model.optimizer.lr
            decay = self.model.optimizer.decay
            iterations = self.model.optimizer.iterations
            lr_with_decay = lr / (1. + decay * K.cast(iterations, K.dtype(decay)))
            print(K.eval(iterations))
            print(K.eval(lr_with_decay))

        self.losses.append(logs.get('loss'))
        if self.save_modulo_epochs is not None:
            if (epoch + 1) % self.save_modulo_epochs == 0:
                self.ModelClass.save_weights(increase=self.save_modulo_epochs)
                if self.collectLoss:
                    with open(self.ModelClass.path + '_losses_' + str(len(self.losses)), 'wb') as handle:
                        pickle.dump(self.losses, handle)

        return


from django.conf.urls import url
from rest_framework.urlpatterns import format_suffix_patterns
from tictactoeapp import views

urlpatterns = [
    url(r'^userprofile_update/$', views.UserProfileView.as_view()),
    url(r'^check_user/$', views.CheckLoginView.as_view()),
    url(r'^logout/$', views.LogoutView.as_view()),
    url(r'^new/$', views.new_room, name='new_room'),
    url(r'^online_user/$', views.online_user, name='online'),
    url(r'^get_online_user/$', views.get_online_user, name='online_user_list'),
    url(r'^game/$', views.GameView.as_view(), name='game'),
    url(r'^available_room/$', views.RoomView.as_view(), name='available_room'),
    url(r'^extendfb/$', views.extendFbToken, name='fbtoken'),
    url(r'^rank/$', views.rank, name='rank'),
    url(r'^imageedit/$', views.imageEdit, name='imageedit'),
    url(r'^imageframe/$', views.imageMerge, name='imagemerge'),
]

urlpatterns = format_suffix_patterns(urlpatterns)


from app.services.spotify_service import SpotifyService
import app.data.models as models
import datetime
import app.data.mongo_functions as mongo_utils

def save_or_update_user_data(access_token, refresh_token):
    spotify_service = SpotifyService(access_token, refresh_token)
    user_spotify_data = spotify_service.get_user_spotify_data()
    if user_spotify_data is not None and "email" not in user_spotify_data:
        return None
    user = mongo_utils.find_user_by_email(user_spotify_data["email"])
    if not user:
        user_top_artists = spotify_service.get_top_artists()
        user_data = models.user_from_spotify_data(user_spotify_data, user_top_artists)
        return mongo_utils.save_user_data(user_data)
    else:
        # if last updated < 2 days
        time_difference = datetime.datetime.utcnow() - user.last_update
        # update every 2 days
        if time_difference.days > 2:
            user_top_artists = spotify_service.get_top_artists()
            user_data = models.user_from_spotify_data(user_spotify_data, user_top_artists)
            return mongo_utils.save_user_data(user_data)
        else:
            return user

def update_user_location(user, location):
    return mongo_utils.update_user_location(user, location)

def get_near_users(user):
    user_db = mongo_utils.find_user_by_email(user.email)
    return [u.name for u in mongo_utils.find_users_in_distance_for_user(user_db)]

def login_user():
    pass


# -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import unicode_literals

from inlineplz.parsers.base import ParserBase

class CodenarcParser(ParserBase):
    """Parse Codenarc output."""

    def parse(self, lint_data):
        messages = set()
        path = ""
        msg = ""
        line_no = -1
        for line in lint_data.split("\n"):
            try:
                if line.strip().startswith("File:"):
                    path = line.split("File:")[-1].strip()
                    continue

                if line.strip().startswith("Violation:"):
                    parts = line.strip().split()
                    line_no = int(parts[3].split("=")[-1])
                    msg = line.strip()
                else:
                    msg += "\n" + line
                if "Src=" in line:
                    messages.add((path, line_no, msg))
                    msg = ""
            except (ValueError, IndexError, TypeError):
                print("Invalid message: {0}".format(line))
        return messages


import os
import cv2
import numpy as np

############################################################################################################

############################################################################################################
####### Still in development, update soon.
############################################################################################################

import tensorflow as tf
from object_detection.utils import label_map_util
from object_detection.utils import visualization_utils as viz_utils
from object_detection.builders import model_builder
from object_detection.utils import config_util
from object_detection.protos import pipeline_pb2
from google.protobuf import text_format

###########################################################################################################

class LymphoCounterModel:
    def __init__(self, IMAGE_PATH, ANNOTATION_PATH, CONFIG_PATH, CHECKPOINT_PATH):
        self.image_path = IMAGE_PATH
        self.annotation_path = ANNOTATION_PATH
        self.config_path = CONFIG_PATH
        self.checkpoint_path = CHECKPOINT_PATH

    def load_config(self):
        configs = config_util.get_configs_from_pipeline_file(self.config_path)
        detection_model = model_builder.build(model_config=configs['model'], is_training=False)

        ckpt = tf.compat.v2.train.Checkpoint(model=detection_model)
        ckpt.restore(self.checkpoint_path).expect_partial()
        return detection_model

    def cat_index(self):
        return label_map_util.create_category_index_from_labelmap(self.annotation_path)

    def load_image(self):
        frame = cv2.imread(self.image_path)
        frame_np = np.array(frame)
        return frame_np

    @tf.function
    def detect_fn(self, image):
    	detection_model = self.load_config()
    	image, shapes = detection_model.preprocess(image)
    	prediction_dict = detection_model.predict(image, shapes)
    	detections = detection_model.postprocess(prediction_dict, shapes)
    	return detections

    def detect_image(self, label_id_offset=1, save=True, extract=None, result=None):
        frame = self.load_image()
        category_index = self.cat_index()
        tf.config.run_functions_eagerly(True)

        input_tensor = tf.convert_to_tensor(np.expand_dims(frame, 0), dtype=tf.float32)
        detections = self.detect_fn(input_tensor)

        num_detections = int(detections.pop('num_detections'))
        detections = {key: value[0, :num_detections].numpy() for key, value in detections.items()}
        detections['num_detections'] = num_detections

        # detection_classes should be ints.
        detections['detection_classes'] = detections['detection_classes'].astype(np.int64)

        image_np_with_detections = frame.copy()

        image_np_with_detections = viz_utils.visualize_boxes_and_labels_on_image_array(
												image_np_with_detections,
												detections['detection_boxes'],
												detections['detection_classes']+label_id_offset,
												detections['detection_scores'],
												category_index,
												use_normalized_coordinates=True,
												max_boxes_to_draw=20,
												min_score_thresh=.5,
												agnostic_mode=False)

        # print(image_np_with_detections)

        result_len = len(os.listdir(result)) + 1
        result_paths = os.path.join(result, f"{result_len}.jpg")
        status = cv2.imwrite(result_paths, image_np_with_detections)

        return result_paths

############################################################################################################

############################################################################################################
####### Down below only for development process, use face detection model from built in Open Cv2 module
############################################################################################################

# def get_cascade(cascade_xml='haarcascade_frontalface_default.xml'):
# 	"""
# 	func: get classifier model
# 	input: name of xml param
# 	output: model object
# 	"""
# 	cascade_path = os.path.join(cv2.data.haarcascades, cascade_xml)
# 	cascade = cv2.CascadeClassifier(cascade_path)
# 	return cascade

# def load_image(image_path):
# 	"""
# 	func: load image from folder
# 	input: image path
# 	output: frame of image (readable)
# 	"""
# 	frame = cv2.imread(image_path)
# 	return frame

# def faces_extract(frame, extract=True, result=True, extract_dir=None, result_dir=None):
# 	"""
# 	func: main process, from preprocess up to giving result
# 	input: frame img, save extract img or not, save result img or not, extract dir path, result dir path
# 	output: image path from respective folder, depends on condition
# 	"""
# 	cascade = get_cascade()

# 	## preprocess image from frame
# 	gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# 	faces = cascade.detectMultiScale(gray, scaleFactor=1.5, minNeighbors=5, minSize=(30, 30))

# 	final_dest = None
# 	if extract_dir != None:
# 		## check for extract directory
# 		extract_len = len(os.listdir(extract_dir)) + 1
# 		final_dest = os.path.join(extract_dir, f"Extract_{extract_len}")
# 		os.makedirs(final_dest, exist_ok=True)

# 	## bounding box coordinate checking
# 	extract_paths = []
# 	for i, (x, y, w, h) in enumerate(faces):
# 		## create bounding box
# 		cv2.rectangle(frame, (x, y), (x+w, y+h), (255, 0, 0), 2)
# 		## for counter
# 		roi_color = frame[y:y+h, x:x+w]
# 		if extract == True:
# 			current_path = os.path.join(final_dest, f"{i+1}.jpg")
# 			cv2.imwrite(current_path, roi_color)
# 			extract_paths.append(current_path)

# 	if result == True and result_dir != None:
# 		result_len = len(os.listdir(result_dir)) + 1
# 		result_paths = os.path.join(result_dir, f"{result_len}.jpg")
# 		status = cv2.imwrite(result_paths, frame)
# 		return result_paths

# 	return extract_paths, extract_len


import setuptools

VERSION = "4.2.0"

setuptools.setup(
    name='pytest_mproc',
    author='John Rusnak',
    author_email='jrusnak69@gmail.com',
    version=VERSION,
    description="low-startup-overhead, scalable, distributed-testing pytest plugin",
    package_dir={'': 'src'},
    packages=setuptools.find_packages('src'),
    entry_points={
       "pytest11": ["name_of_plugin = pytest_mproc.plugin"],
    },
    classifiers=["Framework :: Pytest",
                 "Development Status :: 4 - Beta",
                 "License :: OSI Approved :: BSD License"],
    license='BSD 2-CLAUSE',
    keywrds='pytest distributed multiprocessing',
    url='https://github.com/nak/pytest_mproc',
    download_url="https://github.com/nak/pytest_mproc/dist/%s" % VERSION,
    install_requires=[
        'pytest',
        'pytest-cov'
    ]
)


from typing import List

class Solution:
  """
  Given an m x n matrix, return all elements of the matrix in spiral order.
  """
  def spiralOrder(self, matrix: List[List[int]]) -> List[int]:
    res = []
    r, c = len(matrix), len(matrix[0])
    seen = [[False for _ in range(c)] for _ in range(r)]
    print(seen)
    x_steps = [0, 1, 0, -1]
    y_steps = [1, 0, -1, 0]
    x = y = s = 0
    for _ in range(r * c):
      res.append(matrix[x][y])
      seen[x][y] = True
      xt = x + x_steps[s]
      yt = y + y_steps[s]
      if 0 <= xt < r and 0 <= yt < c and not seen[xt][yt]:
        x = xt
        y = yt
      else:
        s = (s + 1) % 4
        x += x_steps[s]
        y += y_steps[s]
    return res


import time
import datetime
import typing

import jk_utils
import jk_logging
from jk_typing import checkFunctionSignature

from .ProcessingFallThroughError import ProcessingFallThroughError
from .ThaniyaBackupContext import ThaniyaBackupContext
from .BD2 import BD2

"""
def thaniya_context(description):
	def thaniya_context2(func):
		def wrapped(*wargs, **kwargs):
			print(func)
			print(func.__annotations__)
			kwargs["c"] = "cc"
			return 'I got a wrapped up {} for you'.format(str(func(*wargs, **kwargs)))
		return wrapped
	return thaniya_context2
#
"""

"""
class MeasureDuration(object):

	def __init__(self, taskName:str, log:jk_logging.AbstractLogger):
		self.__taskName = taskName
		self.__log = log

		self.__t0 = None
	#

	def __enter__(self):
		self.__t0 = time.time()
		d0 = datetime.datetime.fromtimestamp(self.__t0)
		self.__log.info("Starting " + self.__taskName + " at: " + str(d0) + " (" + str(int(self.__t0)) + ")")
	#

	def __exit__(self, ex_type, exception, ex_traceback):
		t1 = time.time()
		d1 = datetime.datetime.fromtimestamp(t1)
		self.__log.info("Terminating " + self.__taskName + " at: " + str(d1) + " (" + str(int(t1)) + ")")

		fDurationSeconds = t1 - self.__t0
		self.__log.info("Time spent on " + self.__taskName + ": " + jk_utils.formatTime(fDurationSeconds))
	#

#
"""

#
# This context is similar to ThaniyaBackupContext, but it comprises a major processing step and measures performance.
#
class ProcessingContext(object):

	################################################################################################################################
	## Constructor
	################################################################################################################################

	#
	# @param	str name						The text to write to the log on descend.
	# @param	str targetDirPath				The destination directory to write files to. This argument is <c>None</c> if a specific
	#											section of activities (wrapped by this context) should not perform any writing.
	# @param	bool bMeasureDuration			If <c>True</c> after completing this context time measurement information is written to the log.
	# @param	str statsDurationKey			The name of the variable under which to store the stats data if processing is completed.
	#
	@checkFunctionSignature()
	def __init__(self,
		text:str,
		bd2:BD2,
		bMeasureDuration:bool,
		statsDurationKey:typing.Union[str,None],
		):

		self.__bd2 = bd2
		self.__text = text
		self.__nestedLog = None
		self.__bMeasureDuration = bMeasureDuration
		self.__statsDurationKey = statsDurationKey

		self.__t0 = None
		self.__t1 = None
	#

	################################################################################################################################
	## Public Properties
	################################################################################################################################

	@property
	def log(self) -> jk_logging.AbstractLogger:
		return self.__nestedLog
	#

	@property
	def bd2(self) -> BD2:
		return self.__bd2
	#

	#
	# The duration measured from the time when the context is entered.
	#
	@property
	def duration(self) -> float:
		if self.__t0 is None:
			return -1
		else:
			if self.__t1 is None:
				return time.time() - self.__t0
			else:
				return self.__t1 - self.__t0
	#

	################################################################################################################################
	## Public Methods
	################################################################################################################################

	def __enter__(self):
		self.__nestedLog = self.__bd2.log.descend(self.__text)

		self.__t0 = time.time()
		# d0 = datetime.datetime.fromtimestamp(self.__t0)
		# self.__nestedLog.info("Starting this activity at: " + str(d0) + " (" + str(int(self.__t0)) + ")")

		return ThaniyaBackupContext(self.__bd2, self.__nestedLog)
	#

	def __exit__(self, ex_type, ex_value, ex_traceback):
		self.__t1 = time.time()
		#d1 = datetime.datetime.fromtimestamp(t1)
		fDurationSeconds = self.__t1 - self.__t0

		if ex_type:
			self.__bd2.setErrorFlag()
			if (ex_type != jk_logging.ExceptionInChildContextException) and (ex_type != ProcessingFallThroughError):
				self.__nestedLog.error(ex_value)
				raise ProcessingFallThroughError()

		# NOTE: we skip fall through errors as they already have been logged

		#if exception and (exception is not jk_logging.AbstractLogger._EINSTANCE):
		if ex_value and not isinstance(ex_value, ProcessingFallThroughError) and not ex_type.__name__.endswith("ExceptionInChildContextException"):			# TODO: simplify!
			self.__nestedLog.error(ex_value)

		if self.__bMeasureDuration:
			if ex_value:
				#self.__nestedLog.error("Terminating with error at: " + str(d1) + " (" + str(int(t1)) + ")")
				#self.__nestedLog.error("Time spent: " + jk_utils.formatTime(fDurationSeconds))
				self.__nestedLog.notice("Terminating with error after: " + jk_utils.formatTime(fDurationSeconds))
			else:
				#self.__nestedLog.success("Terminating without error at: " + str(d1) + " (" + str(int(t1)) + ")")
				#self.__nestedLog.success("Time spent: " + jk_utils.formatTime(fDurationSeconds))
				self.__nestedLog.notice("Terminating with success after: " + jk_utils.formatTime(fDurationSeconds))

		if self.__bMeasureDuration:
			if not ex_value:
				if (self.__bd2.statsContainer is not None) and self.__statsDurationKey:
					self.__bd2.statsContainer.setValue(self.__statsDurationKey, fDurationSeconds)

		if ex_value and not isinstance(ex_value, ProcessingFallThroughError) and not ex_value.__class__.__name__.endswith("ExceptionInChildContextException"):			# TODO: simplify!
			raise ProcessingFallThroughError()
	#

#


import sys
import os
import json
import openpyxl
from openpyxl import workbook, worksheet
import pandas as pd

filepath = r'\\Galileo\Public\Legal Intelligence\Customer Segmentation\BA\Ad Hoc Reports & Requests\2019\201909 - September\DAI-2093 - Kenneth Ume - Market Product Penetration Data Request - REPORT\checking.xlsx'
wb = openpyxl.load_workbook(filepath)
nomore = wb.get_sheet_by_name('nomore')
elist = []
for cell in nomore['A']:
    elist.append(cell.value)
    if cell.value is None:
        break
#print(elist)

scriptpath = r".\lawlibrary.py"
sys.path.append(os.path.abspath(scriptpath))
with open (r".\lawlibrary.py", "r+") as f:
    Dict = json.load(f)

kl = []
vl = []
for key, value in Dict.items():
    if value in elist:
        pass
    else:

        if value in ['Cripps','Pemberton Greenish']:
            value = 'Cripps Pemberton Greenish'
            #print([key,value])
        if value in ['Ince & Co' ,'Gordon Dadds']:
            value = 'Ince'
            #print ([key,value])
        if value in ['Pitmans' ,'BDB']:
            value = 'BDB Pitmans'
            #print([key, value])
        if value in ['Bond Dickinson','Womble']:
            value = 'Womble Bond Dickinson'
        # print([key, value])
        if value in ['Berwin Leighton Paisner','Bryan Cave Leighton Paisner']:
            value == 'BCLP'

        else:
            pass

        kl.append(key)
        vl.append(value)

new_dic = dict()
for key, value in zip (kl, vl):
    new_dic[key] = value
    dict.update(new_dic)

Dict.update(new_dic)
with open(r'.\lawlibrary.py', 'w') as outfile:
   json.dump(Dict, outfile)
#print(Dict)


import logging

from enthought.io.api import File
from enthought.pyface.api import FileDialog, OK
from enthought.pyface.action.api import Action
from enthought.traits.api import Any

from cviewer.plugins.text_editor.editor.text_editor import TextEditor
from cviewer.plugins.ui.preference_manager import preference_manager

# Logging imports
import logging
logger = logging.getLogger('root.'+__name__)

class ComputeNBS(Action):
    tooltip = "Compute NBS"
    description = "Compute NBS"

    # The WorkbenchWindow the action is attached to.
    window = Any()

    def perform(self, event=None):

        from cnetwork_nbs_action import NBSNetworkParameter, NBSMoreParameter
        from scripts import nbsscript
        cfile = self.window.application.get_service('cviewer.plugins.cff2.cfile.CFile')

        no = NBSNetworkParameter(cfile)
        no.edit_traits(kind='livemodal')

        if (len(no.selected1) == 0 or len(no.selected2) == 0):
            return

        mo = NBSMoreParameter(cfile, no.selected1[0], no.selected2[0])
        mo.edit_traits(kind='livemodal')

        import datetime as dt
        a=dt.datetime.now()
        ostr = '%s%s%s' % (a.hour, a.minute, a.second)

        if not (len(no.selected1) == 0 or len(no.selected2) == 0):
            # if cancel, not create surface
            # create a temporary file
            import tempfile
            myf = tempfile.mktemp(suffix='.py', prefix='my')
            f=open(myf, 'w')
            f.write(nbsscript % (str(no.selected1),
                                 mo.first_edge_value,
                                 str(no.selected2),
                                 mo.second_edge_value,
                                 mo.THRES,
                                 mo.K,
                                 mo.TAIL,
                                 ostr))
            f.close()

            self.window.workbench.edit(File(myf), kind=TextEditor,use_existing=False)

class ShowNetworks(Action):
    tooltip = "Create a surface"
    description = "Create a surface"

    # The WorkbenchWindow the action is attached to.
    window = Any()

    def perform(self, event=None):

        from cnetwork_action import NetworkParameter
        from scripts import netscript
        cfile = self.window.application.get_service('cviewer.plugins.cff2.cfile.CFile')

        no = NetworkParameter(cfile)
        no.edit_traits(kind='livemodal')

        if not no.netw[no.graph]['name'] == "None":
            # if cancel, not create surface
            # create a temporary file
            import tempfile
            myf = tempfile.mktemp(suffix='.py', prefix='my')
            f=open(myf, 'w')
            f.write(netscript % (no.netw[no.graph]['name'],
                                  no.node_position,
                                  no.edge_value,
                                  no.node_label))
            f.close()

            self.window.workbench.edit(File(myf), kind=TextEditor,use_existing=False)

class ShowSurfaces(Action):
    """ Open a new file in the text editor
    """
    tooltip = "Create a surface"
    description = "Create a surface"

    # The WorkbenchWindow the action is attached to.
    window = Any()

    def perform(self, event=None):

        from csurface_action import SurfaceParameter
        from scripts import surfscript
        cfile = self.window.application.get_service('cviewer.plugins.cff2.cfile.CFile')

        so = SurfaceParameter(cfile)
        so.edit_traits(kind='livemodal')

        if not so.pointset_da[so.pointset]['name'] == "None":
            # if cancel, not create surface
            # create a temporary file
            import tempfile
            myf = tempfile.mktemp(suffix='.py', prefix='my')
            f=open(myf, 'w')
            if so.labels_da[so.labels].has_key('da_idx'):
                labels = so.labels_da[so.labels]['da_idx']
            else:
                labels = ""
            f.write(surfscript % (so.pointset_da[so.pointset]['name'],
                                  so.pointset_da[so.pointset]['da_idx'],
                                  so.faces_da[so.faces]['name'],
                                  so.faces_da[so.faces]['da_idx'],
                                  so.labels_da[so.labels]['name'],
                                  labels))
            f.close()

            self.window.workbench.edit(File(myf), kind=TextEditor,use_existing=False)

class ShowVolumes(Action):
    """ Open a new file in the text editor
    """
    tooltip = "Create a volume"
    description = "Create a volume"

    # The WorkbenchWindow the action is attached to.
    window = Any()

    def perform(self, event=None):

        from cvolume_action import VolumeParameter
        from scripts import volslice
        cfile = self.window.application.get_service('cviewer.plugins.cff2.cfile.CFile')

        so = VolumeParameter(cfile)
        so.edit_traits(kind='livemodal')

        if True: #not so.pointset_da[so.pointset]['name'] == "None":
            # if cancel, not create surface
            # create a temporary file
            import tempfile
            myf = tempfile.mktemp(suffix='.py', prefix='my')
            f=open(myf, 'w')
            f.write(volslice % so.volumes[so.myvolume]['name'])
            f.close()

            self.window.workbench.edit(File(myf), kind=TextEditor,use_existing=False)


"""
Example of how to train the Guided Cost Learning (GCL) algorithm from scratch.

Also includes notes on how to resume training from an earlier checkpoint,
perform testing/evaluation, and run the baselines from the model_zoo.
"""
import logging
import os

from ilpyt.agents.gcl_agent import GCLAgent
from ilpyt.agents.ppo_agent import PPOAgent
from ilpyt.algos.gcl import GCL
from ilpyt.utils.env_utils import build_env
from ilpyt.utils.net_utils import choose_net
from ilpyt.utils.seed_utils import set_seed

def build(
    save_path: str,
    load_path: str,
    env_id: str,
    num_env: int,
    use_gpu: bool,
    seed: int = 24,
):
    # Set random seed
    set_seed(seed)

    # Build environment
    env = build_env(env_id=env_id, num_env=num_env, seed=seed)

    # Build agent
    actor = PPOAgent(
        actor=choose_net(env, activation='tanh'),
        critic=choose_net(env, activation='tanh', output_shape=1),
        lr=0.0009,
        gamma=0.99,
        clip_ratio=0.05,
        entropy_coeff=0.05,
    )
    agent = GCLAgent(
        actor=actor,
        cost=choose_net(
            env, activation='relu', output_shape=1, with_action_shape=True
        ),
        lr=0.00045,
        lcr_reg_cost=False,
        mono_reg_cost=False,
    )

    # Build learner
    algo = GCL(
        env=env,
        agent=agent,
        use_gpu=use_gpu,
        save_path=save_path,
        load_path=load_path,
    )

    return algo

def evaluate_baselines():
    envs = ['CartPole-v0', 'LunarLander-v2', 'LunarLanderContinuous-v2']

    for env in envs:
        logging.debug(env)
        save_path = os.path.join('logs/GCL/', env)
        load_path = os.path.join('model_zoo/GCL/', env)

        # Build experiment -----
        algo = build(
            save_path=save_path,
            load_path=load_path,
            env_id=env,
            num_env=16,
            use_gpu=True,
        )

        algo.env.close()

        algo.test(num_episodes=100)

if __name__ == '__main__':

    # NOTE
    # To train a new model:
    #       save_path = 'dir/to/save/to/
    #       load_path = ''
    # To continue training a model:
    #       save_path = 'dir/to/save/to/
    #       load_path = 'dir/to/load/from/'
    # To test an old model:
    #       Comment out train() method and algo.agent.load()
    #       save_path = 'dir/to/save/to/'
    #       load_path = 'dir/to/load/from/'

    if not os.path.exists('logs/GCL'):
        os.makedirs('logs/GCL')

    # LunarLander-v2 -----------------------------------------------------------

    save_path = 'logs/GCL_LunarLander-v2'
    load_path = ''

    # Build experiment
    algo = build(
        save_path=save_path,
        load_path=load_path,
        env_id='LunarLander-v2',
        num_env=16,
        use_gpu=True,
    )

    # Train
    algo.train(
        num_episodes=10000,
        expert_demos='/mnt/IMLEARN/demos/LunarLander-v2/demos.pkl',
    )
    # Close training environmnet
    algo.env.close()

    # Load
    algo.agent.load(save_path)

    # Test
    algo.test(num_episodes=100)

    # CartPole-v0 --------------------------------------------------------------

    save_path = 'logs/GCL/CartPole-v0'
    load_path = ''

    # Build experiment
    algo = build(
        save_path=save_path,
        load_path=load_path,
        env_id='CartPole-v0',
        num_env=16,
        use_gpu=True,
    )

    # Train
    algo.train(
        num_episodes=10000,
        expert_demos='demos/CartPole-v0/demos.pkl',
    )
    # Close training environmnet
    algo.env.close()

    # Load
    algo.agent.load(save_path)

    # Test
    algo.test(num_episodes=100)

    # LunarLanderContinuous-v2 -------------------------------------------------

    save_path = 'logs/GCL/LunarLanderContinuous-v2'
    load_path = ''

    # Build experiment
    algo = build(
        save_path=save_path,
        load_path=load_path,
        env_id='LunarLanderContinuous-v2',
        num_env=16,
        use_gpu=True,
    )

    # Train
    algo.train(
        num_episodes=10000,
        expert_demos='/mnt/IMLEARN/demos/LunarLanderContinuous-v2/demos.pkl',
    )
    # Close environment
    algo.env.close()

    # Load
    algo.agent.load(save_path)

    # Test
    algo.test(num_episodes=100)