#!/usr/bin/env python3

import ply.lex as lex
import ply.yacc as yacc
import sys
import argparse
import keyword
import logging
import binascii
import os
import sys
from subprocess import Popen, PIPE

assert sys.version_info >= (3, 6), \
    "Not supported Python version: {}".format(sys.version)
log = logging.getLogger('vppapigen')

# Ensure we don't leave temporary files around
sys.dont_write_bytecode = True

#
# VPP API language
#

# Global dictionary of new types (including enums)
global_types = {}

seen_imports = {}

def global_type_add(name, obj):
    '''Add new type to the dictionary of types '''
    type_name = 'vl_api_' + name + '_t'
    if type_name in global_types:
        raise KeyError("Attempted redefinition of {!r} with {!r}.".format(
            name, obj))
    global_types[type_name] = obj

# All your trace are belong to us!
def exception_handler(exception_type, exception, traceback):
    print("%s: %s" % (exception_type.__name__, exception))

#
# Lexer
#
class VPPAPILexer(object):
    def __init__(self, filename):
        self.filename = filename

    reserved = {
        'service': 'SERVICE',
        'rpc': 'RPC',
        'returns': 'RETURNS',
        'null': 'NULL',
        'stream': 'STREAM',
        'events': 'EVENTS',
        'define': 'DEFINE',
        'typedef': 'TYPEDEF',
        'enum': 'ENUM',
        'typeonly': 'TYPEONLY',
        'manual_print': 'MANUAL_PRINT',
        'manual_endian': 'MANUAL_ENDIAN',
        'dont_trace': 'DONT_TRACE',
        'autoreply': 'AUTOREPLY',
        'option': 'OPTION',
        'u8': 'U8',
        'u16': 'U16',
        'u32': 'U32',
        'u64': 'U64',
        'i8': 'I8',
        'i16': 'I16',
        'i32': 'I32',
        'i64': 'I64',
        'f64': 'F64',
        'bool': 'BOOL',
        'string': 'STRING',
        'import': 'IMPORT',
        'true': 'TRUE',
        'false': 'FALSE',
        'union': 'UNION',
    }

    tokens = ['STRING_LITERAL',
              'ID', 'NUM'] + list(reserved.values())

    t_ignore_LINE_COMMENT = '//.*'

    def t_FALSE(self, t):
        r'false'
        t.value = False
        return t

    def t_TRUE(self, t):
        r'false'
        t.value = True
        return t

    def t_NUM(self, t):
        r'0[xX][0-9a-fA-F]+|-?\d+\.?\d*'
        base = 16 if t.value.startswith('0x') else 10
        if '.' in t.value:
            t.value = float(t.value)
        else:
            t.value = int(t.value, base)
        return t

    def t_ID(self, t):
        r'[a-zA-Z_][a-zA-Z_0-9]*'
        # Check for reserved words
        t.type = VPPAPILexer.reserved.get(t.value, 'ID')
        return t

    # C string
    def t_STRING_LITERAL(self, t):
        r'\"([^\\\n]|(\\.))*?\"'
        t.value = str(t.value).replace("\"", "")
        return t

    # C or C++ comment (ignore)
    def t_comment(self, t):
        r'(/\*(.|\n)*?\*/)|(//.*)'
        t.lexer.lineno += t.value.count('\n')

    # Error handling rule
    def t_error(self, t):
        raise ParseError("Illegal character '{}' ({})"
                         "in {}: line {}".format(t.value[0],
                                                 hex(ord(t.value[0])),
                                                 self.filename,
                                                 t.lexer.lineno))

    # Define a rule so we can track line numbers
    def t_newline(self, t):
        r'\n+'
        t.lexer.lineno += len(t.value)

    literals = ":{}[];=.,"

    # A string containing ignored characters (spaces and tabs)
    t_ignore = ' \t'

def crc_block_combine(block, crc):
    s = str(block).encode()
    return binascii.crc32(s, crc) & 0xffffffff

def vla_is_last_check(name, block):
    vla = False
    for i, b in enumerate(block):
        if isinstance(b, Array) and b.vla:
            vla = True
            if i + 1 < len(block):
                raise ValueError(
                    'VLA field "{}" must be the last field in message "{}"'
                    .format(b.fieldname, name))
        elif b.fieldtype.startswith('vl_api_'):
            if global_types[b.fieldtype].vla:
                vla = True
                if i + 1 < len(block):
                    raise ValueError(
                        'VLA field "{}" must be the last '
                        'field in message "{}"'
                        .format(b.fieldname, name))
        elif b.fieldtype == 'string' and b.length == 0:
            vla = True
            if i + 1 < len(block):
                raise ValueError(
                    'VLA field "{}" must be the last '
                    'field in message "{}"'
                    .format(b.fieldname, name))
    return vla

class Service():
    def __init__(self, caller, reply, events=None, stream_message=None, stream=False):
        self.caller = caller
        self.reply = reply
        self.stream = stream
        self.stream_message = stream_message
        self.events = [] if events is None else events

class Typedef():
    def __init__(self, name, flags, block):
        self.name = name
        self.flags = flags
        self.block = block
        self.crc = str(block).encode()
        self.manual_print = False
        self.manual_endian = False
        for f in flags:
            if f == 'manual_print':
                self.manual_print = True
            elif f == 'manual_endian':
                self.manual_endian = True

        global_type_add(name, self)

        self.vla = vla_is_last_check(name, block)

    def __repr__(self):
        return self.name + str(self.flags) + str(self.block)

class Using():
    def __init__(self, name, flags, alias):
        self.name = name
        self.vla = False
        self.block = []
        self.manual_print = True
        self.manual_endian = True

        self.manual_print = False
        self.manual_endian = False
        for f in flags:
            if f == 'manual_print':
                self.manual_print = True
            elif f == 'manual_endian':
                self.manual_endian = True

        if isinstance(alias, Array):
            a = {'type': alias.fieldtype,
                 'length': alias.length}
        else:
            a = {'type': alias.fieldtype}
        self.alias = a
        self.crc = str(alias).encode()
        global_type_add(name, self)

    def __repr__(self):
        return self.name + str(self.alias)

class Union():
    def __init__(self, name, flags, block):
        self.type = 'Union'
        self.manual_print = False
        self.manual_endian = False
        self.name = name

        for f in flags:
            if f == 'manual_print':
                self.manual_print = True
            elif f == 'manual_endian':
                self.manual_endian = True

        self.block = block
        self.crc = str(block).encode()
        self.vla = vla_is_last_check(name, block)

        global_type_add(name, self)

    def __repr__(self):
        return str(self.block)

class Define():
    def __init__(self, name, flags, block):
        self.name = name
        self.flags = flags
        self.block = block
        self.dont_trace = False
        self.manual_print = False
        self.manual_endian = False
        self.autoreply = False
        self.singular = False
        self.options = {}
        for f in flags:
            if f == 'dont_trace':
                self.dont_trace = True
            elif f == 'manual_print':
                self.manual_print = True
            elif f == 'manual_endian':
                self.manual_endian = True
            elif f == 'autoreply':
                self.autoreply = True

        remove = []
        for b in block:
            if isinstance(b, Option):
                if b[1] == 'singular' and b[2] == 'true':
                    self.singular = True
                else:
                    self.options[b.option] = b.value
                remove.append(b)

        block = [x for x in block if x not in remove]
        self.block = block
        self.vla = vla_is_last_check(name, block)
        self.crc = str(block).encode()

    def __repr__(self):
        return self.name + str(self.flags) + str(self.block)

class Enum():
    def __init__(self, name, block, enumtype='u32'):
        self.name = name
        self.enumtype = enumtype
        self.vla = False

        count = 0
        for i, b in enumerate(block):
            if type(b) is list:
                count = b[1]
            else:
                count += 1
                block[i] = [b, count]

        self.block = block
        self.crc = str(block).encode()
        global_type_add(name, self)

    def __repr__(self):
        return self.name + str(self.block)

class Import():

    def __new__(cls, *args, **kwargs):
        if args[0] not in seen_imports:
            instance = super().__new__(cls)
            instance._initialized = False
            seen_imports[args[0]] = instance

        return seen_imports[args[0]]

    def __init__(self, filename, revision):
        if self._initialized:
            return
        else:
            self.filename = filename
            # Deal with imports
            parser = VPPAPI(filename=filename, revision=revision)
            dirlist = dirlist_get()
            f = filename
            for dir in dirlist:
                f = os.path.join(dir, filename)
                if os.path.exists(f):
                    break
            self.result = parser.parse_filename(f, None)
            self._initialized = True

    def __repr__(self):
        return self.filename

class Option():
    def __init__(self, option, value=None):
        self.type = 'Option'
        self.option = option
        self.value = value
        self.crc = str(option).encode()

    def __repr__(self):
        return str(self.option)

    def __getitem__(self, index):
        return self.option[index]

class Array():
    def __init__(self, fieldtype, name, length, modern_vla=False):
        self.type = 'Array'
        self.fieldtype = fieldtype
        self.fieldname = name
        self.modern_vla = modern_vla
        if type(length) is str:
            self.lengthfield = length
            self.length = 0
            self.vla = True
        else:
            self.length = length
            self.lengthfield = None
            self.vla = False

    def __repr__(self):
        return str([self.fieldtype, self.fieldname, self.length,
                    self.lengthfield])

class Field():
    def __init__(self, fieldtype, name, limit=None):
        self.type = 'Field'
        self.fieldtype = fieldtype

        if self.fieldtype == 'string':
            raise ValueError("The string type {!r} is an "
                             "array type ".format(name))

        if name in keyword.kwlist:
            raise ValueError("Fieldname {!r} is a python keyword and is not "
                             "accessible via the python API. ".format(name))
        self.fieldname = name
        self.limit = limit

    def __repr__(self):
        return str([self.fieldtype, self.fieldname])

class Coord(object):
    """ Coordinates of a syntactic element. Consists of:
            - File name
            - Line number
            - (optional) column number, for the Lexer
    """
    __slots__ = ('file', 'line', 'column', '__weakref__')

    def __init__(self, file, line, column=None):
        self.file = file
        self.line = line
        self.column = column

    def __str__(self):
        str = "%s:%s" % (self.file, self.line)
        if self.column:
            str += ":%s" % self.column
        return str

class ParseError(Exception):
    pass

#
# Grammar rules
#
class VPPAPIParser(object):
    tokens = VPPAPILexer.tokens

    def __init__(self, filename, logger, revision=None):
        self.filename = filename
        self.logger = logger
        self.fields = []
        self.revision = revision

    def _parse_error(self, msg, coord):
        raise ParseError("%s: %s" % (coord, msg))

    def _parse_warning(self, msg, coord):
        if self.logger:
            self.logger.warning("%s: %s" % (coord, msg))

    def _coord(self, lineno, column=None):
        return Coord(
                file=self.filename,
                line=lineno, column=column)

    def _token_coord(self, p, token_idx):
        """ Returns the coordinates for the YaccProduction object 'p' indexed
            with 'token_idx'. The coordinate includes the 'lineno' and
            'column'. Both follow the lex semantic, starting from 1.
        """
        last_cr = p.lexer.lexdata.rfind('\n', 0, p.lexpos(token_idx))
        if last_cr < 0:
            last_cr = -1
        column = (p.lexpos(token_idx) - (last_cr))
        return self._coord(p.lineno(token_idx), column)

    def p_slist(self, p):
        '''slist : stmt
                 | slist stmt'''
        if len(p) == 2:
            p[0] = [p[1]]
        else:
            p[0] = p[1] + [p[2]]

    def p_stmt(self, p):
        '''stmt : define
                | typedef
                | option
                | import
                | enum
                | union
                | service'''
        p[0] = p[1]

    def p_import(self, p):
        '''import : IMPORT STRING_LITERAL ';' '''
        p[0] = Import(p[2], revision=self.revision)

    def p_service(self, p):
        '''service : SERVICE '{' service_statements '}' ';' '''
        p[0] = p[3]

    def p_service_statements(self, p):
        '''service_statements : service_statement
                        | service_statements service_statement'''
        if len(p) == 2:
            p[0] = [p[1]]
        else:
            p[0] = p[1] + [p[2]]

    def p_service_statement(self, p):
        '''service_statement : RPC ID RETURNS NULL ';'
                             | RPC ID RETURNS ID ';'
                             | RPC ID RETURNS STREAM ID ';'
                             | RPC ID RETURNS ID EVENTS event_list ';' '''
        if p[2] == p[4]:
            # Verify that caller and reply differ
            self._parse_error(
                'Reply ID ({}) should not be equal to Caller ID'.format(p[2]),
                self._token_coord(p, 1))
        if len(p) == 8:
            p[0] = Service(p[2], p[4], p[6])
        elif len(p) == 7:
            p[0] = Service(p[2], p[5], stream=True)
        else:
            p[0] = Service(p[2], p[4])

    def p_service_statement2(self, p):
        '''service_statement : RPC ID RETURNS ID STREAM ID ';' '''
        p[0] = Service(p[2], p[4], stream_message=p[6], stream=True)

    def p_event_list(self, p):
        '''event_list : events
                      | event_list events '''
        if len(p) == 2:
            p[0] = [p[1]]
        else:
            p[0] = p[1] + [p[2]]

    def p_event(self, p):
        '''events : ID
                  | ID ',' '''
        p[0] = p[1]

    def p_enum(self, p):
        '''enum : ENUM ID '{' enum_statements '}' ';' '''
        p[0] = Enum(p[2], p[4])

    def p_enum_type(self, p):
        ''' enum : ENUM ID ':' enum_size '{' enum_statements '}' ';' '''
        if len(p) == 9:
            p[0] = Enum(p[2], p[6], enumtype=p[4])
        else:
            p[0] = Enum(p[2], p[4])

    def p_enum_size(self, p):
        ''' enum_size : U8
                      | U16
                      | U32 '''
        p[0] = p[1]

    def p_define(self, p):
        '''define : DEFINE ID '{' block_statements_opt '}' ';' '''
        self.fields = []
        p[0] = Define(p[2], [], p[4])

    def p_define_flist(self, p):
        '''define : flist DEFINE ID '{' block_statements_opt '}' ';' '''
        # Legacy typedef
        if 'typeonly' in p[1]:
            self._parse_error('legacy typedef. use typedef: {} {}[{}];'
                              .format(p[1], p[2], p[4]),
                              self._token_coord(p, 1))
        else:
            p[0] = Define(p[3], p[1], p[5])

    def p_flist(self, p):
        '''flist : flag
                 | flist flag'''
        if len(p) == 2:
            p[0] = [p[1]]
        else:
            p[0] = p[1] + [p[2]]

    def p_flag(self, p):
        '''flag : MANUAL_PRINT
                | MANUAL_ENDIAN
                | DONT_TRACE
                | TYPEONLY
                | AUTOREPLY'''
        if len(p) == 1:
            return
        p[0] = p[1]

    def p_typedef(self, p):
        '''typedef : TYPEDEF ID '{' block_statements_opt '}' ';' '''
        p[0] = Typedef(p[2], [], p[4])

    def p_typedef_flist(self, p):
        '''typedef : flist TYPEDEF ID '{' block_statements_opt '}' ';' '''
        p[0] = Typedef(p[3], p[1], p[5])

    def p_typedef_alias(self, p):
        '''typedef : TYPEDEF declaration '''
        p[0] = Using(p[2].fieldname, [], p[2])

    def p_typedef_alias_flist(self, p):
        '''typedef : flist TYPEDEF declaration '''
        p[0] = Using(p[3].fieldname, p[1], p[3])

    def p_block_statements_opt(self, p):
        '''block_statements_opt : block_statements '''
        p[0] = p[1]

    def p_block_statements(self, p):
        '''block_statements : block_statement
                            | block_statements block_statement'''
        if len(p) == 2:
            p[0] = [p[1]]
        else:
            p[0] = p[1] + [p[2]]

    def p_block_statement(self, p):
        '''block_statement : declaration
                           | option '''
        p[0] = p[1]

    def p_enum_statements(self, p):
        '''enum_statements : enum_statement
                           | enum_statements enum_statement'''
        if len(p) == 2:
            p[0] = [p[1]]
        else:
            p[0] = p[1] + [p[2]]

    def p_enum_statement(self, p):
        '''enum_statement : ID '=' NUM ','
                          | ID ',' '''
        if len(p) == 5:
            p[0] = [p[1], p[3]]
        else:
            p[0] = p[1]

    def p_field_options(self, p):
        '''field_options : field_option
                           | field_options field_option'''
        if len(p) == 2:
            p[0] = p[1]
        else:
            p[0] = {**p[1], **p[2]}

    def p_field_option(self, p):
        '''field_option : ID
                        | ID '=' assignee ','
                        | ID '=' assignee

        '''
        if len(p) == 2:
            p[0] = {p[1]: None}
        else:
            p[0] = {p[1]: p[3]}

    def p_declaration(self, p):
        '''declaration : type_specifier ID ';'
                       | type_specifier ID '[' field_options ']' ';' '''
        if len(p) == 7:
            p[0] = Field(p[1], p[2], p[4])
        elif len(p) == 4:
            p[0] = Field(p[1], p[2])
        else:
            self._parse_error('ERROR', self._token_coord(p, 1))
        self.fields.append(p[2])

    def p_declaration_array_vla(self, p):
        '''declaration : type_specifier ID '[' ']' ';' '''
        p[0] = Array(p[1], p[2], 0, modern_vla=True)

    def p_declaration_array(self, p):
        '''declaration : type_specifier ID '[' NUM ']' ';'
                       | type_specifier ID '[' ID ']' ';' '''

        if len(p) != 7:
            return self._parse_error(
                'array: %s' % p.value,
                self._coord(lineno=p.lineno))

        # Make this error later
        if type(p[4]) is int and p[4] == 0:
            # XXX: Line number is wrong
            self._parse_warning('Old Style VLA: {} {}[{}];'
                                .format(p[1], p[2], p[4]),
                                self._token_coord(p, 1))

        if type(p[4]) is str and p[4] not in self.fields:
            # Verify that length field exists
            self._parse_error('Missing length field: {} {}[{}];'
                              .format(p[1], p[2], p[4]),
                              self._token_coord(p, 1))
        p[0] = Array(p[1], p[2], p[4])

    def p_option(self, p):
        '''option : OPTION ID '=' assignee ';'
                  | OPTION ID ';' '''
        if len(p) == 4:
            p[0] = Option(p[2])
        else:
            p[0] = Option(p[2], p[4])

    def p_assignee(self, p):
        '''assignee : NUM
                    | TRUE
                    | FALSE
                    | STRING_LITERAL '''
        p[0] = p[1]

    def p_type_specifier(self, p):
        '''type_specifier : U8
                          | U16
                          | U32
                          | U64
                          | I8
                          | I16
                          | I32
                          | I64
                          | F64
                          | BOOL
                          | STRING'''
        p[0] = p[1]

    # Do a second pass later to verify that user defined types are defined
    def p_typedef_specifier(self, p):
        '''type_specifier : ID '''
        if p[1] not in global_types:
            self._parse_error('Undefined type: {}'.format(p[1]),
                              self._token_coord(p, 1))
        p[0] = p[1]

    def p_union(self, p):
        '''union : UNION ID '{' block_statements_opt '}' ';' '''
        p[0] = Union(p[2], [], p[4])

    def p_union_flist(self, p):
        '''union : flist UNION ID '{' block_statements_opt '}' ';' '''
        p[0] = Union(p[3], p[1], p[5])

    # Error rule for syntax errors
    def p_error(self, p):
        if p:
            self._parse_error(
                'before: %s' % p.value,
                self._coord(lineno=p.lineno))
        else:
            self._parse_error('At end of input', self.filename)

class VPPAPI(object):

    def __init__(self, debug=False, filename='', logger=None, revision=None):
        self.lexer = lex.lex(module=VPPAPILexer(filename), debug=debug)
        self.parser = yacc.yacc(module=VPPAPIParser(filename, logger,
                                                    revision=revision),
                                write_tables=False, debug=debug)
        self.logger = logger
        self.revision = revision
        self.filename = filename

    def parse_string(self, code, debug=0, lineno=1):
        self.lexer.lineno = lineno
        return self.parser.parse(code, lexer=self.lexer, debug=debug)

    def parse_fd(self, fd, debug=0):
        data = fd.read()
        return self.parse_string(data, debug=debug)

    def parse_filename(self, filename, debug=0):
        if self.revision:
            git_show = f'git show  {self.revision}:{filename}'
            proc = Popen(git_show.split(), stdout=PIPE, encoding='utf-8')
            try:
                data, errs = proc.communicate()
                if proc.returncode != 0:
                    print(f'File not found: {self.revision}:{filename}', file=sys.stderr)
                    sys.exit(2)
                return self.parse_string(data, debug=debug)
            except Exception as e:
                sys.exit(3)
        else:
            try:
                with open(filename, encoding='utf-8') as fd:
                    return self.parse_fd(fd, None)
            except FileNotFoundError:
                print(f'File not found: {filename}', file=sys.stderr)
                sys.exit(2)

    def autoreply_block(self, name, parent):
        block = [Field('u32', 'context'),
                 Field('i32', 'retval')]
        # inherhit the parent's options
        for k, v in parent.options.items():
            block.append(Option(k, v))
        return Define(name + '_reply', [], block)

    def process(self, objs):
        s = {}
        s['Option'] = {}
        s['Define'] = []
        s['Service'] = []
        s['types'] = []
        s['Import'] = []
        crc = 0
        for o in objs:
            tname = o.__class__.__name__
            try:
                crc = binascii.crc32(o.crc, crc) & 0xffffffff
            except AttributeError:
                pass
            if isinstance(o, Define):
                s[tname].append(o)
                if o.autoreply:
                    s[tname].append(self.autoreply_block(o.name, o))
            elif isinstance(o, Option):
                s[tname][o.option] = o.value
            elif type(o) is list:
                for o2 in o:
                    if isinstance(o2, Service):
                        s['Service'].append(o2)
            elif (isinstance(o, Enum) or
                  isinstance(o, Typedef) or
                  isinstance(o, Using) or
                  isinstance(o, Union)):
                s['types'].append(o)
            else:
                if tname not in s:
                    raise ValueError('Unknown class type: {} {}'
                                     .format(tname, o))
                s[tname].append(o)

        msgs = {d.name: d for d in s['Define']}
        svcs = {s.caller: s for s in s['Service']}
        replies = {s.reply: s for s in s['Service']}
        seen_services = {}

        s['file_crc'] = crc

        for service in svcs:
            if service not in msgs:
                raise ValueError(
                    'Service definition refers to unknown message'
                    ' definition: {}'.format(service))
            if svcs[service].reply != 'null' and \
               svcs[service].reply not in msgs:
                raise ValueError('Service definition refers to unknown message'
                                 ' definition in reply: {}'
                                 .format(svcs[service].reply))
            if service in replies:
                raise ValueError('Service definition refers to message'
                                 ' marked as reply: {}'.format(service))
            for event in svcs[service].events:
                if event not in msgs:
                    raise ValueError('Service definition refers to unknown '
                                     'event: {} in message: {}'
                                     .format(event, service))
                seen_services[event] = True

        # Create services implicitly
        for d in msgs:
            if d in seen_services:
                continue
            if msgs[d].singular is True:
                continue
            if d.endswith('_reply'):
                if d[:-6] in svcs:
                    continue
                if d[:-6] not in msgs:
                    raise ValueError('{} missing calling message'
                                     .format(d))
                continue
            if d.endswith('_dump'):
                if d in svcs:
                    continue
                if d[:-5]+'_details' in msgs:
                    s['Service'].append(Service(d, d[:-5]+'_details',
                                                stream=True))
                else:
                    raise ValueError('{} missing details message'
                                     .format(d))
                continue

            if d.endswith('_details'):
                if d[:-8]+'_dump' not in msgs:
                    raise ValueError('{} missing dump message'
                                     .format(d))
                continue

            if d in svcs:
                continue
            if d+'_reply' in msgs:
                s['Service'].append(Service(d, d+'_reply'))
            else:
                raise ValueError(
                    '{} missing reply message ({}) or service definition'
                    .format(d, d+'_reply'))

        return s

    def process_imports(self, objs, in_import, result):
        imported_objs = []
        for o in objs:
            # Only allow the following object types from imported file
            if in_import and not (isinstance(o, Enum) or
                                  isinstance(o, Union) or
                                  isinstance(o, Typedef) or
                                  isinstance(o, Import) or
                                  isinstance(o, Using)):
                continue
            if isinstance(o, Import):
                result.append(o)
                result = self.process_imports(o.result, True, result)
            else:
                result.append(o)
        return result

# Add message ids to each message.
def add_msg_id(s):
    for o in s:
        o.block.insert(0, Field('u16', '_vl_msg_id'))
    return s

dirlist = []

def dirlist_add(dirs):
    global dirlist
    if dirs:
        dirlist = dirlist + dirs

def dirlist_get():
    return dirlist

def foldup_blocks(block, crc):
    for b in block:
        # Look up CRC in user defined types
        if b.fieldtype.startswith('vl_api_'):
            # Recursively
            t = global_types[b.fieldtype]
            try:
                crc = crc_block_combine(t.block, crc)
                crc = foldup_blocks(t.block, crc)
            except AttributeError:
                pass
    return crc

# keep the CRCs of the existing types of messages compatible with the
# old "erroneous" way of calculating the CRC. For that - make a pointed
# adjustment of the CRC function.
# This is the purpose of the first element of the per-message dictionary.
# The second element is there to avoid weakening the duplicate-detecting
# properties of crc32. This way, if the new way of calculating the CRC
# happens to collide with the old (buggy) way - we will still get
# a different result and fail the comparison.

fixup_crc_dict = {
        "abf_policy_add_del": { 0xc6131197: 0xee66f93e },
        "abf_policy_details": { 0xb7487fa4: 0x6769e504 },
        "acl_add_replace": { 0xee5c2f18: 0x1cabdeab },
        "acl_details": { 0x95babae0: 0x7a97f21c },
        "macip_acl_add": { 0xce6fbad0: 0xd648fd0a },
        "macip_acl_add_replace": { 0x2a461dd4: 0xe34402a7 },
        "macip_acl_details": { 0x27135b59: 0x57c7482f },
        "dhcp_proxy_config": { 0x4058a689: 0x6767230e },
        "dhcp_client_config": { 0x1af013ea: 0x959b80a3 },
        "dhcp_compl_event": { 0x554a44e5: 0xe908fd1d },
        "dhcp_client_details": { 0x3c5cd28a: 0xacd82f5a },
        "dhcp_proxy_details": { 0xdcbaf540: 0xce16f044 },
        "dhcp6_send_client_message": { 0xf8222476: 0xf6f14ef0 },
        "dhcp6_pd_send_client_message": { 0x3739fd8d: 0x64badb8 },
        "dhcp6_reply_event": { 0x85b7b17e: 0x9f3af9e5 },
        "dhcp6_pd_reply_event": { 0x5e878029: 0xcb3e462b },
        "ip6_add_del_address_using_prefix": { 0x3982f30a: 0x9b3d11e0 },
        "gbp_bridge_domain_add": { 0x918e8c01: 0x8454bfdf },
        "gbp_bridge_domain_details": { 0x51d51be9: 0x2acd15f9 },
        "gbp_route_domain_add": { 0x204c79e1: 0x2d0afe38 },
        "gbp_route_domain_details": { 0xa78bfbca: 0x8ab11375 },
        "gbp_endpoint_add": { 0x7b3af7de: 0x9ce16d5a },
        "gbp_endpoint_details": { 0x8dd8fbd3: 0x8aecb60 },
        "gbp_endpoint_group_add": { 0x301ddf15: 0x8e0f4054 },
        "gbp_endpoint_group_details": { 0xab71d723: 0x8f38292c },
        "gbp_subnet_add_del": { 0xa8803c80: 0x888aca35 },
        "gbp_subnet_details": { 0xcbc5ca18: 0x4ed84156 },
        "gbp_contract_add_del": { 0xaa8d652d: 0x553e275b },
        "gbp_contract_details": { 0x65dec325: 0x2a18db6e },
        "gbp_ext_itf_add_del": { 0x7606d0e1: 0x12ed5700 },
        "gbp_ext_itf_details": { 0x519c3d3c: 0x408a45c0 },
        "gtpu_add_del_tunnel": { 0xca983a2b: 0x9a9c0426 },
        "gtpu_tunnel_update_tteid": { 0x79f33816: 0x8a2db108 },
        "gtpu_tunnel_details": { 0x27f434ae: 0x4535cf95 },
        "igmp_listen": { 0x19a49f1e: 0x3f93a51a },
        "igmp_details": { 0x38f09929: 0x52f12a89 },
        "igmp_event": { 0x85fe93ec: 0xd7696eaf },
        "igmp_group_prefix_set": { 0x5b14a5ce: 0xd4f20ac5 },
        "igmp_group_prefix_details": { 0x259ccd81: 0xc3b3c526 },
        "ikev2_set_responder": { 0xb9aa4d4e: 0xf0d3dc80 },
        "vxlan_gpe_ioam_export_enable_disable": { 0xd4c76d3a: 0xe4d4ebfa },
        "ioam_export_ip6_enable_disable": { 0xd4c76d3a: 0xe4d4ebfa },
        "vxlan_gpe_ioam_vni_enable": { 0xfbb5fb1: 0x997161fb },
        "vxlan_gpe_ioam_vni_disable": { 0xfbb5fb1: 0x997161fb },
        "vxlan_gpe_ioam_transit_enable": { 0x3d3ec657: 0x553f5b7b },
        "vxlan_gpe_ioam_transit_disable": { 0x3d3ec657: 0x553f5b7b },
        "udp_ping_add_del": { 0xfa2628fc: 0xc692b188 },
        "l3xc_update": { 0xe96aabdf: 0x787b1d3 },
        "l3xc_details": { 0xbc5bf852: 0xd4f69627 },
        "sw_interface_lacp_details": { 0xd9a83d2f: 0x745ae0ba },
        "lb_conf": { 0x56cd3261: 0x22ddb739 },
        "lb_add_del_vip": { 0x6fa569c7: 0xd15b7ddc },
        "lb_add_del_as": { 0x35d72500: 0x78628987 },
        "lb_vip_dump": { 0x56110cb7: 0xc7bcb124 },
        "lb_vip_details": { 0x1329ec9b: 0x8f39bed },
        "lb_as_details": { 0x8d24c29e: 0x9c39f60e },
        "mactime_add_del_range": { 0xcb56e877: 0x101858ef },
        "mactime_details": { 0xda25b13a: 0x44921c06 },
        "map_add_domain": { 0x249f195c: 0x7a5a18c9 },
        "map_domain_details": { 0x796edb50: 0xfc1859dd },
        "map_param_add_del_pre_resolve": { 0xdae5af03: 0x17008c66 },
        "map_param_get_reply": { 0x26272c90: 0x28092156 },
        "memif_details": { 0xda34feb9: 0xd0382c4c },
        "dslite_add_del_pool_addr_range": { 0xde2a5b02: 0xc448457a },
        "dslite_set_aftr_addr": { 0x78b50fdf: 0x1e955f8d },
        "dslite_get_aftr_addr_reply": { 0x8e23608e: 0x38e30db1 },
        "dslite_set_b4_addr": { 0x78b50fdf: 0x1e955f8d },
        "dslite_get_b4_addr_reply": { 0x8e23608e: 0x38e30db1 },
        "nat44_add_del_address_range": { 0x6f2b8055: 0xd4c7568c },
        "nat44_address_details": { 0xd1beac1: 0x45410ac4 },
        "nat44_add_del_static_mapping": { 0x5ae5f03e: 0xe165e83b },
        "nat44_static_mapping_details": { 0x6cb40b2: 0x1a433ef7 },
        "nat44_add_del_identity_mapping": { 0x2faaa22: 0x8e12743f },
        "nat44_identity_mapping_details": { 0x2a52a030: 0x36d21351 },
        "nat44_add_del_interface_addr": { 0x4aed50c0: 0xfc835325 },
        "nat44_interface_addr_details": { 0xe4aca9ca: 0x3e687514 },
        "nat44_user_session_details": { 0x2cf6e16d: 0x1965fd69 },
        "nat44_add_del_lb_static_mapping": { 0x4f68ee9d: 0x53b24611 },
        "nat44_lb_static_mapping_add_del_local": { 0x7ca47547: 0x2910a151 },
        "nat44_lb_static_mapping_details": { 0xed5ce876: 0x2267b9e8 },
        "nat44_del_session": { 0x15a5bf8c: 0x4c49c387 },
        "nat_det_add_del_map": { 0x1150a190: 0x112fde05 },
        "nat_det_map_details": { 0xad91dc83: 0x88000ee1 },
        "nat_det_close_session_out": { 0xf6b259d1: 0xc1b6cbfb },
        "nat_det_close_session_in": { 0x3c68e073: 0xa10ef64 },
        "nat64_add_del_pool_addr_range": { 0xa3b944e3: 0x21234ef3 },
        "nat64_add_del_static_bib": { 0x1c404de5: 0x90fae58a },
        "nat64_bib_details": { 0x43bc3ddf: 0x62c8541d },
        "nat64_st_details": { 0xdd3361ed: 0xc770d620 },
        "nat66_add_del_static_mapping": { 0x3ed88f71: 0xfb64e50b },
        "nat66_static_mapping_details": { 0xdf39654b: 0x5c568448 },
        "nsh_add_del_map": { 0xa0f42b0: 0x898d857d },
        "nsh_map_details": { 0x2fefcf49: 0xb34ac8a1 },
        "nsim_cross_connect_enable_disable": { 0x9c3ead86: 0x16f70bdf },
        "pppoe_add_del_session": { 0xf6fd759e: 0x46ace853 },
        "pppoe_session_details": { 0x4b8e8a4a: 0x332bc742 },
        "stn_add_del_rule": { 0x224c6edd: 0x53f751e6 },
        "stn_rules_details": { 0xa51935a6: 0xb0f6606c },
        "svs_route_add_del": { 0xe49bc63c: 0xd39e31fc },
        "svs_details": { 0x6282cd55: 0xb8523d64 },
        "vmxnet3_details": { 0x6a1a5498: 0x829ba055 },
        "vrrp_vr_add_del": { 0xc5cf15aa: 0x6dc4b881 },
        "vrrp_vr_details": { 0x46edcebd: 0x412fa71 },
        "vrrp_vr_set_peers": { 0x20bec71f: 0xbaa2e52b },
        "vrrp_vr_peer_details": { 0x3d99c108: 0xabd9145e },
        "vrrp_vr_track_if_add_del": { 0xd67df299: 0x337f4ba4 },
        "vrrp_vr_track_if_details": { 0x73c36f81: 0x99bcca9c },
        "proxy_arp_add_del": { 0x1823c3e7: 0x85486cbd },
        "proxy_arp_details": { 0x5b948673: 0x9228c150 },
        "bfd_udp_get_echo_source_reply": { 0xe3d736a1: 0x1e00cfce },
        "bfd_udp_add": { 0x939cd26a: 0x7a6d1185 },
        "bfd_udp_mod": { 0x913df085: 0x783a3ff6 },
        "bfd_udp_del": { 0xdcb13a89: 0x8096514d },
        "bfd_udp_session_details": { 0x9fb2f2d: 0x60653c02 },
        "bfd_udp_session_set_flags": { 0x4b4bdfd: 0xcf313851 },
        "bfd_udp_auth_activate": { 0x21fd1bdb: 0x493ee0ec },
        "bfd_udp_auth_deactivate": { 0x9a05e2e0: 0x99978c32 },
        "bier_route_add_del": { 0xfd02f3ea: 0xf29edca0 },
        "bier_route_details": { 0x4008caee: 0x39ee6a56 },
        "bier_disp_entry_add_del": { 0x9eb80cb4: 0x648323eb },
        "bier_disp_entry_details": { 0x84c218f1: 0xe5b039a9 },
        "bond_create": { 0xf1dbd4ff: 0x48883c7e },
        "bond_enslave": { 0xe7d14948: 0x76ecfa7 },
        "sw_interface_bond_details": { 0xbb7c929b: 0xf5ef2106 },
        "pipe_create_reply": { 0xb7ce310c: 0xd4c2c2b3 },
        "pipe_details": { 0xc52b799d: 0x43ac107a },
        "tap_create_v2": { 0x2d0d6570: 0x445835fd },
        "sw_interface_tap_v2_details": { 0x1e2b2a47: 0xe53c16de },
        "sw_interface_vhost_user_details": { 0xcee1e53: 0x98530df1 },
        "virtio_pci_create": { 0x1944f8db: 0xa9f1370c },
        "sw_interface_virtio_pci_details": { 0x6ca9c167: 0x16187f3a },
        "p2p_ethernet_add": { 0x36a1a6dc: 0xeeb8e717 },
        "p2p_ethernet_del": { 0x62f81c8c: 0xb62c386 },
        "geneve_add_del_tunnel": { 0x99445831: 0x976693b5 },
        "geneve_tunnel_details": { 0x6b16eb24: 0xe27e2748 },
        "gre_tunnel_add_del": { 0xa27d7f17: 0x6efc9c22 },
        "gre_tunnel_details": { 0x24435433: 0x3bfbf1 },
        "sw_interface_set_flags": { 0xf5aec1b8: 0x6a2b491a },
        "sw_interface_event": { 0x2d3d95a7: 0xf709f78d },
        "sw_interface_details": { 0x6c221fc7: 0x17b69fa2 },
        "sw_interface_add_del_address": { 0x5463d73b: 0x5803d5c4 },
        "sw_interface_set_unnumbered": { 0x154a6439: 0x938ef33b },
        "sw_interface_set_mac_address": { 0xc536e7eb: 0x6aca746a },
        "sw_interface_set_rx_mode": { 0xb04d1cfe: 0x780f5cee },
        "sw_interface_rx_placement_details": { 0x9e44a7ce: 0xf6d7d024 },
        "create_subif": { 0x790ca755: 0xcb371063 },
        "ip_neighbor_add_del": { 0x607c257: 0x105518b6 },
        "ip_neighbor_dump": { 0xd817a484: 0xcd831298 },
        "ip_neighbor_details": { 0xe29d79f0: 0x870e80b9 },
        "want_ip_neighbor_events": { 0x73e70a86: 0x1a312870 },
        "ip_neighbor_event": { 0xbdb092b2: 0x83933131 },
        "ip_route_add_del": { 0xb8ecfe0d: 0xc1ff832d },
        "ip_route_details": { 0xbda8f315: 0xd1ffaae1 },
        "ip_route_lookup": { 0x710d6471: 0xe2986185 },
        "ip_route_lookup_reply": { 0x5d8febcb: 0xae99de8e },
        "ip_mroute_add_del": { 0x85d762f3: 0xf6627d17 },
        "ip_mroute_details": { 0x99341a45: 0xc1cb4b44 },
        "ip_address_details": { 0xee29b797: 0xb1199745 },
        "ip_unnumbered_details": { 0xcc59bd42: 0xaa12a483 },
        "mfib_signal_details": { 0x6f4a4cfb: 0x64398a9a },
        "ip_punt_redirect": { 0x6580f635: 0xa9a5592c },
        "ip_punt_redirect_details": { 0x2cef63e7: 0x3924f5d3 },
        "ip_container_proxy_add_del": { 0x7df1dff1: 0x91189f40 },
        "ip_container_proxy_details": { 0xa8085523: 0xee460e8 },
        "ip_source_and_port_range_check_add_del": { 0x92a067e3: 0x8bfc76f2 },
        "sw_interface_ip6_set_link_local_address": { 0x1c10f15f: 0x2931d9fa },
        "ip_reassembly_enable_disable": { 0xeb77968d: 0x885c85a6 },
        "set_punt": { 0xaa83d523: 0x83799618 },
        "punt_socket_register": { 0x95268cbf: 0xc8cd10fa },
        "punt_socket_details": { 0xde575080: 0x1de0ce75 },
        "punt_socket_deregister": { 0x98fc9102: 0x98a444f4 },
        "sw_interface_ip6nd_ra_prefix": { 0x82cc1b28: 0xe098785f },
        "ip6nd_proxy_add_del": { 0xc2e4a686: 0x3fdf6659 },
        "ip6nd_proxy_details": { 0x30b9ff4a: 0xd35be8ff },
        "ip6_ra_event": { 0x364c1c5: 0x47e8cfbe },
        "set_ipfix_exporter": { 0x5530c8a0: 0x69284e07 },
        "ipfix_exporter_details": { 0xdedbfe4: 0x11e07413 },
        "ipip_add_tunnel": { 0x2ac399f5: 0xa9decfcd },
        "ipip_6rd_add_tunnel": { 0xb9ec1863: 0x56e93cc0 },
        "ipip_tunnel_details": { 0xd31cb34e: 0x53236d75 },
        "ipsec_spd_entry_add_del": { 0x338b7411: 0x9f384b8d },
        "ipsec_spd_details": { 0x5813d7a2: 0xf2222790 },
        "ipsec_sad_entry_add_del": { 0xab64b5c6: 0xb8def364 },
        "ipsec_tunnel_protect_update": { 0x30d5f133: 0x143f155d },
        "ipsec_tunnel_protect_del": { 0xcd239930: 0xddd2ba36 },
        "ipsec_tunnel_protect_details": { 0x21663a50: 0xac6c823b },
        "ipsec_tunnel_if_add_del": { 0x20e353fa: 0x2b135e68 },
        "ipsec_sa_details": { 0x345d14a7: 0xb30c7f41 },
        "l2_xconnect_details": { 0x472b6b67: 0xc8aa6b37 },
        "l2_fib_table_details": { 0xa44ef6b8: 0xe8d2fc72 },
        "l2fib_add_del": { 0xeddda487: 0xf29d796c },
        "l2_macs_event": { 0x44b8fd64: 0x2eadfc8b },
        "bridge_domain_details": { 0xfa506fd: 0x979f549d },
        "l2_interface_pbb_tag_rewrite": { 0x38e802a8: 0x612efa5a },
        "l2_patch_add_del": { 0xa1f6a6f3: 0x522f3445 },
        "sw_interface_set_l2_xconnect": { 0x4fa28a85: 0x1aaa2dbb },
        "sw_interface_set_l2_bridge": { 0xd0678b13: 0x2e483cd0 },
        "bd_ip_mac_add_del": { 0x257c869: 0x5f2b84e2 },
        "bd_ip_mac_details": { 0x545af86a: 0xa52f8044 },
        "l2_arp_term_event": { 0x6963e07a: 0x85ff71ea },
        "l2tpv3_create_tunnel": { 0x15bed0c2: 0x596892cb },
        "sw_if_l2tpv3_tunnel_details": { 0x50b88993: 0x1dab5c7e },
        "lisp_add_del_local_eid": { 0x4e5a83a2: 0x21f573bd },
        "lisp_add_del_map_server": { 0xce19e32d: 0x6598ea7c },
        "lisp_add_del_map_resolver": { 0xce19e32d: 0x6598ea7c },
        "lisp_use_petr": { 0xd87dbad9: 0x9e141831 },
        "show_lisp_use_petr_reply": { 0x22b9a4b0: 0xdcad8a81 },
        "lisp_add_del_remote_mapping": { 0x6d5c789e: 0xfae8ed77 },
        "lisp_add_del_adjacency": { 0x2ce0e6f6: 0xcf5edb61 },
        "lisp_locator_details": { 0x2c620ffe: 0xc0c4c2a7 },
        "lisp_eid_table_details": { 0x1c29f792: 0x4bc32e3a },
        "lisp_eid_table_dump": { 0x629468b5: 0xb959b73b },
        "lisp_adjacencies_get_reply": { 0x807257bf: 0x3f97bcdd },
        "lisp_map_resolver_details": { 0x3e78fc57: 0x82a09deb },
        "lisp_map_server_details": { 0x3e78fc57: 0x82a09deb },
        "one_add_del_local_eid": { 0x4e5a83a2: 0x21f573bd },
        "one_add_del_map_server": { 0xce19e32d: 0x6598ea7c },
        "one_add_del_map_resolver": { 0xce19e32d: 0x6598ea7c },
        "one_use_petr": { 0xd87dbad9: 0x9e141831 },
        "show_one_use_petr_reply": { 0x84a03528: 0x10e744a6 },
        "one_add_del_remote_mapping": { 0x6d5c789e: 0xfae8ed77 },
        "one_add_del_l2_arp_entry": { 0x1aa5e8b3: 0x33209078 },
        "one_l2_arp_entries_get_reply": { 0xb0dd200f: 0xb0a47bbe },
        "one_add_del_ndp_entry": { 0xf8a287c: 0xd1629a2f },
        "one_ndp_entries_get_reply": { 0x70719b1a: 0xbd34161 },
        "one_add_del_adjacency": { 0x9e830312: 0xe48e7afe },
        "one_locator_details": { 0x2c620ffe: 0xc0c4c2a7 },
        "one_eid_table_details": { 0x1c29f792: 0x4bc32e3a },
        "one_eid_table_dump": { 0xbd190269: 0x95151038 },
        "one_adjacencies_get_reply": { 0x85bab89: 0xa8ed89a5 },
        "one_map_resolver_details": { 0x3e78fc57: 0x82a09deb },
        "one_map_server_details": { 0x3e78fc57: 0x82a09deb },
        "one_stats_details": { 0x2eb74678: 0xff6ef238 },
        "gpe_add_del_fwd_entry": { 0xf0847644: 0xde6df50f },
        "gpe_fwd_entries_get_reply": { 0xc4844876: 0xf9f53f1b },
        "gpe_fwd_entry_path_details": { 0x483df51a: 0xee80b19a },
        "gpe_add_del_native_fwd_rpath": { 0x43fc8b54: 0x812da2f2 },
        "gpe_native_fwd_rpaths_get_reply": { 0x7a1ca5a2: 0x79d54eb9 },
        "sw_interface_set_lldp": { 0x57afbcd4: 0xd646ae0f },
        "mpls_ip_bind_unbind": { 0xc7533b32: 0x48249a27 },
        "mpls_tunnel_add_del": { 0x44350ac1: 0xe57ce61d },
        "mpls_tunnel_details": { 0x57118ae3: 0xf3c0928e },
        "mpls_route_add_del": { 0x8e1d1e07: 0x343cff54 },
        "mpls_route_details": { 0x9b5043dc: 0xd0ac384c },
        "policer_add_del": { 0x2b31dd38: 0xcb948f6e },
        "policer_details": { 0x72d0e248: 0xa43f781a },
        "qos_store_enable_disable": { 0xf3abcc8b: 0x3507235e },
        "qos_store_details": { 0x3ee0aad7: 0x38a6d48 },
        "qos_record_enable_disable": { 0x2f1a4a38: 0x25b33f88 },
        "qos_record_details": { 0xa425d4d3: 0x4956ccdd },
        "session_rule_add_del": { 0xe4895422: 0xe31f9443 },
        "session_rules_details": { 0x28d71830: 0x304b91f0 },
        "sw_interface_span_enable_disable": { 0x23ddd96b: 0xacc8fea1 },
        "sw_interface_span_details": { 0x8a20e79f: 0x55643fc },
        "sr_mpls_steering_add_del": { 0x64acff63: 0x7d1b0a0b },
        "sr_mpls_policy_assign_endpoint_color": { 0xe7eb978: 0x5e1c5c13 },
        "sr_localsid_add_del": { 0x5a36c324: 0x26fa3309 },
        "sr_policy_add": { 0x44ac92e8: 0xec79ee6a },
        "sr_policy_mod": { 0xb97bb56e: 0xe531a102 },
        "sr_steering_add_del": { 0xe46b0a0f: 0x3711dace },
        "sr_localsids_details": { 0x2e9221b9: 0x6a6c0265 },
        "sr_policies_details": { 0xdb6ff2a1: 0x7ec2d93 },
        "sr_steering_pol_details": { 0xd41258c9: 0x1c1ee786 },
        "syslog_set_sender": { 0xb8011d0b: 0xbb641285 },
        "syslog_get_sender_reply": { 0x424cfa4e: 0xd3da60ac },
        "tcp_configure_src_addresses": { 0x67eede0d: 0x4b02b946 },
        "teib_entry_add_del": { 0x8016cfd2: 0x5aa0a538 },
        "teib_details": { 0x981ee1a1: 0xe3b6a503 },
        "udp_encap_add": { 0xf74a60b1: 0x61d5fc48 },
        "udp_encap_details": { 0x8cfb9c76: 0x87c82821 },
        "vxlan_gbp_tunnel_add_del": { 0x6c743427: 0x8c819166 },
        "vxlan_gbp_tunnel_details": { 0x66e94a89: 0x1da24016 },
        "vxlan_gpe_add_del_tunnel": { 0xa645b2b0: 0x7c6da6ae },
        "vxlan_gpe_tunnel_details": { 0x968fc8b: 0x57712346 },
        "vxlan_add_del_tunnel": { 0xc09dc80: 0xa35dc8f5 },
        "vxlan_tunnel_details": { 0xc3916cb1: 0xe782f70f },
        "vxlan_offload_rx": { 0x9cc95087: 0x89a1564b },
        "log_details": { 0x3d61cc0: 0x255827a1 },
}

def foldup_crcs(s):
    for f in s:
        f.crc = foldup_blocks(f.block,
                              binascii.crc32(f.crc) & 0xffffffff)

        # fixup the CRCs to make the fix seamless
        if f.name in fixup_crc_dict:
            if f.crc in fixup_crc_dict.get(f.name):
                f.crc = fixup_crc_dict.get(f.name).get(f.crc)

#
# Main
#
def main():
    if sys.version_info < (3, 5,):
        log.exception('vppapigen requires a supported version of python. '
                      'Please use version 3.5 or greater. '
                      'Using {}'.format(sys.version))
        return 1

    cliparser = argparse.ArgumentParser(description='VPP API generator')
    cliparser.add_argument('--pluginpath', default=""),
    cliparser.add_argument('--includedir', action='append'),
    cliparser.add_argument('--outputdir', action='store'),
    cliparser.add_argument('--input')
    cliparser.add_argument('--output', nargs='?',
                           type=argparse.FileType('w', encoding='UTF-8'),
                           default=sys.stdout)

    cliparser.add_argument('output_module', nargs='?', default='C')
    cliparser.add_argument('--debug', action='store_true')
    cliparser.add_argument('--show-name', nargs=1)
    cliparser.add_argument('--git-revision',
                           help="Git revision to use for opening files")
    args = cliparser.parse_args()

    dirlist_add(args.includedir)
    if not args.debug:
        sys.excepthook = exception_handler

    # Filename
    if args.show_name:
        filename = args.show_name[0]
    elif args.input:
        filename = args.input
    else:
        filename = ''

    if args.debug:
        logging.basicConfig(stream=sys.stdout, level=logging.WARNING)
    else:
        logging.basicConfig()

    parser = VPPAPI(debug=args.debug, filename=filename, logger=log,
                    revision=args.git_revision)

    try:
        if not args.input:
            parsed_objects = parser.parse_fd(sys.stdin, log)
        else:
            parsed_objects = parser.parse_filename(args.input, log)
    except ParseError as e:
        print('Parse error: ', e, file=sys.stderr)
        sys.exit(1)

    # Build a list of objects. Hash of lists.
    result = []

    if args.output_module == 'C':
        s = parser.process(parsed_objects)
    else:
        result = parser.process_imports(parsed_objects, False, result)
        s = parser.process(result)

    # Add msg_id field
    s['Define'] = add_msg_id(s['Define'])

    # Fold up CRCs
    foldup_crcs(s['Define'])

    #
    # Debug
    if args.debug:
        import pprint
        pp = pprint.PrettyPrinter(indent=4, stream=sys.stderr)
        for t in s['Define']:
            pp.pprint([t.name, t.flags, t.block])
        for t in s['types']:
            pp.pprint([t.name, t.block])

    #
    # Generate representation
    #
    from importlib.machinery import SourceFileLoader

    # Default path
    pluginpath = ''
    if not args.pluginpath:
        cand = []
        cand.append(os.path.dirname(os.path.realpath(__file__)))
        cand.append(os.path.dirname(os.path.realpath(__file__)) +
                    '/../share/vpp/')
        for c in cand:
            c += '/'
            if os.path.isfile('{}vppapigen_{}.py'
                              .format(c, args.output_module.lower())):
                pluginpath = c
                break
    else:
        pluginpath = args.pluginpath + '/'
    if pluginpath == '':
        log.exception('Output plugin not found')
        return 1
    module_path = '{}vppapigen_{}.py'.format(pluginpath,
                                             args.output_module.lower())

    try:
        plugin = SourceFileLoader(args.output_module,
                                  module_path).load_module()
    except Exception as err:
        log.exception('Error importing output plugin: {}, {}'
                      .format(module_path, err))
        return 1

    result = plugin.run(args, filename, s)
    if result:
        print(result, file=args.output)
    else:
        log.exception('Running plugin failed: {} {}'
                      .format(filename, result))
        return 1
    return 0

if __name__ == '__main__':
    sys.exit(main())


import io
import numpy as np

def load_vec(emb_path, nmax=50000):
    vectors = []
    word2id = {}
    with io.open(emb_path, 'r', encoding='utf-8', newline='\n', errors='ignore') as f:
        next(f)
        for i, line in enumerate(f):
            word, vect = line.rstrip().split(' ', 1)
            vect = np.fromstring(vect, sep=' ')
            assert word not in word2id, 'word found twice'
            vectors.append(vect)
            word2id[word] = len(word2id)
            if len(word2id) == nmax:
                break
    id2word = {v: k for k, v in word2id.items()}
    embeddings = np.vstack(vectors)

    return embeddings, id2word, word2id


from zope.component import getUtility
from zope.site.hooks import getSite
from zope.globalrequest import getRequest
from collective.beaker.interfaces import ISession
from plone.registry.interfaces import IRegistry

from chromsystems.shop.interfaces import ICSShopSettings
from chromsystems.shop.interfaces import IShoppingCartUtility

def format_price(price):
    if price < 0:
        return '-%0.2f EUR' % abs(price)
    else:
        return '%0.2f EUR' % price

def get_cart():
    return getUtility(IShoppingCartUtility).get(getSite())

def wipe_cart():
    return getUtility(IShoppingCartUtility).destroy(getSite())

def update_cart():
    return getUtility(IShoppingCartUtility).update(getSite())

def redirect_to_shop():
    site = getSite()
    request = getRequest()
    registry = getUtility(IRegistry)
    settings = registry.forInterface(ICSShopSettings)
    shop_url = settings.shop_url
    portal_url = site.absolute_url()
    url = portal_url + shop_url
    request.response.redirect(url)

def redirect_to_cart():
    site = getSite()
    request = getRequest()
    registry = getUtility(IRegistry)
    settings = registry.forInterface(ICSShopSettings)
    shop_url = settings.shop_url
    portal_url = site.absolute_url()
    url = portal_url + shop_url + '/@@cart'
    request.response.redirect(url)

def redirect_to_checkout():
    site = getSite()
    request = getRequest()
    session = ISession(request)
    session.save()
    registry = getUtility(IRegistry)
    settings = registry.forInterface(ICSShopSettings)
    shop_url = settings.shop_url
    portal_url = site.absolute_url()
    url = portal_url + shop_url + '/@@checkout'
    request.response.redirect(url)


#!/usr/bin/python
# -*- coding: utf-8 -*-
import sys
import os
from logo_detect import *
import pprint
from selenium import webdriver
from scrapy_mop import scrapy_mop
try:
    import tkinter as tk
    from tkinter import messagebox
except:
    import Tkinter as tk
    import tkMessageBox as messagebox

sys.path.append(os.path.join(os.path.dirname(__file__), '../'))

import pygubu

class Myapp:
    def __init__(self, master):

        self.check_1 = 0
        self.check_2 = 0
        self.check_3 = 0
        self.check_4 = 0
        self.master = master
        self.builder = builder = pygubu.Builder()
        fpath = os.path.join(os.path.dirname(__file__),"main_ui.ui")
        builder.add_from_file(fpath)
        mainwindow = builder.get_object('mainwindow', master)
        builder.connect_callbacks(self)

    def local_detect_button_click(self):
        src_image_input_txt = self.builder.get_object('src_image_input_txt')
        dst_image_input_txt = self.builder.get_object('dst_image_input_txt')
        src_path = src_image_input_txt.get()
        dst_path = dst_image_input_txt.get()
        if not os.path.isdir(src_path):
            messagebox.showinfo("title","请输入正确的源路径")
        elif not os.path.isdir(dst_path):
            messagebox.showinfo("title","请输入正确的目标路径")
        elif self.check_1==0 and self.check_2==0:
            messagebox.showinfo("title","请选择一个模型")
        elif self.check_1==1 and self.check_2==1:
            messagebox.showinfo("title","只能选择一个模型")
        else:
            if self.check_1==1:
                #initialize the object
                logo_detect_obj = detector('logo_detect.prototxt','logo_iter_98000.caffemodel','mean.npy',(128,128),0.0078125,1)
                logo_detect_obj.local_detect(src_path,dst_path)
                messagebox.showinfo("title","检测完毕")
            elif self.check_2==1:
                logo_detect_obj = detector('logo_detect.prototxt','logo_iter_198000.caffemodel','mean.npy',(128,128),0.0078125,1)
                logo_detect_obj.local_detect(src_path,dst_path)
                messagebox.showinfo("title","检测完毕")
    def Checkbutton_1_click(self):
        if self.check_1==0:
            self.check_1=1
        else:
            self.check_1=0

    def Checkbutton_2_click(self):
        if self.check_2==0:
            self.check_2=1
        else:
            self.check_2=0
    def Checkbutton_3_click(self):
        if self.check_3==0:
            self.check_3=1
        else:
            self.check_3=0
    def Checkbutton_4_click(self):
        if self.check_4==0:
            self.check_4=1
        else:
            self.check_4=0
    def open_managment_click(self):
        browser = webdriver.Chrome()
        browser.get("http://localhost/admin/sensitive.php")

    def net_detect_button_click(self):

        address_input_txt= self.builder.get_object('address_input_txt')
        address_input = address_input_txt.get()
        if self.check_3==1 and self.check_4==1:
            messagebox.showinfo("title","只能选择一个模型")
        if self.check_3==0 and self.check_4==0:
            messagebox.showinfo("title","必须要选择一个模型")
        if address_input=='mop':
            scarpy_model = scrapy_mop()
            if self.check_3==1:
                logo_detect_obj = detector('logo_detect_full_conv.prototxt','logo_iter_95000_full.caffemodel',111,(333,333),1,1)
                scarpy_model.run(logo_detect_obj)
            else:
                logo_detect_obj = detector('logo_detect_full_conv.prototxt','logo_iter_95000_full.caffemodel',111,(333,333),1,1)
                scarpy_model.run(logo_detect_obj)
            messagebox.showinfo("title","爬取，检测完毕！")

if __name__ == '__main__':
    root = tk.Tk()
    app = Myapp(root)
    root.mainloop()


"""
Create or import top-level classes/methods.
isort:skip_file
"""
from typing import Tuple, Union

from sklearn.base import RegressorMixin as SkLearnModel
from statsmodels.base.wrapper import ResultsWrapper as StatsModel

# Here we list any class we might use. In principle, they should be expected to have a fit(X, y) and a predict(X)
# method.
# However, the statsmodels library provides Model and Results classes. The former only hold X and y data, while the
# latter represents a fitted (parameterised) Model (with a working predict method), plus the result statistics
# (the Model object is also in there).
MODEL_CLASSES = (StatsModel, SkLearnModel)
MODEL_TYPES = Union[MODEL_CLASSES]

# First public import block
from timetomodel.speccing import (
    DBSeriesSpecs,
    DFFileSeriesSpecs,
    ModelSpecs,
    ObjectSeriesSpecs,
)

class ModelState(object):
    """
    This class abstracts all information we need to describe a model and how it was made.
    It is simply a container to hold all model-relevant state information:
    The trained model and the ModelSpecs it is based on."""

    model: MODEL_TYPES  # a fitted model
    specs: ModelSpecs

    def __init__(self, model, specs):
        if not isinstance(model, MODEL_CLASSES):
            raise Exception(
                "ModelState(): model parameter needs to be of type %s instead of type %s"
                % (str(MODEL_CLASSES), type(model))
            )
        self.model = model
        if not isinstance(specs, ModelSpecs):
            raise Exception(
                "ModelState(): specs parameter needs to be of type <ModelSpecs>"
            )
        self.specs = specs

    def split(self) -> Tuple[MODEL_TYPES, ModelSpecs]:
        """Return model and specs separately"""
        return self.model, self.specs

    def __repr__(self):
        return "ModelState: <%s, %s>" % (self.model, self.specs)

# Second public import block
from timetomodel.forecasting import make_forecast_for, make_rolling_forecasts
from timetomodel.modelling import (
    create_fitted_model,
    evaluate_models,
    model_param_grid_search,
)


#! /usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import division, print_function, absolute_import

import os
from timeit import time
import warnings
import sys
import cv2
import numpy as np
from PIL import Image
from src.yolo import YOLO
import colorsys
from deep_sort import preprocessing
from deep_sort import nn_matching
from deep_sort.detection import Detection
from deep_sort.tracker import Tracker
from tools import generate_detections as gdet
from deep_sort.detection import Detection as ddet
import argparse
from src.demo_image import POSE
warnings.filterwarnings('ignore')

def arg_parse():

    parser = argparse.ArgumentParser(description='openpose Video Detection Module')

    parser.add_argument("--video", dest = 'video', help = "Video to run detection upon",
                default = "/home/ftp/DataSet/Hand-Dataset/video/tennis.mp4", type = str)
    parser.add_argument('--model', dest = 'model', type=str, default='model/keras/model.h5', help='path to the weights file')
    return parser.parse_args()

def deep_sort():
    # Definition of the parameters
    max_cosine_distance = 0.3
    nn_budget = None
    nms_max_overlap = 1.0
    # deep_sort
    model_filename = 'model/mars-small128.pb'
    encoder = gdet.create_box_encoder(model_filename,batch_size=1)
    metric = nn_matching.NearestNeighborDistanceMetric("cosine", max_cosine_distance, nn_budget)
    tracker = Tracker(metric)
    return tracker, encoder, nms_max_overlap

def main(yolo):

    # Generate colors for drawing bounding boxes.
    hsv_tuples = [(x / 80, 1., 1.)
                for x in range(80)]
    colors = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
    colors = list(
        map(lambda x: (int(x[0] * 255), int(x[1] * 255), int(x[2] * 255)),
            colors))
    tracker, encoder, nms_max_overlap = deep_sort()
    classes = load_classes('model/coco_classes.txt')
    writeVideo_flag = True
    args = arg_parse()
    videofile = args.video
    keras_weights_file = args.model
    # 加载openpose模型
    pose = POSE(keras_weights_file)
    video_capture = cv2.VideoCapture(videofile)
    assert video_capture.isOpened(), 'Cannot capture source'

    if writeVideo_flag:
    # Define the codec and create VideoWriter object
        w = int(video_capture.get(3))
        h = int(video_capture.get(4))
        fourcc = cv2.VideoWriter_fourcc(*'MP4V')
        outname = os.path.join('result_video',videofile.split('/')[-1])
        out = cv2.VideoWriter(outname, fourcc, 25, (w, h))
        list_file = open('detection.txt', 'w')
        frame_index = -1

    fps = 0.0
    #athlete_flag = [0,0]

    while video_capture.isOpened():
        ret, frame = video_capture.read()  # frame shape 640*480*3
        if ret != True:
            break
        t1 = time.time()

        image = Image.fromarray(frame)
        boxs = yolo.detect_image_track(image)
        boxs = np.array(boxs)
        #print("box_num",len(boxs))
        print("box_des",boxs)
        #cls = int(boxs[-1])
        #label = "{0}".format(classes[cls])
        #t_size = cv2.getTextSize(boxs[0][-1], cv2.FONT_HERSHEY_PLAIN, 1 , 1)[0]
        features = encoder(frame,boxs)

        # score to 1.0 here).
        detections = [Detection(bbox[:-1],bbox[-1], 1.0, feature) for bbox, feature in zip(boxs, features)]

        # Run non-maxima suppression.
        boxes = np.array([d.tlwh for d in detections])
        scores = np.array([d.confidence for d in detections])
        indices = preprocessing.non_max_suppression(boxes, nms_max_overlap, scores)
        detections = [detections[i] for i in indices]

        # Call the tracker
        tracker.predict()
        tracker.update(detections)
        flag_boxs = 0
        for track in tracker.tracks:
            if track.is_confirmed() and track.time_since_update >1 :
                continue
            bbox = track.to_tlbr()
            label = "{0}".format(classes[int(track.nameclass)])
            c1 = (int(bbox[0]),int(bbox[1]))
            c2 = (int(bbox[2]),int(bbox[3]))
            if label == 'person':
                if c1[0]>256 and c1[1]>95 and c2[0]<1053 and c2[1]<717:
                    label = 'athlete'
                elif c1[0]>261 and c1[1]>20 and c2[0]<1012 and c2[1]<162:
                    label = 'baseline_referee'
                elif c1[0]>1 and c1[1]>95 and c2[0]<175 and c2[1]<413:
                    label = 'sideline_referee'
                elif c1[0]>1002 and c1[1]>128 and c2[0]<1181 and c2[1]<387:
                    label = 'referee'
                elif c1[0]>1002 and c1[1]>245 and c2[0]<1170 and c2[1]<387:
                    label = 'Caddy'
                elif c1[0]>166 and c1[1]>232 and c2[0]<295 and c2[1]<347:
                    label = 'Caddy'
            color = colors[int(track.nameclass)]
            cv2.rectangle(frame, c1, c2,color, 2)
            t_size = cv2.getTextSize(label, cv2.FONT_HERSHEY_PLAIN, 1 , 2)[0]
            name = label +'_' +str(track.track_id)
            c2 = c1[0] + t_size[0] + 3, c1[1] + t_size[1] + 4
            cv2.rectangle(frame, c1, c2,color, -1)
            cv2.putText(frame, name, (c1[0], c1[1] + t_size[1] + 4), cv2.FONT_HERSHEY_PLAIN, 1,(0,0,0), 1);
            #cv2.putText(frame, str(track.nameclass)+'_'+str(track.track_id),(int(bbox[0]), int(bbox[1])),0, 5e-3 * 200, (0,255,0),2)

            #cv2.putText(frame, str(boxs[flag_boxs][-1])+'_'+str(track.track_id),(int(bbox[0]), int(bbox[1])), cv2.FONT_HERSHEY_PLAIN, 1, [225,255,255], 1)
            flag_boxs += 1
            #canvas = pose.detect_pose(frame[37:704,69:1153,:])
            #frame[37:704,69:1153,:] = canvas

        cv2.imshow('', frame)

        if writeVideo_flag:
            # save a frame
            out.write(frame)
            frame_index = frame_index + 1
            list_file.write(str(frame_index)+' ')
            if len(boxs) != 0:
                for i in range(0,len(boxs)):
                    list_file.write(str(boxs[i][0]) + ' '+str(boxs[i][1]) + ' '+str(boxs[i][2]) + ' '+str(boxs[i][3]) + ' ')
            list_file.write('\n')

        fps  = ( fps + (1./(time.time()-t1)) ) / 2
        print("fps= %f"%(fps))

        # Press Q to stop!
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

    video_capture.release()
    if writeVideo_flag:
        out.release()
        list_file.close()
    cv2.destroyAllWindows()

def load_classes(namesfile):
    fp = open(namesfile, "r")
    names = fp.read().split("\n")[:-1]
    return names

if __name__ == '__main__':
    main(YOLO())


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.shortcuts import render

from novaclient.client import Client
import uuid

# Create your views here.
nova = 0
def process():
	# api: https://docs.openstack.org/developer/python-novaclient/ref/v2/client.html
	d = {}
	d['version'] = '2.1'
	d['username'] = 'testuser_1u'  # os.environ['OS_USERNAME']
	d['password'] = 'PASSW0RD1'  # os.environ['OS_PASSWORD']
	d['auth_url'] = 'https://acopenstack.rit.edu:5000/v3'  # os.environ['OS_AUTH_URL']
	d['project_name'] = 'testdev_1'  # os.environ['OS_TENANT_NAME']
	# NOTE. If the username is an RIT DCE (contains @ad.rit.edu), use `default` as the domain
	d['user_domain_name'] = 'local'
	d['project_domain_name'] = 'local'
	d['insecure'] = True
	global nova
	nova = Client(**d)

process()

def index(request):
	output = "Spawn a VM\n\n"
	global nova
	for server in nova.servers.list():
        	print(server.networks)
        	try:
            		console = server.get_vnc_console('novnc')
            		line = "Console Delete  {1}  {2}  {3}\n".format(console['console']['url'], server.name, server.id, server.networks['Shared'][0])
        	except Exception:
            		line = "{0} is in motion (running task).\n".format(server.id)
        	output += line

	return render(request, 'createvm/home.html', {'content':[output, 'test1']})

def spawn(request):
    global nova
    id = str(uuid.uuid4())
    image = 'd611ada3-b77b-4727-bbdf-b305eb89ed07'
    flavor = nova.flavors.find(name="m1.small")
    nics = [{'net-id': '027e77e6-6599-4b26-8b6c-f5d953453fdc'}]
    instance = nova.servers.create(name="server-" + id, image=image, flavor=flavor, nics=nics)
    output = "Task started.\n\nID: {0}\nName: {1}".format(instance.id, instance.name)

    return render(request, 'createvm/spawn.html', {'media':[output, 'test1']})

def delete(request, id):
    global nova
    server = nova.servers.get(id)
    server.delete()
    return render(request, 'createvm/delete.html', {'op':['deleted:', server.id]})


"""
Work with datasets

Written by Evgeniy Ryzhkov

------------------------------------------------------------

Usage:

    # Split datasets (datasets/filtered dir) into train and val datasets
    python - m scripts.dataset --action=split_train_val_images

"""

import http.client, urllib.parse, json
import config.main as config
import config.access as config_access
import requests
import time
import os
import shutil
import argparse
from imutils import paths
import random

class Dataset:

    def __init__(self):
        self.subscription_key = config_access.AZURE_SUBSCRIPTION_KEY
        self.search_url = 'https://api.cognitive.microsoft.com/bing/v7.0/images/search'

        self.downloaded_images_path = config.DOWNLADED_IMG_DIR
        self.search_term = ['card visa'] # for diversity use different term 'debit card', 'card bank of america' and so on
        self.images_amount_for_class = 300

    def run(self):
        try:
            user_arguments = self._get_command_line_arguments()

            if user_arguments['action'] == 'download_images':
                self._download_images_for_train()
            elif user_arguments['action'] == 'split_train_val_images':
                self._split_train_val_images()
            else:
                print('Incorrect action type. You should choose one of possible values.')
        except:
            print('Argument --action is required. Possible values: "download_images", "split_train_val_images"')

    @staticmethod
    def _get_command_line_arguments():
        ap = argparse.ArgumentParser()
        ap.add_argument("--action", required=True,
                        help="What's action you need? Possible values: 'download_images', 'split_train_val_images' ")
        args = vars(ap.parse_args())
        return args

    def _download_images_for_train(self):
        print('[INFO] Creating dataset...')
        self._clear_directory(self.downloaded_images_path)
        search_results = self._get_json(self.search_term)

        url_images = gd_o.get_first_n_image_urls(search_results, n_images=self.images_amount_for_class)
        self._parse_and_save_images(url_images, self.downloaded_images_path)

        print('[OK] Images for train has been saved successfully into dir {}.'.format(self.downloaded_images_path))

    @staticmethod
    def _clear_directory(path):
        for root, dirs, files in os.walk(path):
            for f in files:
                os.unlink(os.path.join(root, f))
            for d in dirs:
                shutil.rmtree(os.path.join(root, d))

    def _get_json(self, search_term):
        headers = {"Ocp-Apim-Subscription-Key": self.subscription_key}
        offset = 0
        n_grab_images = 0
        value_results = []

        while n_grab_images < self.images_amount_for_class:
            params = {"q": search_term, "imageType": "photo", "offset": offset}
            response = requests.get(self.search_url, headers=headers, params=params)
            response.raise_for_status()
            search_results = response.json()

            value_field = search_results['value']
            n_grab_images += len(value_field)
            offset = search_results['nextOffset']
            value_results += value_field

            # print(json.dumps(search_results, indent=4, sort_keys=True))
            # There is a limit on requests per sec for free azure account
            time.sleep(1)

        return value_results

    @staticmethod
    def _parse_and_save_images(url_images, dir_path):

        file_name_pattern = 'img_'
        file_name_ext = '.jpg'
        start_idx = 122 # for increasing amount of dataset. take last number of image from current dataset

        for idx, img in enumerate(url_images):
            file = requests.get(img, timeout=30)
            file_name = file_name_pattern + str(start_idx + idx) + file_name_ext
            file_path = dir_path + file_name
            print('[INFO] saving ' + file_name)

            # write the image to disk
            f = open(file_path, "wb")
            f.write(file.content)
            f.close()

    @staticmethod
    def get_first_n_image_urls(search_results, n_images):
        image_urls = [img["contentUrl"] for img in search_results[:n_images]] # using big image as on small can't see guitars
        return image_urls

    def _split_train_val_images(self):
        print('[INFO] Splitting filtered images to train and valid sets...')
        list_filtered_image_paths = list(paths.list_images(config.FILTERED_IMG_DIR))
        n_full_dataset = len(list_filtered_image_paths)
        print('[INFO] Full dataset amount = ', n_full_dataset)

        # shuffle images for better model learning and simplier split data by train / valid
        random.Random(4).shuffle(list_filtered_image_paths)
        val_size = 0.2
        n_val_size = round(n_full_dataset * val_size)
        n_train_size = n_full_dataset - n_val_size
        train_images_paths = list_filtered_image_paths[:n_train_size]
        val_images_paths = list_filtered_image_paths[-n_val_size:]

        self._copy_images(train_images_paths, config.TRAIN_IMG_DIR)
        self._copy_images(val_images_paths, config.VAL_IMG_DIR)
        print('[OK] Train ({}) and validation ({}) images were copied.'.format(n_train_size, n_val_size))

    @staticmethod
    def _copy_images(img_list, new_dir_path):
        for img in img_list:
            shutil.copy(img, new_dir_path)

    @staticmethod
    def _create_train_test_txt():
        print('[INFO] Creating train and test.txt...')

        list_labeled_image_paths = list(paths.list_images(config.LABELED_TRAIN_IMG_DIR))
        n_full_dataset = len(list_labeled_image_paths)
        random.Random(4).shuffle(list_labeled_image_paths) # shuffle list for better model learning and simplier split data by train / test

        test_size = 0.2
        n_test_size = round(n_full_dataset * test_size)
        n_train_size = n_full_dataset - n_test_size
        train_images_paths = list_labeled_image_paths[:n_train_size]
        test_images_paths = list_labeled_image_paths[-n_test_size:]

        # creating and saving txt files for darknet training
        # we need full path to images in order to darknet could fild them
        file_train = open(config.TXT_TRAIN_FILE, 'w')
        for img in train_images_paths:
            file_train.write(os.path.abspath(img) + '\n')

        file_test = open(config.TXT_TEST_FILE, 'w')
        for img in test_images_paths:
            file_test.write(os.path.abspath(img) + '\n')

        print('[OK] ' + config.TXT_TRAIN_FILE + ' and ' + config.TXT_TEST_FILE + ' have been created successfully.')

# ----------------------------------------------------
gd_o = Dataset()
gd_o.run()


#!/usr/bin/env python
# coding: utf-8

import requests
from bs4 import BeautifulSoup
import pandas as pd
from splinter import Browser
from bs4 import BeautifulSoup as bs
import time

def init_browser():
    # @NOTE: Replace the path with your actual path to the chromedriver
    executable_path = {"executable_path": "/usr/local/bin/chromedriver"}
    return Browser("chrome", **executable_path, headless=False)

def scrape():

    r = requests.get('https://mars.nasa.gov/news/?page=0&per_page=40&order=publish_date+desc%2Ccreated_at+desc&search=&category=19%2C165%2C184%2C204&blank_scope=Latest')

    soup = BeautifulSoup(r.text, 'html.parser')

    result1 = soup.find_all('div', attrs={'class':'content_title'})

    news_title = result1[0].a.text

    result2 = soup.find_all('div', attrs={'class':'rollover_description_inner'})

    news_p = result2[0].text

    # # Nasa .gov Site

    short_url = "https://www.jpl.nasa.gov/"
    url = "https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars"
    r2 = requests.get(url)

    soup2 = BeautifulSoup(r2.text, 'html.parser')

    result3 = soup2.find_all('article', attrs={'class':'carousel_item'})

    split1 = result3[0]["style"].split("(")

    short_url_2 = split1[1].split("'")[1]

    featured_image_url = short_url + short_url_2

    from IPython.display import Image
    Image(featured_image_url)

    # #  Nasa Twitter Weather
    #

    url_tw = "https://twitter.com/marswxreport?lang=en"
    r3 = requests.get(url_tw)
    soup3 = BeautifulSoup(r3.text, 'html.parser')

    result4 = soup3.find_all('div', attrs={'class':'js-tweet-text-container'})

    mars_weather = result4[0].p.text.split("pic")[0]

    # # Mars Facts

    fact_url = "https://space-facts.com/mars/"
    r4 = requests.get(fact_url)
    soup4 = BeautifulSoup(r4.text, 'html.parser')

    result5 = soup4.find_all(attrs={'class':'column-2'})
    result6 = soup4.find_all(attrs={'class':'column-1'})

    data_list1 = []
    data_list2 = []
    for x in list(range(7,16)):
        data_list1.append(result5[x].text)
        data_list2.append(result6[x].text)

    df1 = pd.DataFrame({"Value":data_list1},index=data_list2)

    df1.to_html('mars_weather.html')

    # # Mars Image

    img_url1 = "https://astrogeology.usgs.gov/cache/images/cfa62af2557222a02478f1fcd781d445_cerberus_enhanced.tif_full.jpg"
    Image(img_url1)

    title1 = "Cerberus Hemisphere"

    img_url2 = "https://astrogeology.usgs.gov/cache/images/3cdd1cbf5e0813bba925c9030d13b62e_schiaparelli_enhanced.tif_full.jpg"
    Image(img_url2)

    title2 = "Schiaparelli Hemisphere"

    img_url3 = "https://astrogeology.usgs.gov/cache/images/ae209b4e408bb6c3e67b6af38168cf28_syrtis_major_enhanced.tif_full.jpg"
    Image(img_url3)

    title3 = "Syrtis Major Hemisphere"

    img_url4 = "https://astrogeology.usgs.gov/cache/images/7cf2da4bf549ed01c17f206327be4db7_valles_marineris_enhanced.tif_full.jpg"
    Image(img_url4)

    title4 = "Valles Marineris Hemisphere"

    # Store data in a dictionary
    mars_data = {
       "news_title":news_title,
        "news_p":news_p,
        "featured_image_url":featured_image_url,
        "value":"Value",
        "mars_weather":mars_weather,
        "df1i1":df1.index[0],
        "df1i2":df1.index[1],
        "df1i3":df1.index[2],
        "df1i4":df1.index[3],
        "df1i5":df1.index[4],
        "df1i6":df1.index[5],
        "df1i7":df1.index[6],
        "df1i8":df1.index[7],
        "df1i9":df1.index[8],
        "df1v1":df1.Value[0],
        "df1v2":df1.Value[1],
        "df1v3":df1.Value[2],
        "df1v4":df1.Value[3],
        "df1v5":df1.Value[4],
        "df1v6":df1.Value[5],
        "df1v7":df1.Value[6],
        "df1v8":df1.Value[7],
        "df1v9":df1.Value[8],
        "df2t1":title1,
        "df2t2":title2,
        "df2t3":title3,
        "df2t4":title4,
        "df2i1":img_url1,
        "df2i2":img_url2,
        "df2i3":img_url3,
        "df2i4":img_url4

    }

 # Return results
    return mars_data


# -*- coding:utf-8 -*-
# @author :adolf
# !python3
import os
import cv2
import math
import torch
import numpy as np
from concern.config import Config, Configurable
import os

os.environ['CUDA_VISIBLE_DEVICES'] = "5"

params = {
    'exp': 'experiments/seg_detector/rctw17_resnet50_deform_thre.yaml',
    'resume': '/home/shizai/data3/adolf/ocr_hup/DB/output_aws/outputs/workspace/DB/SegDetectorModel-seg_detector/deformable_resnet50/L1BalanceCELoss_rctw200_x30/model/final',
    'image_short_side': 512,
    'thresh': 0.1,
    'box_thresh': 0.1,
    'resize': False,
    'polygon': False,
    'verbose': False
}

class SegDetector(object):
    def __init__(self, args=params):
        self.RGB_MEAN = np.array([122.67891434, 116.66876762, 104.00698793])

        conf = Config()
        experiment_args = conf.compile(conf.load(args['exp']))['Experiment']
        experiment_args.update(cmd=args)
        self.experiment = Configurable.construct_class_from_config(experiment_args)
        self.experiment.load('evaluation', **args)
        self.args = args
        self.structure = self.experiment.structure
        self.model_path = args['resume']

        self.init_torch_tensor()
        self.model = self.init_model()
        self.resume(self.model, self.model_path)
        self.model.eval()
        self.is_resize = args['resize']

    def init_torch_tensor(self):
        # Use gpu or not
        torch.set_default_tensor_type('torch.FloatTensor')
        if torch.cuda.is_available():
            self.device = torch.device('cuda')
            torch.set_default_tensor_type('torch.cuda.FloatTensor')
        else:
            self.device = torch.device('cpu')

    def init_model(self):
        model = self.structure.builder.build(self.device)
        return model

    def resume(self, model, path):
        if not os.path.exists(path):
            print("Checkpoint not found: " + path)
            return
        print("Resuming from " + path)
        states = torch.load(
            path, map_location=self.device)
        model.load_state_dict(states, strict=False)
        print("Resumed from " + path)

    def resize_image(self, img, is_resize=True):
        height, width, _ = img.shape
        if is_resize:
            if height < width:
                new_height = self.args['image_short_side']
                new_width = int(math.ceil(new_height / height * width / 32) * 32)
            else:
                new_width = self.args['image_short_side']
                new_height = int(math.ceil(new_width / width * height / 32) * 32)
            resized_img = cv2.resize(img, (new_width, new_height))
        else:
            if height < width:
                scale = int(height / 32)
                new_image_short_side = scale * 32
                new_height = new_image_short_side
                new_width = int(math.ceil(new_height / height * width / 32) * 32)
            else:
                scale = int(width / 32)
                new_image_short_side = scale * 32
                new_width = new_image_short_side
                new_height = int(math.ceil(new_width / width * height / 32) * 32)
            resized_img = cv2.resize(img, (new_width, new_height))
            print(new_height, new_width)
        return resized_img

    def load_image(self, img):
        original_shape = img.shape[:2]
        img = self.resize_image(img, is_resize=self.is_resize)
        img = img.astype(np.float32)
        img -= self.RGB_MEAN
        img /= 255.
        img = torch.from_numpy(img).permute(2, 0, 1).float().unsqueeze(0)
        return img, original_shape

    def predict(self, imgs):
        outputs = dict()
        for i, img in enumerate(imgs):
            batch = dict()
            timg, original_shape = self.load_image(img)
            batch['shape'] = [original_shape]
            batch['image'] = timg

            with torch.no_grad():
                pred = self.model.forward(batch, training=False)
                output = self.structure.representer.represent(batch, pred, is_output_polygon=self.args['polygon'])
                text_lines, scores = output
                text_lines = text_lines[0]
                scores = scores[0]
                new_tls = []
                new_scs = []
                for text_line, score in zip(text_lines, scores):
                    if len(text_line) == 0 or score < 0.2:
                        continue
                    new_tls.append(text_line)
                    new_scs.append(score)

                outputs[i] = []
                for j, (text_line, score) in enumerate(zip(new_tls, new_scs)):
                    one_output = dict()
                    one_output['text_line'] = text_line.tolist()
                    one_output['score'] = score
                    outputs[i].append(one_output)

                    # cut_image = self.cut_image(text_line, img)
                # cv2.imwrite("results_2/{}.jpg".format(j), cut_image)
                # part_imgs.append((cut_image, text_line.reshape(-1)))
                # outputs[i] = part_imgs
        return outputs

if __name__ == "__main__":
    # debug
    img = cv2.imread("images/2.png")
    img = img.astype(np.float32)
    detector = SegDetector()
    outp = detector.predict([img])
    print(outp)


"""
Imports site and captain data into ROOM.
Intended to be run with the remote_api:
bash> export PYTHONPATH=.:$PYTHONPATH
bash> python $(which remote_api_shell.py) -s rebuildingtogethercaptain.appspot.com
s~rebuildingtogethercaptain-hrd> import import_csv

Great instructions at
https://github.com/babybunny/rebuildingtogethercaptain/issues/181

# or for development..
bash> python $(which remote_api_shell.py) -s localhost:8081
dev~rebuildingtogethercaptain> reload(import_csv)
<module 'import_csv' from '/Users/babybunny/appengine/rebuildingtogethercaptain/import_csv.py'>
dev~rebuildingtogethercaptain> import_csv.import_sites(input_csv="../2012_ROOMS_site_info_sample.csv")
dev~rebuildingtogethercaptain> import_csv.import_captains(input_csv="../2012_ROOMS_Captain_email_sample.csv")
"""

import csv
import logging
import main  # to initialize Django
from room import models
from google.appengine.ext import db

##############
# update me! #
##############
PROGRAM = '2017 NRD'

def import_photos(input_csv="../2012_ROOMS_phote.csv"):
  """Change input_csv to actual input file - the default is test data."""
  reader = csv.DictReader(open(input_csv))
  for s in reader:
    number = s["Site ID"]
    site = models.NewSite.all().filter('number =', number).get()
    if not site:
      continue
    if s['Flickr Pages']:
      site.photo_link = s['Flickr Pages']
      site.put()

def import_sites(input_csv="../2012_ROOMS_site_info_sample.csv"):
  """Change input_csv to actual input file - the default is test data."""
  reader = csv.DictReader(open(input_csv))
  for s in reader:
    number = s["Site ID"]
    site = models.NewSite.all().filter('number =', number).get()
    if site:
      logging.info('site %s exists, skipping', number)
      continue
    else:
      site = models.NewSite(number=number)
    site.program = PROGRAM
    site.budget = int(s["Budgeted Cost in Campaign"]) if s["Budgeted Cost in Campaign"] else 0

    # Because Python 2.x csv module only reads ascii.
    def clean_s(k):
      return s[k].replace('\n', ' ').replace('\xe2', "'").replace('\x80', "'").replace('\x99', '').replace('\xc3', '').replace('\x95', '').replace('\xb1', '').encode('ascii', 'replace')

    site.name = clean_s("Repair Application: Applicant's Name")
    site.street_number = clean_s("Street Address")
    site.city_state_zip = "%s CA, %s" % (
        clean_s("Repair Application: Recipient's City"),
        clean_s("Repair Application: Recipient's Zip Code"))
    site.applicant = clean_s("Repair Application: Applicant's Name")
    site.applicant_home_phone = clean_s("Repair Application: Applicant Home Phone")
    site.applicant_work_phone = clean_s("Repair Application: Applicant Work Phone")
    site.applicant_mobile_phone = clean_s("Repair Application: Applicant Mobile Phone")
    site.sponsor = clean_s("(Sponsor) Campaign Description")
    site.rrp_test = clean_s("Repair Application: RRP Test Results")
    site.rrp_level = clean_s("Repair Application: RRP Result Notes")
    # site.roof = clean_s("Roof?")
    site.jurisdiction = clean_s("Jurisdiction")
    site.announcement_subject = clean_s("Announcement Subject")
    site.announcement_body = clean_s("Announcement Body")
    site.put()
    logging.info('put site %s', number)

def import_captains(input_csv="../2012_ROOMS_Captain_email_sample.csv"):
  """Change input_csv to actual input file - the default is test data."""
  reader = csv.DictReader(open(input_csv))
  for s in reader:
    def clean_s(k):
      return s[k].replace('\n', ' ').replace('\xe2', "'").replace('\x80', "'").replace('\x99', '').replace('\xc3', '').replace('\x95', '').encode('ascii', 'replace')

    key = s.get('key')
    email = clean_s("Email")
    rooms_id = clean_s("ROOMS Captain ID")
    # name = "%s %s" % (clean_s("First Name"),
    #                   clean_s("Last Name"))
    name = clean_s("Name")
    captain = None
    if key:
      captain = models.Captain.get_by_id(int(key))
      if captain:
        logging.info('got captain from key %s', key)
    if not captain:
      captain = models.Captain.all().filter('rooms_id =', rooms_id).get()
      if captain:
        logging.info('got captain from rooms_id %s', rooms_id)
    if not captain:
      captain = models.Captain.all().filter('email =', email).get()
      if captain:
        logging.info('got captain from email %s', email)
    if not captain:
      logging.info('creating captain key %s name %s email %s rooms_id %s',
                   key, name, email, rooms_id)
      captain = models.Captain(name=name, email=email, rooms_id=rooms_id)

    # Over-write these values, assume volunteer database is more up to
    # date.
    captain.name = name
    captain.email = email
    captain.rooms_id = rooms_id
    # captain.phone1 = clean_s("Preferred Phone") or None
    # captain.phone_mobile = clean_s("Phone mobile")
    # captain.phone_work = clean_s("Phone work")
    # captain.phone_home = clean_s("Phone home")
    # captain.phone_fax = clean_s("Phones Fax::number")
    # captain.phone_other = clean_s("Phones Other::number")
    captain.put()

    number = s["Site ID"]
    site = models.NewSite.all().filter('number =', number).get()
    if not site:
      logging.error('site %s does not exist, skipping', number)
      continue

    # In input type is like "Volunteer Captain" but in model it's
    # "Volunteer"
    input_type = s["Captain Type"]
    for t in models.SiteCaptain.type.choices:
      if t in input_type:
        break

    query = models.SiteCaptain.all()
    query.filter('site =', site).filter('captain =', captain)
    sitecaptain = query.get()
    if sitecaptain is None:
      logging.info('Creating new SiteCaptain mapping %s to %s',
                   site.number, captain.name)
      sitecaptain = models.SiteCaptain(site=site, captain=captain, type=t)
    else:
      logging.info('Found existing SiteCaptain')
      sitecaptain.type = t
    sitecaptain.put()

ANNOUNCEMENT_BODY = """Please remember that your Home Depot card will be held until we receive your scope of work form.
Thank you for serving as a captain this year!  Please use this space to include notes/correspondence with staff.
Contact RTP staff for assistance:
Cari, 650-366-6597 x224, cari@rebuildingtogetherpeninsula.org
Adam, 650-366-6597 x223, adam@rebuildingtogetherpeninsula.org
"""
ANNOUNCEMENT_SUBJECT = """Scope of work is due March 2 for CDBG sites; all forms due March 30."""

# SAH
ANNOUNCEMENT_SUBJECT = """Thank you for your help with SAH!"""
ANNOUNCEMENT_BODY = ''

# NRD 2014
ANNOUNCEMENT_BODY = """
Thank you for serving as a captain this year!  Please use this space to include
notes/correspondence with staff.
Contact RTP staff for assistance:
Roger, 650-366-6597 x227, roger@rebuildingtogetherpeninsula.org
Adam, 650-366-6597 x223, adam@rebuildingtogetherpeninsula.org
"""
ANNOUNCEMENT_SUBJECT = """Scope of work is due March 5; all forms due March 28."""

# NRD 2015
ANNOUNCEMENT_BODY = """
Thank you for serving as a captain this year!  REMINDER: Captain's BBQ @RTP
11AM-4PM on Saturday, March 21.
(Deadline to Identify a Runner).  Please use this space to include notes or correspondence with staff.
Contact RTP staff for assistance:
Lindsay, 650-366-6597 x226, lindsay@rebuildingtogetherpeninsula.org
Adam, 650-366-6597 x223, adam@rebuildingtogetherpeninsula.org
"""
ANNOUNCEMENT_SUBJECT = """Scope of work is due March 13; all forms due March 28."""

def set_announcement():
  for s in models.NewSite.all().filter('program =', PROGRAM):
    s.announcement_subject = ANNOUNCEMENT_SUBJECT
    s.announcement_body = ANNOUNCEMENT_BODY
    s.put()


import numpy as np

class Powder(object):
    """
    Описание пороха
    """
    def __init__(self, param_powder):
        self.I_k = param_powder['I_k']
        self.T_1 = param_powder['T_1']
        self.z_k = param_powder['Z_k']
        self.alpha_k = param_powder['alpha_k']
        self.etta = param_powder['etta']
        self.f = param_powder['f']
        self.k_1 = param_powder['k_1']
        self.k_2 = param_powder['k_2']
        self.k_f = param_powder['k_f']
        self.k_l = param_powder['k_l']
        self.lambda_1 = param_powder['lambda_1']
        self.lambda_2 = param_powder['lambda_2']
        self.name = param_powder['name']
        self.ro = param_powder['ro']

        self.f = self.f * 1e6
        self.alpha_k = self.alpha_k / 1e3
        self.ro = self.ro * 1e3
        self.I_k = self.I_k * 1e6

    def psi(self, z):
        """
        Функция газоприхода (относительный объем сгоревшего свода)
        :param z: относительная толщина сгоревщего свода
        :return: функция газоприхода
        """
        cond1 = z < 0
        cond2 = np.logical_and(z >= 0, z < 1)
        cond3 = np.logical_and(1 <= z, z < self.z_k)
        cond4 = z >= self.z_k

        psi = np.zeros_like(z)

        psi[cond1] = 0
        psi[cond2] = self.k_1 * z[cond2] * (1 + self.lambda_1 * z[cond2])
        psi[cond3] = self.k_1 * (1 + self.lambda_1) + self.k_2 * (z[cond3] - 1) * (1 + self.lambda_2 * (z[cond3] - 1))
        psi[cond4] = 1

        return psi

    def dpsi_dz(self, z):
        """
        Производна psi по z
        :param z: относительная толщина сгоревщего свода
        :return: производна psi по z
        """
        cond1 = z < 0
        cond2 = np.logical_and(z >= 0, z < 1)
        cond3 = np.logical_and(1 <= z, z < self.z_k)
        cond4 = z >= self.z_k

        dpsi = np.zeros_like(z)

        dpsi[cond1] = 0
        dpsi[cond2] = self.k_1 * (1 + 2 * self.lambda_1 * z[cond2])
        dpsi[cond3] = self.k_2 * (1 + 2 * self.lambda_2 * (z[cond3] - 1))
        dpsi[cond4] = 0

        return dpsi


#!/usr/bin/env python3

import argparse
import configparser
import json

import trino_api
import utils

def calculate_throughput(size, time):
    return (size / time) if time > 0 else 0

def parse_query_json(query_json):
    query_stats = query_json['queryStats']
    print('=== Query Info ===')
    print('query ID        : ', query_json['queryId'])
    print('state           : ', query_json['state'])
    print('elapsed time    : ', query_stats['elapsedTime'])
    print('total splits    : ', query_stats['totalDrivers'])
    print('completed splits: ', query_stats['completedDrivers'])
    print('running splits  : ', query_stats['runningDrivers'])
    print('queued splits   : ', query_stats['queuedDrivers'])
    print('blocked splits  : ', query_stats['blockedDrivers'])
    stages = utils.build_stages(query_json)
    for stage in stages:
        stage_plan = json.loads(stage['plan']['jsonRepresentation'])
        # we can only care about scan stages
        if 'scan' not in stage_plan['name'].lower():
            continue
        table_name = utils.get_table_name(stage_plan)
        print (' === Source table - {0} ==='.format(table_name))
        print('  Stage ID is: ', stage['stageId'])
        print('  State      : ', stage['state'])
        print('  type       : ', stage_plan['name'])
        print('  table      : ', stage_plan['identifier'])
        stage_catalog_name = utils.get_catalog_name(stage_plan)
        if stage_catalog_name is None:
            stage_catalog_name = "UNKNOWN"
        stage_stats = stage['stageStats']
        physical_input_stage_throughput = calculate_throughput(
            utils.parse_read_data_size(stage_stats['physicalInputDataSize']),
            utils.parse_read_time(stage_stats['physicalInputReadTime'])
        )
        print('  physical input data size        : ', stage_stats['physicalInputDataSize'])
        print('  physical input read time        : ', stage_stats['physicalInputReadTime'])
        print('  physical input throughput (MB/s): ', physical_input_stage_throughput)
        tasks = stage['tasks']
        for task in tasks:
            worker_host = task['taskStatus']['self'].split("//")[1].split(":")[0]
            print('    === Task {0} on worker {1} ==='.format(task['taskStatus']['taskId'], worker_host))
            task_stats = task['stats']
            physical_input_task_throughput = calculate_throughput(
                utils.parse_read_data_size(task_stats['physicalInputDataSize']),
                utils.parse_read_time(task_stats['physicalInputReadTime'])
            )
            print('    physical input data size        : ', task_stats['physicalInputDataSize'])
            print('    physical input read time        : ', task_stats['physicalInputReadTime'])
            print('    physical input throughput (MB/s): ', physical_input_task_throughput)

    session_info = query_json['session']
    print(' === Client Destination - {0} ({1}) ==='.format(session_info['remoteUserAddress'], session_info['source']))
    output_stage_stats = query_json['outputStage']['stageStats']
    print('  buffered                       : ', output_stage_stats['bufferedDataSize'])
    print('  output size                    : ', output_stage_stats['outputDataSize'])
    query_elapsed_time = utils.parse_read_time(query_json['queryStats']['elapsedTime'])
    client_output_throughput = (utils.parse_read_data_size(output_stage_stats['outputDataSize']) / query_elapsed_time) if query_elapsed_time > 0 else 0
    print('  client output throughput (MB/s): ', client_output_throughput)

parser = argparse.ArgumentParser(description='Accepted parameters')
parser.add_argument('-q', '--query', help='query ID', required=True)
args = parser.parse_args()

config = configparser.ConfigParser()
config.read('config.ini')
trino_config = config['trino']

session = trino_api.new_session(trino_config['user'], trino_config['password'])
coordinator_uri = trino_config['http_scheme'] + '://' + trino_config['host'] + ':' + trino_config['port']
verify_certs = trino_config['verify_certs'].lower() == 'true'

parse_query_json(
    trino_api.get_query_json(session, verify_certs, coordinator_uri, trino_config['user'], args.query)
)