import sys, iptc, re, socket

single_options = False

predesigned_rules = ['BlockIncomingSSH', 'BlockOutgoingSSH', 'BlockAllSSH', 'BlockIncomingHTTP', 'BlockIncomingHTTPS',\
    'BlockIncomingPing', 'BlockInvalidPackets', 'BlockSYNFlooding', 'BlockXMASAttack', 'ForceSYNPackets']

accepted_protocols = ['ah','egp','esp','gre','icmp','idp','igmp','ip','pim','pum','pup','raw','rsvp','sctp','tcp','tp','udp']
ipsrc = None
ipsrc_range = None
ipdst = None
ipdst_range = None
portsrc = None
portsrc_range = None
portdst = None
portdst_range = None
protocol = None
interfacein = None
interfaceout = None
target = None
custom_position = 0
direction = None

checker = False

############################### List of Predefined Rules #############################

def block_incoming_ssh():
    chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), "INPUT")
    rule = iptc.Rule()
    rule.protocol = "tcp"
    match = rule.create_match("tcp")
    match.dport = "22"
    match = rule.create_match("state")
    match.state = "NEW"
    target = iptc.Target(rule, "DROP")
    rule.target = target
    chain.insert_rule(rule)
    print("Successfully Created")

def block_outgoing_ssh():
    chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), "OUTPUT")
    rule = iptc.Rule()
    rule.protocol = "tcp"
    match = rule.create_match("tcp")
    match.dport = "22"
    match = rule.create_match("state")
    match.state = "NEW"
    target = iptc.Target(rule, "DROP")
    rule.target = target
    chain.insert_rule(rule)
    print("Successfully Created")

def block_all_ssh():
    chain1 = iptc.Chain(iptc.Table(iptc.Table.FILTER), "INPUT")
    chain2 = iptc.Chain(iptc.Table(iptc.Table.FILTER), "OUTPUT")
    rule = iptc.Rule()
    rule.protocol = "tcp"
    match = rule.create_match("tcp")
    match.dport = "22"
    match = rule.create_match("state")
    match.state = "NEW"
    target = iptc.Target(rule, "DROP")
    rule.target = target
    chain1.insert_rule(rule)
    chain2.insert_rule(rule)
    print("Successfully Created")

def block_incoming_http():
    chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), "INPUT")
    rule = iptc.Rule()
    rule.protocol = "tcp"
    match = rule.create_match("tcp")
    match.dport = "80"
    match = rule.create_match("state")
    match.state = "NEW"
    target = iptc.Target(rule, "DROP")
    rule.target = target
    chain.insert_rule(rule)
    print("Successfully Created")

def block_incoming_https():
    chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), "INPUT")
    rule = iptc.Rule()
    rule.protocol = "tcp"
    match = rule.create_match("tcp")
    match.dport = "443"
    match = rule.create_match("state")
    match.state = "NEW"
    target = iptc.Target(rule, "DROP")
    rule.target = target
    chain.insert_rule(rule)
    print("Successfully Created")

def block_incoming_ping():
    chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), "INPUT")
    rule = iptc.Rule()
    rule.protocol = "icmp"
    match = rule.create_match("icmp")
    match.icmp_type = "echo-reply"
    target = iptc.Target(rule, "DROP")
    rule.target = target
    chain.insert_rule(rule)
    print("Successfully Created")

def block_invalid_packets():
    chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), "INPUT")
    rule = iptc.Rule()
    match = rule.create_match("state")
    match.state = "iNVALID"
    target = iptc.Target(rule, "DROP")
    rule.target = target
    chain.insert_rule(rule)
    print("Successfully Created")

def syn_flooding():
    chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), "INPUT")
    rule = iptc.Rule()
    rule.protocol = "tcp"
    match = rule.create_match("tcp")
    match.tcp_flags = [ 'FIN,SYN,RST,ACK', 'SYN' ]
    match = rule.create_match("limit")
    match.limit = "10/second"
    target = iptc.Target(rule, "ACCEPT")
    rule.target = target
    chain.insert_rule(rule)
    print("Successfully Created")

def block_xmas_attack():
    chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), "INPUT")
    rule = iptc.Rule()
    rule.protocol = "tcp"
    match = rule.create_match("tcp")
    match.tcp_flags = [ 'ALL', 'ALL' ]
    target = iptc.Target(rule, "DROP")
    rule.target = target
    chain.insert_rule(rule)
    print("Successfully Created")

def force_syn_packets():
    chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), "INPUT")
    rule = iptc.Rule()
    rule.protocol = "tcp"
    match = rule.create_match("tcp")
    match.syn = "!1"
    match = rule.create_match("state")
    match.state = "NEW"
    target = iptc.Target(rule, "DROP")
    rule.target = target
    chain.insert_rule(rule)
    print("Successfully Created")

# Function to delete rules

all_rules_deleted = True
def delete_rules(table):
    global all_rules_deleted
    all_rules_deleted = True
    for chain in table.chains:
        #print(chain.name)
        for rule in chain.rules:
            try:
                chain.delete_rule(rule)
                print(rule.protocol, rule.src, rule.dst, rule.target.name, "is DELETED")
            except:
                all_rules_deleted = False
    if(all_rules_deleted==False):
        #print("First Iteration Failed")
        delete_rules(table)

# Function to delete a single rule

def delete_rule(rule, table, direction = None):

    if(direction == 'input'):
        chain = iptc.Chain(table, "INPUT")
        deleted1 = False
        for index, rule in enumerate(chain.rules):
            if(int(rule_number) == index):
                try:
                    chain.delete_rule(rule)
                    print("Rule Successfully Deleted for Input")
                    deleted1 = True
                except:
                    sys.exit("The rule could not be deleted for Input. Please, try again.")
        if(deleted1 == False):
            print("The Rule Could Not Be Found for Input")

    elif (direction == 'output'):
        chain = iptc.Chain(table, "OUTPUT")
        deleted1 = False
        for index, rule in enumerate(chain.rules):
            if(int(rule_number) == index):
                try:
                    chain.delete_rule(rule)
                    print("Rule Successfully Deleted for Output")
                    deleted1 = True
                except:
                    sys.exit("The rule could not be deleted for Input. Please, try again.")
        if(deleted1 == False):
            print("The Rule Could Not Be Found for Output")

    else:
        sys.exit("Delete rule function error. Incorrect parameter")

# First check, for options that should be used alone

for index, value in enumerate(sys.argv):

    if(value == '-l' ):
        if (len(sys.argv)) != 2:
            sys.exit("The option -l does not accept additional options. Please, type: myFirewall -l")
        single_options = True
        table = iptc.Table(iptc.Table.FILTER)
        for chain in table.chains:
            #print ("Chain ",chain.name)
            rule_type = chain.name[:3]
            for index, rule in enumerate(chain.rules):
                dport = None
                sport = None
                ip_src_range = None
                ip_dst_range = None
                match_state = None
                match_tcp_flags = None
                for match in rule.matches:
                    if (match.dport != None):
                        dport = match.dport
                    if (match.sport != None):
                        sport = match.sport
                    if (match.src_range != None):
                        ip_src_range = match.src_range
                    if (match.dst_range != None):
                        ip_dst_range = match.dst_range
                    if (match.state != None):
                        match_state = match.state
                    if (match.tcp_flags != None):
                        match_tcp_flags = match.tcp_flags[match.tcp_flags.find(' ')+1:]
                if(ip_src_range != None):
                    source_ip = ip_src_range
                else:
                    source_ip = rule.src
                if(ip_dst_range != None):
                    destination_ip = ip_dst_range
                else:
                    destination_ip = rule.dst
                print ("==========================================")
                print ("RULE("+ rule_type+")", index, "||", "proto:", rule.protocol + " ||", "sport:", str(sport) + " ||",
                 "dport:", str(dport) + " ||", "src:", source_ip + " ||", "dst:", destination_ip + " ||\n", "|| inInt:",
                 str(rule.in_interface) + " ||", "outInt:", str(rule.out_interface) + " ||",
                 "tcpflags:", str(match_tcp_flags) + " ||", "state:", str(match_state) + " ||", "Target:", rule.target.name)
                print ("==========================================")

    elif(value == '-r'):
        if (len(sys.argv)) != 2:
            sys.exit("The option -r does not accept additional options. Please, type: myFirewall -r")
        single_options = True
        table1 = iptc.Table(iptc.Table.FILTER)
        delete_rules(table1)
        table2 = iptc.Table(iptc.Table.MANGLE)
        delete_rules(table2)
        table3 = iptc.Table(iptc.Table.NAT)
        delete_rules(table3)
        table4 = iptc.Table(iptc.Table.RAW)
        delete_rules(table4)
        table5 = iptc.Table(iptc.Table.SECURITY)
        delete_rules(table5)

    elif(value == '-d'):
        if (len(sys.argv) != 3 and len(sys.argv) != 4):
            sys.exit("The option -d does not accept these options. Please, type: myFirewall -d RuleNumer [-in|-out]")
        single_options = True
        table = iptc.Table(iptc.Table.FILTER)
        rule_number = sys.argv[2]
        if(len(sys.argv) == 4):
            if (sys.argv[3] == '-in'):
                delete_rule(rule_number, table, direction = 'input')
            elif (sys.argv[3] == '-out'):
                delete_rule(rule_number, table, direction = 'output')
            else:
                sys.exit("Incorrect parameter. Please, type: myFirewall -d RuleNumer [-in|-out]")
        else:
            delete_rule(rule_number, table, direction = 'input')
            delete_rule(rule_number, table, direction = 'output')
        #for chain in table.chains:
            #for rule in chain.rules:
            #    chain.delete_rule(rule)

    elif(value == '-all'):
        if ((len(sys.argv) != 3) and (sys.argv[index+1]!='ACCEPT') and (sys.argv[index+1]!='DROP')):
            sys.exit("The -all option lets the user to ACCEPT or DROP all packets, independently of ports,"+\
            " protocols or IPs. Please, specify a ACCEPT or DROP argument")
        else:
            single_options = True
            rule = iptc.Rule()
            rule.target = rule.create_target(sys.argv[index+1])
            chain1 = iptc.Chain(iptc.Table(iptc.Table.FILTER), "INPUT")
            chain2 = iptc.Chain(iptc.Table(iptc.Table.FILTER), "OUTPUT")
            chain1.insert_rule(rule)
            chain2.insert_rule(rule)

    elif(value == '-rule'):
        single_options = True
        if (len(sys.argv)) != 3:
            if (len(sys.argv) == 2):
                print("The list of rules available is:\n")
                for i in predesigned_rules:
                    print(i)
            else:
                sys.exit("The option -r does not accept additional options. Please, type: -rule RULE")
        elif(sys.argv[index+1] == 'BlockIncomingSSH'):
            block_incoming_ssh()
        elif(sys.argv[index+1] == 'BlockOutgoingSSH'):
            block_outgoing_ssh()
        elif(sys.argv[index+1] == 'BlockAllSSH'):
            block_all_ssh()
        elif(sys.argv[index+1] == 'BlockIncomingHTTP'):
            block_incoming_http()
        elif(sys.argv[index+1] == 'BlockIncomingHTTPS'):
            block_incoming_https()
        elif(sys.argv[index+1] == 'BlockIncomingPing'):
            block_incoming_ping()
        elif(sys.argv[index+1] == 'BlockInvalidPackets'):
            block_invalid_packets()
        elif(sys.argv[index+1] == 'BlockSYNFlooding'):
            syn_flooding()
        elif(sys.argv[index+1] == 'BlockXMASAttack'):
            block_xmas_attack()
        elif(sys.argv[index+1] == 'ForceSYNPackets'):
            force_syn_packets()
        else:
            print("Rule not available. The list of available rules is:\n")
            for i in predesigned_rules:
                print(i)
            print("")

if(not single_options):
    # Iterator to retrieve all information and create a Rule
    for index, value in enumerate(sys.argv):

        if(value == '-ipsrc'):
            match_single = re.search('^(([0-9]?[0-9]\.)|(1[0-9][0-9]\.)|(2[0-5][0-5]\.)){3}(([0-9]?[0-9])|(1[0-9][0-9])|(2[0-5][0-5]))$', sys.argv[index+1])
            match_range = re.search('^(([0-9]?[0-9]\.)|(1[0-9][0-9]\.)|(2[0-5][0-5]\.)){3}(([0-9]?[0-9])|(1[0-9][0-9])|(2[0-5][0-5]))-(([0-9]?[0-9]\.)|(1[0-9][0-9]\.)|(2[0-5][0-5]\.)){3}(([0-9]?[0-9])|(1[0-9][0-9])|(2[0-5][0-5]))$', sys.argv[index+1])
            if((match_single==None) and (match_range==None)):
                sys.exit("The IP address format is incorrect")
            else:
                checker = True
                if(match_single!=None):
                    ipsrc = sys.argv[index+1]
                if(match_range!=None):
                    ipsrc_range = sys.argv[index+1]

        elif(value == '-ipdst'):
            match_single = re.search('^(([0-9]?[0-9]\.)|(1[0-9][0-9]\.)|(2[0-5][0-5]\.)){3}(([0-9]?[0-9])|(1[0-9][0-9])|(2[0-5][0-5]))$', sys.argv[index+1])
            match_range = re.search('^(([0-9]?[0-9]\.)|(1[0-9][0-9]\.)|(2[0-5][0-5]\.)){3}(([0-9]?[0-9])|(1[0-9][0-9])|(2[0-5][0-5]))-(([0-9]?[0-9]\.)|(1[0-9][0-9]\.)|(2[0-5][0-5]\.)){3}(([0-9]?[0-9])|(1[0-9][0-9])|(2[0-5][0-5]))$', sys.argv[index+1])
            if(match_single==None and match_range==None):
                sys.exit("The IP address format is incorrect")
            else:
                checker = True
                if(match_single!=None):
                    ipdst = sys.argv[index+1]
                if(match_range!=None):
                    ipdst_range = sys.argv[index+1]

        elif(value == '-portsrc'):
            match_single = re.search('^[0-9]+$', sys.argv[index+1])
            match_range = re.search('^[0-9]+:[0-9]+$', sys.argv[index+1])
            if(match_single==None and match_range==None):
                sys.exit("The Port/Port range format is incorrect")
            checker = True
            if(match_single != None):
                if(int(sys.argv[index+1])<65536 and int(sys.argv[index+1])>0):
                    portsrc = sys.argv[index+1]
                else:
                    sys.exit("The specified port is out of the boundaries. Please, type a value between 1 and 65535")
            elif(match_range != None):
                first_port_group = int(sys.argv[index+1][:sys.argv[index+1].find(':')])
                second_port_group = int(sys.argv[index+1][sys.argv[index+1].find(':')+1:])
                if(((first_port_group<65536) and (first_port_group>0) and (second_port_group<65536) and (second_port_group>0))):
                    portsrc_range = sys.argv[index+1]
                else:
                    sys.exit("The specified port range is out of the boundaries. Please, type values between 1 and 65535")
            else:
                sys.exit("Port incorrectly parsed")

        elif(value == '-portdst'):
            match_single = re.search('^[0-9]+$', sys.argv[index+1])
            match_range = re.search('^[0-9]+:[0-9]+$', sys.argv[index+1])
            if(match_single==None and match_range==None):
                sys.exit("The Port/Port range format is incorrect")
            checker = True
            if(match_single != None):
                if(int(sys.argv[index+1])<65536 and int(sys.argv[index+1])>0):
                    portdst = sys.argv[index+1]
                else:
                    sys.exit("The specified port is out of the boundaries. Please, type a value between 1 and 65535")
            elif(match_range != None):
                first_port_group = int(sys.argv[index+1][:sys.argv[index+1].find(':')])
                second_port_group = int(sys.argv[index+1][sys.argv[index+1].find(':')+1:])
                if(((first_port_group<65536) and (first_port_group>0) and (second_port_group<65536) and (second_port_group>0))):
                    portdst_range = sys.argv[index+1]
                else:
                    sys.exit("The specified port range is out of the boundaries. Please, type values between 1 and 65535")
            else:
                sys.exit("Port incorrectly parsed")

        elif(value == '-proto'):
            accepted = False
            for i in accepted_protocols:
                if(i == sys.argv[index+1]):
                    accepted = True
                else:
                    protocol = sys.argv[index+1]
            if(not accepted):
                sys.exit("The protocol provided is not accepted. The list of accepted protocols is:",'ah',
                'egp','esp','gre','icmp','idp','igmp','ip','pim','pum','pup','raw','rsvp','sctp','tcp','tp','udp')
            checker = True

        elif(value == '-intin'):
            available_interface = False
            for i in socket.if_nameindex():
                if(i[1] == sys.argv[index+1]):
                    available_interface = True
            if(available_interface == False):
                sys.exit("The selected interface is not available on this system")
            else:
                interfacein = sys.argv[index+1]
            checker = True

        elif(value == '-intout'):
            available_interface = False
            for i in socket.if_nameindex():
                if(i[1] == sys.argv[index+1]):
                    available_interface = True
            if(available_interface == False):
                sys.exit("The selected interface is not available on this system")
            else:
                interfaceout = sys.argv[index+1]
            checker = True

        elif(value == '-pos'):
            match = re.search('^[0-9]*$', sys.argv[index+1])
            if(match==None):
                sys.exit("Incorrect position format. Please, type an integer >= 0")
            else:
                custom_position = sys.argv[index+1]
            checker = True

        elif(value == '-t'):
            if(sys.argv[index+1] == "ACCEPT"):
                target = "ACCEPT"
            elif(sys.argv[index+1] == "DROP"):
                target = "DROP"
            else:
                sys.exit('Incorrect target option. Please, choose between "ACCEPT" and "DROP"')
            checker = True

        elif(value == '-in'):
            direction = 'incoming'

        elif(value == '-out'):
            direction = 'outgoing'
        else:
            if(checker == True or index==0):
                checker = False
            else:
                sys.exit("Incorrect option: " + value)

    rule = iptc.Rule()

    if(ipsrc != None):
        rule.src = ipsrc

    if(ipsrc_range != None or ipdst_range != None):
        match = rule.create_match("iprange")
        if(ipsrc_range != None):
            match.src_range = ipsrc_range
        else:
            match.dst_range = ipdst_range

    if(ipdst != None):
        rule.dst = ipdst

    if(protocol != None):
        rule.protocol = protocol
        if(protocol == "tcp" or protocol == "udp"):
            match = rule.create_match(protocol)

    if(portsrc != None or portdst != None):
        if(protocol == None):
            protocol = "tcp"
            rule.protocol = protocol
            match = rule.create_match(protocol)
        if(portsrc != None):
            match.sport = portsrc
        if(portdst != None):
            match.dport = portdst

    if(portsrc_range != None or portdst_range != None):
        if(protocol == None):
            protocol = "tcp"
            rule.protocol = protocol
            match = rule.create_match(protocol)
        if(portsrc_range != None):
            match.sport = portsrc_range
        if(portdst_range != None):
            match.dport = portdst_range

    if(interfacein != None):
        rule.in_interface = interfacein

    if(interfaceout != None):
        rule.out_interface = interfaceout

    if(target != None):
        rule.target = rule.create_target(target)
    else:
        sys.exit('You must specify a target: -t "ACCEPT" or -t "DROP"')

    if(direction == None):
        chain1 = iptc.Chain(iptc.Table(iptc.Table.FILTER), "INPUT")
        chain2 = iptc.Chain(iptc.Table(iptc.Table.FILTER), "OUTPUT")
        try:
            chain1.insert_rule(rule, position=int(custom_position))
        except:
            sys.exit("Index of insertion out of boundaries for existing Input table. Please, choose a value between 0 and (Max.AmountOfRules-1)")
        try:
            chain2.insert_rule(rule, position=int(custom_position))
        except:
            sys.exit("Index of insertion out of boundaries for Output table. Please, choose a value between 0 and (Max.AmountOfRules-1)")

    elif(direction == "incoming"):
        chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), "INPUT")
        try:
            chain.insert_rule(rule, position=int(custom_position))
        except:
            sys.exit("Index of insertion out of boundaries.  Please, choose a value between 0 and (Max.AmountOfRules-1)")

    elif(direction == "outgoing"):
        chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), "OUTPUT")
        try:
            chain.insert_rule(rule, position=int(custom_position))
        except:
            sys.exit("Index of insertion out of boundaries. Please, choose a value between 0 and (Max.AmountOfRules-1)")


import urllib
from xml.dom import minidom
import re

def buildResponse(node_list):
	return_string = ""
	for i in node_list:
		return_string = return_string + i + "\n"
	return return_string.strip()

def buildURL(key, word):
	return "http://www.dictionaryapi.com/api/v1/references/collegiate/xml/" + word + "?key=" + key

def getXML(word):
	url = buildURL("1a276aec-1aa8-42d4-9575-d29c2d4fb105", word)
	response = urllib.urlopen(url).read()
	data = minidom.parseString(str(response))
	return data

def getDefinition(word):
	data = getXML(word)
	itemlist = data.getElementsByTagName('def')
	node_list = []
	for i in itemlist:
		dts = i.getElementsByTagName('dt')
		node_list.append(str(dts[0].childNodes[0].nodeValue))
	if len(node_list) < 3:
		return buildResponse(node_list)
	else:
		return buildResponse(node_list[:3])


from rest_framework.response import Response
from rest_framework.decorators import api_view
from rest_framework import status
from curriculum.serializers.curriculum_serializers import SubjectLevelListSerializer, SubjectLevelSerializer, SubjectLevelWriteSerializer
from rest_framework.exceptions import NotFound
from rest_framework.views import APIView

from curriculum.models import SubjectLevel

#
# SUBJECT LEVEL VIEWS
#

class SubjectLevelList(APIView):
    """
    List all SubjectLevels, or create a new one.
    """

    def get(self, request, school_pk=None, format=None):
        subject_levels = SubjectLevel.objects.all()

        if school_pk:
            subject_levels = subject_levels.filter(
                subject__school__id=school_pk)

        subject = request.query_params.get('subject', None)
        level = request.query_params.get('level', None)

        if subject:
            subject_levels = subject_levels.filter(subject_id=subject)

        if level:
            subject_levels = subject_levels.filter(level_id=level)

        serializer = SubjectLevelListSerializer(subject_levels, many=True)
        return Response(serializer.data)

    def post(self, request, format=None):
        serializer = SubjectLevelWriteSerializer(data=request.data)
        if serializer.is_valid():
            new_subject_level = serializer.save()
            new_serializer = SubjectLevelListSerializer(new_subject_level)
            return Response(new_serializer.data, status=status.HTTP_201_CREATED)
        return Response(new_serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class SubjectLevelDetail(APIView):
    """
    Retrieve, update or delete a SubjectLevel.
    """

    def get_object(self, subject_level_pk):
        try:
            return SubjectLevel.objects.get(id=subject_level_pk)
        except SubjectLevel.DoesNotExist:
            raise NotFound(detail="Object with this ID not found.")

    def get(self, request, subject_level_pk, format=None):
        subject_level = self.get_object(subject_level_pk)
        serializer = SubjectLevelSerializer(subject_level)
        return Response(serializer.data)

    def put(self, request, subject_level_pk, format=None):
        subject_level = self.get_object(subject_level_pk)
        serializer = SubjectLevelWriteSerializer(
            subject_level, data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

     # Partially update a specific entry by primary key
    def patch(self, request, subject_level_pk):
        subject_level = self.get_object(subject_level_pk)
        serializer = SubjectLevelWriteSerializer(
            subject_level, data=request.data, partial=True)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

    def delete(self, request, subject_level_pk, format=None):
        subject_level = self.get_object(subject_level_pk)
        subject_level.delete()
        return Response(status=status.HTTP_204_NO_CONTENT)


import torch
from models.conformer.activation import GLU, Swish

class DepthWiseConvolution(torch.nn.Module):
    def __init__(self, in_channels, kernel_size, stride, padding):
        super(DepthWiseConvolution, self).__init__()
        self.conv = torch.nn.Conv1d(in_channels, in_channels, kernel_size, stride, padding, groups=in_channels)

    def forward(self, x):
        x = x.permute(0, 2, 1)
        x = self.conv(x)
        x = x.permute(0, 2, 1)
        return x

class PointWiseConvolution(torch.nn.Module):
    def __init__(self, in_channels, out_channels, stride=1):
        super(PointWiseConvolution, self).__init__()
        self.conv = torch.nn.Conv1d(in_channels, out_channels, 1, stride, 0)

    def forward(self, x):
        x = x.permute(0, 2, 1)
        x = self.conv(x)
        x = x.permute(0, 2, 1)
        return x

class Permute(torch.nn.Module):
    def __init__(self, dims):
        super(Permute, self).__init__()
        self.dims = dims

    def forward(self, x):
        return x.permute(*self.dims)

class ConvolutionModule(torch.nn.Module):
    def __init__(self, d_model, dropout, kernel_size=3):
        super(ConvolutionModule, self).__init__()
        self.conv = torch.nn.Sequential(
            torch.nn.LayerNorm(d_model),
            PointWiseConvolution(d_model, 2 * d_model),
            GLU(),
            DepthWiseConvolution(d_model, kernel_size, 1, int(kernel_size / 2)),
            Permute((0, 2, 1)),
            torch.nn.BatchNorm1d(d_model),
            Permute((0, 2, 1)),
            Swish(),
            PointWiseConvolution(d_model, d_model),
            torch.nn.Dropout(dropout),
        )

    def forward(self, x):
        return self.conv(x)


# Import the libraries
import cv2
import os
import numpy as np

class Auxiliary(object):
    """
    Class that provides some auxiliary functions.
    """

    def __init__(self, size_x=100, size_y=100, interpolation=cv2.INTER_CUBIC):
        """
        Set the default values for the image size and the interpolation method.
        Available interpolation methods provided by OpenCV: INTER_CUBIC, INTER_AREA, INTER_LANCZOS4, INTER_LINEAR, INTER_NEAREST
        :param size_x: Set the default image width (default = 100).
        :param size_y: Set the default image height (default = 100).
        :param interpolation: Set the default interpolation method (default cv2.INTER_CUBIC).
        """
        self.size_x = size_x
        self.size_y = size_y
        self.interpolation = interpolation

        # Declare all supported files
        self.supported_files = ["png", "jpg", "jpeg"]

    def set_default_size(self, size_x, size_y):
        """
        Set the default size.
        :param size_x: Image width.
        :param size_y: Image height.
        """
        if size_x > 0:
            self.size_x = size_x
        if size_y > 0:
            self.size_y = size_y

    def get_default_size(self):
        """
        Get the default image size defined (default is 100x100).
        """
        return self.size_x, self.size_y

    def get_interpolation_method_name(self):
        """
        Get the selected interpolation method name.
        :return: A string containing the interpolation method name.
        """
        if self.interpolation == cv2.INTER_CUBIC:
            return "cv2.INTER_CUBIC"
        if self.interpolation == cv2.INTER_AREA:
            return "cv2.INTER_AREA"
        if self.interpolation == cv2.INTER_LANCZOS4:
            return "cv2.INTER_LANCZOS4"
        if self.interpolation == cv2.INTER_LINEAR:
            return "cv2.INTER_LINEAR"
        if self.interpolation == cv2.INTER_NEAREST:
            return "cv2.INTER_NEAREST"

        raise NameError("Invalid interpolation method name")
        return ""

    @staticmethod
    def calc_accuracy(recognized_images, total_face_images):
        """
        Calculates the accuracy (percentage) using the formula:
        acc = (recognized_images / total_face_images) * 100
        :param recognized_images: The number of recognized face images.
        :param total_face_images: The number of total face images.
        :return: The accuracy.
        """
        try:
            return (float(recognized_images) /
                    float(total_face_images)) * 100.0
        except ZeroDivisionError:
            return 0.0

    @staticmethod
    def write_text_file(content, file_name):
        """
        Write the content to a text file based on the file name.
        :param content: The content as a string.
        :param file_name: The file name (e.g. home/user/test.txt)
        """
        # Save the text file
        text_file = open(file_name, "w")
        text_file.write(content)
        text_file.close()

    @staticmethod
    def is_grayscale(image):
        """
        Check if an image is in grayscale.
        :param image: The image.
        :return: True if the image is in grayscale.
        """
        if len(image.shape) <= 2:
            return True

        h, w = image.shape[:2]  # rows, cols, channels
        for i in range(w):
            for j in range(h):
                p = image[i, j]
                if p[0] != p[1] != p[2]:
                    return False
        return True

    @staticmethod
    def to_grayscale(image):
        """
        Convert an image to grayscale
        :param image: The image.
        :return: The image in grayscale.
        """
        if image is None:
            print("Invalid Image: Could not convert to grayscale")
            return None
        return cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

    @staticmethod
    def load_image(path):
        """
        Load an image based on the path passed by parameter.
        :param path: The path to the image file.
        :return: The image object.
        """
        return cv2.imread(path)

    @staticmethod
    def save_image(file_name, image):
        """
        Save an image based on the fileName passed by parameter.
        :param file_name: The file name.
        :param image: The image.
        """
        cv2.imwrite(file_name, image)

    @staticmethod
    def resize_image(image, size_x, size_y, interpolation_method):
        """
        Resize an image.
        :param image: The image object.
        :param size_x: The image width.
        :param size_y: The image height.
        :param interpolation_method: The interpolation method.
        :return: The resized image.
        """
        if image is None:
            print("Invalid Image: Could not be resized")
            return -1

        rows, cols = image.shape
        if rows <= 0 or cols <= 0:
            print("Invalid Image Sizes: Could not be resized")
            return -1

        return cv2.resize(image, (size_x, size_y),
                          interpolation=interpolation_method)

    def preprocess_image(self, path):
        """
        Preprocess an image. Load an image, convert to grayscale and resize it.
        :param path: The image path.
        :return: The preprocessed image.
        """
        # Load the image
        image = self.load_image(path)

        if image is None:
            print("Could not load the image:", path)
            return None

        # Convert to grayscale
        image = self.to_grayscale(image)
        # Resize the image
        image = self.resize_image(
            image, self.size_x, self.size_y, self.interpolation)
        # Return the processed image
        return image

    @staticmethod
    def concatenate_images(left_image, right_image):
        """
        Concatenate two images side by side (horizontally) and returns a new one.
        :param left_image: The image that should be put to the left.
        :param right_image: The image that should be put to the right.
        :return: The new concatenated image.
        """
        try:
            return np.concatenate((left_image, right_image), axis=1)
        except ValueError:
            return None

    def extract_images_paths(self, path):
        """
        Extract all paths for each image in a directory.
        :param path: The directory path.
        :return: A list with all file paths.
        """
        paths = []

        # In the path folder search for all files in all directories
        for dir_name, dir_names, file_names in os.walk(path):
            # For each file found
            for file_name in file_names:
                # Check if it is a valid image file
                if file_name.split(".")[1] in self.supported_files:
                    # Creates the filePath joining the directory name and the
                    # file name
                    paths.append(os.path.join(dir_name, file_name))

        return paths

    @staticmethod
    def extract_files_paths(path):
        """
        Extract all paths for all files type.
        :param path: The directory path.
        :return: A list with all paths for all files.
        """
        paths = []

        # In the path folder search for all files in all directories
        for dir_name, dir_names, file_names in os.walk(path):
            # For each file found
            for file_name in file_names:
                # Creates the filePath joining the directory name and the file
                # name
                paths.append(os.path.join(dir_name, file_name))

        return paths

    def load_all_images_for_train(self, train_path):
        """
        Load all images for training.
        :param train_path: The train path.
        :return: Three lists with the images, labels and file names.
        """
        images = []
        labels = []
        file_name = []

        paths = self.extract_images_paths(train_path)

        # For each file path
        for file_path in paths:
            # Check if it is a valid image file
            if file_path.split(".")[1] in self.supported_files:

                # Get the subject id (label) based on the format:
                # subjectID_imageNumber.png
                path_split = file_path.split("/")
                temp_name = path_split[len(path_split) - 1]
                subject_id = int(temp_name.split("_")[0])

                images.append(self.preprocess_image(file_path))
                labels.append(subject_id)
                file_name.append(temp_name.split(".")[0])

        return images, labels, file_name

    def load_all_images_for_test(self, test_path):
        """
        Load all images for test.
        :param test_path: The test path.
        :return: Three lists with the images, labels and file names.
        """
        images = []
        labels = []
        file_name = []

        paths = self.extract_images_paths(test_path)

        # For each file path
        for file_path in paths:

            # Check if it is a valid image file
            if file_path.split(".")[1] in self.supported_files:

                # Get the subject id (label)
                # IMPORTANT: it follows the pattern: imageNumber_subjectID.png
                # It is different from the pattern on the training set
                path_split = file_path.split("/")
                temp_name = path_split[len(path_split) - 1]
                subject_id = temp_name.split("_")[1]
                subject_id = int(subject_id.split(".")[0])

                image = self.preprocess_image(file_path)

                if image is None:
                    return None, None, None

                images.append(image)
                labels.append(subject_id)
                file_name.append(temp_name.split(".")[0])

        return images, labels, file_name


import numpy as np
from lib import EulerUtils as eu

# Problem 36 solution!

def checkIfNumberIsPalindromeInBothBases(number):
  numberString = str(number)
  baseTwoString = "{0:b}".format(number)
  if (eu.isPalindrome(numberString) and eu.isPalindrome(baseTwoString)):
    return True
  else:
    return False

sum = sum(x for x in range(1000000) if checkIfNumberIsPalindromeInBothBases(x))
print (sum)


from typing import Any, Dict
import os
import json

import httpx

from odt.config import PipeConfig

_TEMPFILENAME = "lgbm_tmp_model.txt"

class ODTManager:

    def __init__(self, server_host: str) -> None:
        self.server_host = server_host

    def update_config(self, config: PipeConfig):
        # serialization from pydandic with .json method doesn't work internally
        json_data = json.loads(config.json())
        r = httpx.post(f"{self.server_host}/config", json=json_data)

        if r.status_code == 200:
            print("Updating config succeeded!")
        else:
            raise Exception(f"Something went wrong updating the config, status code {r.status_code}")

    def update_model(self, model: Any):

        model.save_model(_TEMPFILENAME)

        with open(_TEMPFILENAME, "r") as f:
            in_mem_model: str = f.read()

        os.remove(_TEMPFILENAME)

        r = httpx.post(f"{self.server_host}/model", content=bytes(in_mem_model, encoding='utf-8'))

        if r.status_code == 200:
            print("Updating model succeeded!")
        else:
            raise Exception(f"Something went wrong updating the model, status code {r.status_code}")

    def update_config_and_model(self, config: PipeConfig, model: Any):
        self.update_config(config)
        self.update_model(model)

    def get_prediction(self, data: Dict[str, Any]) -> float:
        r = httpx.post(f"{self.server_host}/predict", json=data)
        return r.json()


from google.appengine.ext import webapp
from google.appengine.ext.webapp.util import run_wsgi_app

from sendQueries import SendQueriesHandler
from ResponseHandler import ResponseHandler

class HomeHandler(webapp.RequestHandler):
  def get(self):
    self.response.out.write("Hello!")

appRoute = webapp.WSGIApplication( [
  ('/', HomeHandler),
  ('/response', ResponseHandler),
  ('/sendQueries', SendQueriesHandler),
], debug=True)

def main():
  run_wsgi_app(appRoute)

if __name__ == '__main__':
  main()


import os
import logging

from novelwriter.enum import nwItemLayout, nwItemClass
from novelwriter.error import formatException
from novelwriter.common import isHandle, sha256sum

logger = logging.getLogger(__name__)

class NWDoc():

    def __init__(self, theProject, theHandle):

        self.theProject = theProject

        # Internal Variables
        self._theItem   = None  # The currently open item
        self._docHandle = None  # The handle of the currently open item
        self._fileLoc   = None  # The file location of the currently open item
        self._docMeta   = {}    # The meta data of the currently open item
        self._docError  = ""    # The latest encountered IO error
        self._prevHash  = None  # Previous sha256sum of the document file
        self._currHash  = None  # Latest sha256sum of the document file

        if isHandle(theHandle):
            self._docHandle = theHandle

        if self._docHandle is not None:
            self._theItem = self.theProject.projTree[theHandle]

        return

    def __repr__(self):
        return f"<NWDoc handle={self._docHandle}>"

    def __bool__(self):
        return self._docHandle is not None and bool(self._theItem)

    ##
    #  Class Methods
    ##

    def readDocument(self, isOrphan=False):
        """Read the document specified by the handle set in the
        contructor, capturing potential file system errors and parse
        meta data. If the document doesn't exist on disk, return an
        empty string. If something went wrong, return None.
        """
        self._docError = ""
        if self._docHandle is None:
            logger.error("No document handle set")
            return None

        if self._theItem is None and not isOrphan:
            logger.error("Unknown novelWriter document")
            return None

        docFile = self._docHandle+".nwd"
        logger.debug("Opening document: %s", docFile)

        docPath = os.path.join(self.theProject.projContent, docFile)
        self._fileLoc = docPath

        theText = ""
        self._docMeta = {}
        self._prevHash = sha256sum(docPath)

        if os.path.isfile(docPath):
            try:
                with open(docPath, mode="r", encoding="utf-8") as inFile:
                    # Check the first <= 10 lines for metadata
                    for i in range(10):
                        inLine = inFile.readline()
                        if inLine.startswith(r"%%~"):
                            self._parseMeta(inLine)
                        else:
                            theText = inLine
                            break

                    # Load the rest of the file
                    theText += inFile.read()

            except Exception as exc:
                self._docError = formatException(exc)
                return None

        else:
            # The document file does not exist, so we assume it's a new
            # document and initialise an empty text string.
            logger.debug("The requested document does not exist")
            return ""

        return theText

    def writeDocument(self, docText, forceWrite=False):
        """Write the document specified by the handle attribute. Handle
        any IO errors in the process  Returns True if successful, False
        if not.
        """
        self._docError = ""
        if self._docHandle is None:
            logger.error("No document handle set")
            return False

        self.theProject.ensureFolderStructure()

        docFile = self._docHandle+".nwd"
        logger.debug("Saving document: %s", docFile)

        docPath = os.path.join(self.theProject.projContent, docFile)
        docTemp = os.path.join(self.theProject.projContent, docFile+"~")

        if self._prevHash is not None and not forceWrite:
            self._currHash = sha256sum(docPath)
            if self._currHash is not None and self._currHash != self._prevHash:
                logger.error("File has been altered on disk since opened")
                return False

        # DocMeta Line
        if self._theItem is None:
            docMeta = ""
        else:
            docMeta = (
                f"%%~name: {self._theItem.itemName}\n"
                f"%%~path: {self._theItem.itemParent}/{self._theItem.itemHandle}\n"
                f"%%~kind: {self._theItem.itemClass.name}/{self._theItem.itemLayout.name}\n"
            )

        try:
            with open(docTemp, mode="w", encoding="utf-8") as outFile:
                outFile.write(docMeta)
                outFile.write(docText)
        except Exception as exc:
            self._docError = formatException(exc)
            return False

        # If we're here, the file was successfully saved, so we can
        # replace the temp file with the actual file
        try:
            os.replace(docTemp, docPath)
        except OSError as exc:
            self._docError = formatException(exc)
            return False

        self._prevHash = sha256sum(docPath)
        self._currHash = self._prevHash

        return True

    def deleteDocument(self):
        """Permanently delete a document source file and related files
        from the project data folder.
        """
        self._docError = ""
        if self._docHandle is None:
            logger.error("No document handle set")
            return False

        chkList = [
            os.path.join(self.theProject.projContent, f"{self._docHandle}.nwd"),
            os.path.join(self.theProject.projContent, f"{self._docHandle}.nwd~"),
        ]

        for chkFile in chkList:
            if os.path.isfile(chkFile):
                try:
                    os.unlink(chkFile)
                    logger.debug("Deleted: %s", chkFile)
                except Exception as exc:
                    self._docError = formatException(exc)
                    return False

        return True

    ##
    #  Getters
    ##

    def getFileLocation(self):
        """Return the file location of the current document.
        """
        return self._fileLoc

    def getCurrentItem(self):
        """Return a pointer to the currently open NWItem.
        """
        return self._theItem

    def getMeta(self):
        """Parse the document meta tag and return the name, parent,
        class and layout meta values.
        """
        theName = self._docMeta.get("name", "")
        theParent = self._docMeta.get("parent", None)
        theClass = self._docMeta.get("class", None)
        theLayout = self._docMeta.get("layout", None)

        return theName, theParent, theClass, theLayout

    def getError(self):
        """Return the last recorded exception.
        """
        return self._docError

    ##
    #  Internal Functions
    ##

    def _parseMeta(self, metaLine):
        """Parse a line from the document starting with the characters
        %%~ that may contain meta data.
        """
        if metaLine.startswith("%%~name:"):
            self._docMeta["name"] = metaLine[8:].strip()

        elif metaLine.startswith("%%~path:"):
            metaVal = metaLine[8:].strip()
            metaBits = metaVal.split("/")
            if len(metaBits) == 2:
                if isHandle(metaBits[0]):
                    self._docMeta["parent"] = metaBits[0]
                if isHandle(metaBits[1]):
                    self._docMeta["handle"] = metaBits[1]

        elif metaLine.startswith("%%~kind:"):
            metaVal = metaLine[8:].strip()
            metaBits = metaVal.split("/")
            if len(metaBits) == 2:
                if metaBits[0] in nwItemClass.__members__:
                    self._docMeta["class"] = nwItemClass[metaBits[0]]
                if metaBits[1] in nwItemLayout.__members__:
                    self._docMeta["layout"] = nwItemLayout[metaBits[1]]

        else:
            logger.debug("Ignoring meta data: '%s'", metaLine.strip())

        return

# END Class NWDoc


import faust
import uuid

app = faust.App(
        'greetings',
        broker='kafka://localhost:9092',
        )

class Greeting(faust.Record, serializer='json', isodates=True):
    message: str
    uuid: str

greetings_topic = app.topic('greetings', value_type=Greeting)

@app.agent(greetings_topic)
async def get_greetings(greetings):
    """Receives the message and prints the greeting in the logger
    """
    async for greeting in greetings:
        print(greeting.message)
        print(greeting.uuid)

@app.timer(5)
async def produce():
    for i in range(100):
        await get_greetings.send(value={
            "message": f'hello from {i}',
            "uuid": uuid.uuid1()
            })

if __name__ == '__main__':
    app.main()


from collections import defaultdict, deque
from enum import Enum

def read(filename):
    with open(filename) as f:
        insts = (line.strip().split(' ') for line in f)
        return [(inst[0], tuple(inst[1:])) for inst in insts]

def isint(exp):
    try:
        int(exp)
        return True
    except ValueError:
        return False

def val(exp, regs):
    if isint(exp):
        return int(exp)
    return regs[exp]

class State(Enum):
    ENDED = 1
    STUCK = 2
    RUNNING = 3

class Program(object):
    def __init__(self, id, insts, inq, outq):
        self.regs = defaultdict(int)
        self.regs['p'] = id
        self.pc = 0
        self.insts = insts
        self.inq = inq
        self.outq = outq
        self.snd_count = 0

    def step(self):
        if not (0 <= self.pc < len(self.insts)):
            return State.ENDED
        op, args = self.insts[self.pc]
        if op == 'snd':
            self.outq.append(val(args[0], self.regs))
            self.pc += 1
            self.snd_count += 1
            return State.RUNNING
        elif op == 'set':
            self.regs[args[0]] = val(args[1], self.regs)
            self.pc += 1
            return State.RUNNING
        elif op == 'add':
            self.regs[args[0]] += val(args[1], self.regs)
            self.pc += 1
            return State.RUNNING
        elif op == 'mul':
            self.regs[args[0]] *= val(args[1], self.regs)
            self.pc += 1
            return State.RUNNING
        elif op == 'mod':
            self.regs[args[0]] = self.regs[args[0]] % val(args[1], self.regs)
            self.pc += 1
            return State.RUNNING
        elif op == 'rcv':
            if len(self.inq) == 0:
                return State.STUCK
            else:
                self.regs[args[0]] = self.inq.popleft()
                self.pc += 1
                return State.RUNNING
        elif op == 'jgz':
            x = val(args[0], self.regs)
            if x > 0:
                self.pc += val(args[1], self.regs)
            else:
                self.pc += 1
            return State.RUNNING

def process(prog_a, prog_b, nsteps=100000):
    for i in range(nsteps):
        res_a = prog_a.step()
        res_b = prog_b.step()

queue_a = deque()
queue_b = deque()

insts = read('input-18.txt')

prog_a = Program(0, insts, queue_a, queue_b)
prog_b = Program(1, insts, queue_b, queue_a)

process(prog_a, prog_b)

print(prog_b.snd_count)


from keras.applications import resnet50
from keras.applications import mobilenetv2
from keras.applications import mobilenet
from keras.applications import vgg19

# from keras_squeezenet import SqueezeNet
import conv.networks.get_vgg16_cifar10 as gvc
import conv.networks.gen_conv_net as gcn
# import conv.networks.MobileNet as mobilenet
import conv.networks.MobileNet_for_mobile as mobilenet_for_mobile
import conv.networks.VGG19_for_mobile as vgg19_for_mobile

import conv.networks.SqueezeNet as sqn
import conv.networks.DenseNet as dn
import conv.networks.ResNet50 as rn50

from keras_applications.imagenet_utils import decode_predictions
from keras.preprocessing.image import img_to_array
from keras.applications import imagenet_utils
from keras.engine.input_layer import Input

from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Conv2D, MaxPooling2D, BatchNormalization
from keras import optimizers
from keras.layers.core import Lambda
from keras import backend as K
from keras import regularizers
from keras.models import Model
from keras import optimizers
import keras

import numpy as np

from os import listdir
from os.path import isfile, join
import os

import matplotlib.image as mpimg
import time

# SqueezeNet: https://github.com/rcmalli/keras-squeezenet/blob/master/examples/example_keras_squeezenet.ipynb
# https://keras.io/applications/

def get_all_nets(network_name, include_top=True, num_filter=4):
	if(network_name=="ResNet50"):
		model = resnet50.ResNet101(weights='imagenet',
			include_top=include_top, input_shape=(224, 224, 3))
		# if(include_top==False):
		# 	model.pop()
	elif(network_name=="MobileNetV2"):
		model = mobilenetv2.MobileNetV2(weights='imagenet',
			include_top=include_top, input_shape=(224, 224, 3))
	elif(network_name=="MobileNet"):
		model = mobilenet.MobileNet(weights='imagenet',
			include_top=include_top,# pooling='avg',
			input_shape=(224, 224, 3))
	elif(network_name=="MobileNet_for_mobile"):
		model = mobilenet_for_mobile.MobileNet(
			include_top=include_top, weights='imagenet',
			input_shape=(224, 224, 3), num_filter=num_filter)
	elif(network_name=="VGG19"):
		model = vgg19.VGG19(weights='imagenet',
			include_top=include_top, input_shape=(224, 224, 3))
	elif(network_name=="VGG19_for_mobile"):
		model = vgg19_for_mobile.VGG19(
			include_top=include_top, weights='imagenet',
			input_shape=(224, 224, 3), num_filter=num_filter)
	elif(network_name=="SqueezeNet"):
		model = SqueezeNet(weights='imagenet',
		include_top=include_top, input_shape=(224, 224, 3))
		# if(include_top==False):
		# 	model.pop()
		# 	model.pop()
		# 	model.pop()
		# 	model.pop()

	if(include_top):
		opt = optimizers.rmsprop(lr=0.0001, decay=1e-6)

		# Let's train the model using RMSprop
		model.compile(loss='categorical_crossentropy',
		      optimizer=opt,
		      metrics=['accuracy'])
	return model

def get_nets_wo_weights(network_name, num_classes, include_top=False,
	input_shape=(32, 32, 3), num_filter=4, use_bias=False):
	if(network_name=="ResNet50"):
		model = rn50.ResNet50(include_top=include_top,
			input_shape=input_shape, weights=None,
			classes=num_classes,  num_vert_filters=num_filter)
	elif(network_name=="DenseNet121"):
		model = dn.DenseNet121(include_top=include_top,
			input_shape=input_shape, weights=None,
			classes=num_classes, num_filter=num_filter)
	elif(network_name=="MobileNetV2"):
		model = mobilenetv2.MobileNetV2(include_top=include_top,
			input_shape=input_shape, weights=None,
			classes=num_classes)
	elif(network_name=="MobileNet"):
		model = mobilenet.MobileNet(
			include_top=include_top, input_shape=input_shape, weights=None,
			classes=num_classes, num_filter=num_filter)
	elif(network_name=="MobileNet_for_mobile"):
		model = mobilenet_for_mobile.MobileNet(
			include_top=include_top, input_shape=input_shape, weights=None,
			classes=num_classes, num_filter=num_filter)
	elif(network_name=="VGG19"):
		model = vgg19.VGG19(input_shape=input_shape,
			include_top=include_top, weights=None,
			classes=num_classes)
	elif(network_name=="SqueezeNet"):
		model = sqn.SqueezeNet(input_shape=input_shape,
			include_top=include_top, weights=None, num_filter=num_filter,
			use_bias=use_bias, classes=num_classes)
	elif(network_name=="vgg"):
		model = gvc.get_conv_vert_net(x_shape=input_shape,
			num_classes=num_classes, num_vert_filters=num_filter,
    		use_bias=use_bias)
	elif(network_name=="conv"):
		model = gcn.get_conv_vert_net(input_shape=input_shape,
			num_classes=num_classes,
			num_extra_conv_layers=2, num_ver_filter=num_filter,
			use_bias=use_bias)

	if(include_top == False):
		x = model.output
		# x = keras.layers.GlobalAveragePooling2D()(x)
		x = Flatten()(x)
		x = Dense(256, activation='relu')(x)
		# x = Activation('relu')(x)
		x = Dropout(0.5)(x)
		# x = Dense(num_output)(x)
		# x = Activation('softmax')(x)
		x = keras.layers.Dense(num_classes, activation='softmax',
	                         use_bias=True, name='Logits')(x)

		full_model = Model(inputs = model.input,outputs = x)
	else:
		full_model = model
	opt = optimizers.rmsprop(lr=0.0001, decay=1e-6)

	# Let's train the model using RMSprop
	full_model.compile(loss='categorical_crossentropy',
	      optimizer=opt,
	      metrics=['accuracy'])
	return full_model

def get_box_nets(network_name, num_classes, include_top=False,
	input_shape=(32, 32, 3), num_filter=4, num_layer=4, use_bias=False):
	if(network_name=="ResNet50"):
		model = resnet50.ResNet50(include_top=include_top,
			input_shape=input_shape, weights=None)
		# if(include_top==False):
		# 	model.pop()
	elif(network_name=="DenseNet121"):
		model = dn.DenseNet121(include_top=include_top,
			input_shape=input_shape, weights=None,
			classes=num_classes, num_filter=num_filter,
			num_layer=num_layer)
	elif(network_name=="MobileNetV2"):
		model = mobilenetv2.MobileNetV2(include_top=include_top,
			input_shape=input_shape, weights=None,
			classes=num_classes)
	elif(network_name=="MobileNet"):
		model = mobilenet.MobileNet(
			include_top=include_top, input_shape=input_shape, weights=None,
			classes=num_classes, num_filter=num_filter, num_layers=num_layer)
	elif(network_name=="MobileNet_for_mobile"):
		model = mobilenet_for_mobile.MobileNet(
			include_top=include_top, input_shape=input_shape, weights=None,
			classes=num_classes, num_filter=num_filter)
	elif(network_name=="VGG19"):
		model = vgg19.VGG19(input_shape=input_shape,
			include_top=include_top, weights=None,
			classes=num_classes)
	elif(network_name=="SqueezeNet"):
		model = sqn.SqueezeNet(input_shape=input_shape,
			include_top=include_top, weights=None, num_filter=num_filter,
			use_bias=use_bias, classes=num_classes,
			num_layers=num_layer)
	elif(network_name=="vgg"):
		model = gvc.get_conv_vert_net(x_shape=input_shape,
			num_classes=num_classes, num_vert_filters=num_filter,
    		use_bias=use_bias)
	elif(network_name=="conv"):
		model = gcn.get_conv_vert_net(input_shape=input_shape,
			num_classes=num_classes,
			num_extra_conv_layers=num_layers, num_ver_filter=num_filter,
			use_bias=use_bias)

	opt = optimizers.rmsprop(lr=0.0001, decay=1e-6)

	# Let's train the model using RMSprop
	model.compile(loss='categorical_crossentropy',
	      optimizer=opt,
	      metrics=['accuracy'])
	return model

def preprocess_image(network_name, x):
	if(network_name=="ResNet50"):
		x = resnet50.preprocess_input(x)
	elif(network_name=="MobileNetV2"):
		x = mobilenetv2.preprocess_input(x)
	elif(network_name=="MobileNet"):
		x = mobilenet.preprocess_input(x)
	elif(network_name=="VGG19"):
		x = vgg19.preprocess_input(x)
	elif(network_name=="SqueezeNet"):
		x = imagenet_utils.preprocess_input(x)
	return x

def preprocess_image_fn(network_name):
	if(network_name=="ResNet50"):
		x = resnet50.preprocess_input
	elif(network_name=="MobileNetV2"):
		x = mobilenetv2.preprocess_input
	elif(network_name=="MobileNet"):
		x = mobilenet.preprocess_input
	elif(network_name=="VGG19"):
		x = vgg19.preprocess_input
	elif(network_name=="SqueezeNet"):
		x = imagenet_utils.preprocess_input
	return x

def decodepred(network_name, preds):
	if(network_name=="ResNet50"):
		preds = resnet50.decode_predictions(preds, top=3)[0]
	elif(network_name=="MobileNetV2"):
		preds = mobilenetv2.decode_predictions(preds, top=3)[0]
	elif(network_name=="MobileNet"):
		preds = mobilenet.decode_predictions(preds, top=3)[0]
	elif(network_name=="VGG19"):
		preds = vgg19.decode_predictions(preds, top=3)[0]
	elif(network_name=="SqueezeNet"):
		preds = imagenet_utils.decode_predictions(preds, top=3)[0]
	return x

def analyse_model(model):
	print("All functions ", dir(model))
	print("Summary model ", model.summary())
	print("Layer details ", dir(model.layers[2]))
	for i, layer in enumerate(model.layers):
		print("Length in each layer ", i, layer.name,
			layer.input_shape, layer.output_shape,
			len(layer.weights))
		if(len(layer.weights)):
			for j, weight in enumerate(layer.weights):
				print("Weights ", j, weight.shape)
	return

def add_classifier(base_model, num_output):

	for layer in base_model.layers:
		layer.trainable = False

	x = base_model.output
	x = keras.layers.GlobalAveragePooling2D()(x)
	# x = Dense(16, kernel_regularizer=regularizers.l2(0.01))(x)
	# x = Activation('relu')(x)
	# x = Dropout(0.5)(x)
	# x = Dense(num_output)(x)
	# x = Activation('softmax')(x)
	x = keras.layers.Dense(num_output, activation='softmax',
                         use_bias=True, name='Logits')(x)

	model = Model(inputs = base_model.input,outputs = x)
	# initiate RMSprop optimizer
	opt = optimizers.rmsprop(lr=0.0001, decay=1e-6)

	# Let's train the model using RMSprop
	model.compile(loss='binary_crossentropy',
	              optimizer=opt,
	              metrics=['accuracy'])
	return model

def get_all_prediction(image_filelist):
	prediction_list = []
	for filename in image_filelist:

		# img = image.load_img(os.path.join(imagenet_path, filename), target_size=(224, 224))
		img = image.load_img(os.path.join(imagenet_path, filename), target_size=(227, 227)) # Squeezenet
		# img1 = mpimg.imread(os.path.join(imagenet_path, filename))
		# print(img1.shape)
		x = image.img_to_array(img)
		x = np.expand_dims(x, axis=0)
		x = imagenet_utils.preprocess_input(x)

		preds = model.predict(x)
		# decode the results into a list of tuples (class, description, probability)
		# (one such list for each sample in the batch)
		print('Predicted:', filename, imagenet_utils.decode_predictions(preds, top=3)[0])
		print("Pred values ", np.argmax(preds))
		# Predicted: [(u'n02504013', u'Indian_elephant', 0.82658225), (u'n01871265', u'tusker', 0.1122357), (u'n02504458', u'African_elephant', 0.061040461)]
		prediction_list.append(preds)
	return prediction_list

if __name__ == '__main__':

	network_types_list = ["MobileNetV2"]#, "ResNet50", "MobileNetV2", "VGG19"] # , "SqueezeNet"
	for network_type in network_types_list:
		print("Network Type ", network_type)
		model = get_all_nets(network_type, include_top=True)
		analyse_model(model)
		# model = get_all_nets(network_type, include_top=False)
		# model = add_classifier(model)

	imagenet_path = "/mnt/additional/aryan/imagenet_validation_data/ILSVRC2012_img_val/"
	# http://www.image-net.org/challenges/LSVRC/2012/
	# https://cv-tricks.com/tensorflow-tutorial/keras/
	# Finding actual predictions
	# http://machinelearninguru.com/deep_learning/data_preparation/hdf5/hdf5.html

	image_filelist = [f for f in listdir(imagenet_path) if isfile(join(imagenet_path, f))]

	print("Number of files ", len(image_filelist))

	start_time = time.time()
	get_all_prediction(image_filelist[:10])
	total_time = time.time() - start_time
	print("Total prediction time ", total_time)

	print("File list ", image_filelist[:10])


#! /usr/bin/env python3
from typing import Any, Dict
import rospy
import dynamic_reconfigure.server
from example_package_with_dynamic_reconfig.cfg import ExampleDynamicParametersConfig

def dynamic_reconfigure_callback(config: Dict[str, Any], level: Any) -> Dict[str, Any]:
    return config

if __name__ == "__main__":

    try:
        rospy.init_node("package_with_dynamic_reconfig", log_level=rospy.WARN)
        dynamic_reconfigure_srv = dynamic_reconfigure.server.Server(ExampleDynamicParametersConfig,
                                                                    dynamic_reconfigure_callback)
        rospy.spin()
    except rospy.ROSInterruptException:
        rospy.loginfo("Shutting down.")


import logging
from kubernetes import client
from kubernetes.client.models.v1_resource_requirements import V1ResourceRequirements
from kubeflow.fairing.constants import constants

logger = logging.getLogger(__name__)

def get_resource_mutator(cpu=None, memory=None, gpu=None, gpu_vendor='nvidia'):
    """The mutator for getting the resource setting for pod spec.

    The useful example:
    https://github.com/kubeflow/fairing/blob/master/examples/train_job_api/main.ipynb

    :param cpu: Limits and requests for CPU resources (Default value = None)
    :param memory: Limits and requests for memory (Default value = None)
    :param gpu: Limits for GPU (Default value = None)
    :param gpu_vendor: Default value is 'nvidia', also can be set to 'amd'.
    :returns: object: The mutator function for setting cpu and memory in pod spec.

    """
    def _resource_mutator(kube_manager, pod_spec, namespace): #pylint:disable=unused-argument
        if cpu is None and memory is None and gpu is None:
            return
        if pod_spec.containers and len(pod_spec.containers) >= 1:
            # All cloud providers specify their instace memory in GB
            # so it is peferable for user to specify memory in GB
            # and we convert it to Gi that K8s needs
            limits = {}
            if cpu:
                limits['cpu'] = cpu
            if memory:
                memory_gib = "{}Gi".format(round(memory/1.073741824, 2))
                limits['memory'] = memory_gib
            if gpu:
                limits[gpu_vendor + '.com/gpu'] = gpu
            if pod_spec.containers[0].resources:
                if pod_spec.containers[0].resources.limits:
                    pod_spec.containers[0].resources.limits = {}
                for k, v in limits.items():
                    pod_spec.containers[0].resources.limits[k] = v
            else:
                pod_spec.containers[0].resources = V1ResourceRequirements(limits=limits)
    return _resource_mutator

def mounting_pvc(pvc_name, pvc_mount_path=constants.PVC_DEFAULT_MOUNT_PATH):
    """The function has been deprecated, please use `volume_mounts`.

    """
    logger.warning("The function mounting_pvc has been deprecated, \
                    please use `volume_mounts`")

    return volume_mounts('pvc', pvc_name, mount_path=pvc_mount_path)

def volume_mounts(volume_type, volume_name, mount_path, sub_path=None):
    """The function for pod_spec_mutators to mount volumes.

    :param volume_type: support type: secret, config_map and pvc
    :param name: The name of volume
    :param mount_path: Path for the volume mounts to.
    :param sub_path: SubPath for the volume mounts to (Default value = None).
    :returns: object: function for mount the pvc to pods.

    """
    mount_name = str(constants.DEFAULT_VOLUME_NAME) + volume_name

    def _volume_mounts(kube_manager, pod_spec, namespace): #pylint:disable=unused-argument
        volume_mount = client.V1VolumeMount(
            name=mount_name, mount_path=mount_path, sub_path=sub_path)
        if pod_spec.containers[0].volume_mounts:
            pod_spec.containers[0].volume_mounts.append(volume_mount)
        else:
            pod_spec.containers[0].volume_mounts = [volume_mount]

        if volume_type == 'pvc':
            volume = client.V1Volume(
                name=mount_name,
                persistent_volume_claim=client.V1PersistentVolumeClaimVolumeSource(
                    claim_name=volume_name))
        elif volume_type == 'secret':
            volume = client.V1Volume(
                name=mount_name,
                secret=client.V1SecretVolumeSource(secret_name=volume_name))
        elif volume_type == 'config_map':
            volume = client.V1Volume(
                name=mount_name,
                config_map=client.V1ConfigMapVolumeSource(name=volume_name))
        else:
            raise RuntimeError("Unsupport type %s" % volume_type)

        if pod_spec.volumes:
            pod_spec.volumes.append(volume)
        else:
            pod_spec.volumes = [volume]
    return _volume_mounts

def add_env(env_vars):
    """The function for pod_spec_mutators to add custom environment vars.

    :param vars: dict of custom environment vars.
    :returns: object: function for add environment vars to pods.

    """
    def _add_env(kube_manager, pod_spec, namespace): #pylint:disable=unused-argument
        env_list = []
        for env_name, env_value in env_vars.items():
            env_list.append(client.V1EnvVar(name=env_name, value=env_value))

        if pod_spec.containers and len(pod_spec.containers) >= 1:
            if pod_spec.containers[0].env:
                pod_spec.containers[0].env.extend(env_list)
            else:
                pod_spec.containers[0].env = env_list
    return _add_env

def get_node_selector(node_selector):
    """This function for pod_spec_mutators to designate node selector.

    :param node_selector: dict of selection constraint
    :return: obejct: The mutator fucntion for setting node selector

    """

    def _node_selector(kube_master, pod_spec, namespace): #pylint:disable=unused-argument
        if node_selector is None:
            return
        if pod_spec.containers and len(pod_spec.containers) >= 1:
            pod_spec.node_selector = node_selector

    return _node_selector


from django.conf.urls.defaults import patterns, include, url
from django.contrib import admin

admin.autodiscover()

urlpatterns = patterns('reviews.views',
    url(r'^$', 'home', name='home'),
    url(r'^courses/$', 'courses', name='courses'),
    url(r'^courses/find/$', 'find_course', name='find_course'),
    url(r'^courses/search/$', 'search', name='search'),
    url(r'^courses/add/$', 'add_course', name='add_course'),

    url(r'^courses/(?P<course_id>\d+)/$', 'course', name="course"),
    url(r'^courses/(?P<course_id>\d+)/choose_class/$', 'choose_class_to_review', name='choose_class'),
    url(r'^courses/(?P<class_id>\d+)/review/$', 'review_course', name="review_course"),
    url(r'^courses/(?P<class_id>\d+)/review/(?P<review_id>\d+)/edit/$', 'review_course', name="edit_review"),
    url(r'^courses/(?P<course_id>\d+)/edit/$', 'edit_course', name="edit_course"),

    url(r'^depts/$', 'departments', name='departments'),
    url(r'^depts/(?P<dept_abb>.+)/$', 'department', name='department'),

    url(r'^instructors/$', 'instructors', name='instructors'),
    url(r'^instructors/add/$', 'add_instructor', name='add_instructor'),
    url(r'^instructors/(?P<instructor_id>\d+)/$', 'instructor', name='instructor'),

    url(r'^tags/$', 'tags', name='tags'),
    url(r'^tags/(?P<tag_name>\w+)/$', 'tag', name='tag'),

    url(r'^allreviews/$', 'reviews', name='reviews'),

    url(r'^students/$', 'students', name='students'),
    url(r'^students/(?P<student_id>\d+)/$', 'student', name='student'),

    url(r'^login/$', 'login', name='login'),
    url(r'^logout/$', 'logout_page', name='logout'),
)

urlpatterns += patterns('',
    url(r'^admin/', include(admin.site.urls)),
)


import sqlite3
from sqlite3 import Error

class Data():
    __error = None
    __result = None

    def __init__(self, db):
        try:
            self.con = sqlite3.connect(db, check_same_thread = False)
            self.cur = self.con.cursor()
        except Error as e:
            print(e)

    def clean_db(self):
        try:
            self.cur.execute("DELETE FROM  file;")
            self.con.commit()
            self.cur.execute("DELETE FROM SQLITE_SEQUENCE WHERE name='file';")
            self.con.commit()

            self.cur.execute("DELETE FROM  directory;")
            self.con.commit()
            self.cur.execute("DELETE FROM SQLITE_SEQUENCE WHERE name='directory';")
            self.con.commit()
        except Error as e:
            self.__error = e
            return self.__error

    def create_tables(self):
        tb_directory ='CREATE TABLE "directory" ("id_directory"	INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, "name"	TEXT)'
        tb_evidence ='CREATE TABLE "evidence" ("case_number" INTEGER, "examiner_name" TEXT, "description" TEXT, "note" TEXT)'
        tb_pull_log ='CREATE TABLE "pull_log" ("id_log" INTEGER PRIMARY KEY AUTOINCREMENT, "file" TEXT, "from" TEXT, "to" TEXT, "md5_source" TEXT, "sha1_source" TEXT, "date" TEXT )'
        tb_file = 'CREATE TABLE "file" ("id_file" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, "id_directory"	INTEGER NOT NULL, "name" TEXT, "permision" TEXT, "date" TEXT, "Size" REAL)'
        try:
            self.cur.execute(tb_directory)
            self.cur.execute(tb_file)
            self.cur.execute(tb_evidence)
            self.cur.execute(tb_pull_log)
        except Error as e:
            self.__error = e
            return self.__error

    def insert_log_pull(self, file, from_path, to_path, md5_source, sha1_source, date):
        try:
            query = "INSERT INTO  pull_log (`file`,`from`, `to`, md5_source, sha1_source,`date` )VALUES ('%s','%s','%s','%s','%s','%s')"%(file, from_path, to_path, md5_source, sha1_source, date)
            self.cur.execute(query)
            self.con.commit()
        except Error as e:
            self.__error = e
            return self.__error

    def insert_evidence(self, case_number, examiner_name, description, note):
        try:
            query = "INSERT INTO evidence (case_number, examiner_name, description, note) VALUES ('%s','%s','%s','%s')"%(case_number, examiner_name, description, note)
            self.cur.execute(query)
            self.con.commit()
        except Error as e:
            self.__error = e
            return self.__error

    def select_evidence(self):
        try:
            self.cur.execute("SELECT * from evidence")
            self.__result = self.cur.fetchone()
            return self.__result
        except Error as e:
            print(e)

    def select_pull_log(self):
        try:
            self.cur.execute("SELECT * from pull_log")
            self.__result = self.cur.fetchall()
            return self.__result
        except Error as e:
            print(e)

    def select_all_data(self, order):
        try:
            select = "SELECT directory.name as loc, directory.id_directory, file.name as file, file.permision, file.Size, file.date"
            frm = " FROM directory, file"
            where = " WHERE directory.id_directory=file.id_directory ORDER BY "+order+" DESC"
            self.cur.execute(select+frm+where)
            self.__result = self.cur.fetchall()
            return self.__result
        except Exception as e:
            self.__error = e.args[0]
            return self.__error

    def select_by_extention(self, ext, order):
        try:
            select = "SELECT directory.name as loc, directory.id_directory, file.name as file, file.permision, file.Size, file.date"
            frm = " FROM directory, file"
            where = " WHERE directory.id_directory=file.id_directory and file.name like'%"+ext+"%' ORDER BY "+order+" DESC"
            self.cur.execute(select+frm+where)
            self.__result = self.cur.fetchall()
            return self.__result
        except Exception as e:
            self.__error = e.args[0]
            return self.__error

    def insert_dir(self, dir):
        try:
            self.cur.execute('INSERT INTO `directory` (`name`) VALUES ("%s")' % (dir))
            self.con.commit()
        except Exception as e :
            self.__error=e.args[0]
            return  self.__error

    def insert_sub_dir(self, id_dir, name):
        try:
            self.cur.execute('INSERT INTO `sub_directory` (`id_directory`,`name`) VALUES (%s,"%s")' % (id_dir, name))
            self.con.commit()
        except Exception as e :
            self.__error=e.args[0]
            return self.__error

    def insert_file(self, id_dir, name, permision, date, size):
        try:
            self.cur.execute('INSERT INTO `file` (`id_directory`,`name`, `permision`, `date`, `size`) VALUES (%s,"%s", "%s", "%s", "%s")' % (id_dir, name, permision, date, size))
            self.con.commit()
        except Exception as e :
            self.__error=e.args[0]
            return self.__error

    def select_name_by_id_dir(self, id_dir):
        try:
            query = 'SELECT `name` FROM sub_directory WHERE id_directory =%s'%(id_dir)
            self.cur.execute(query)
            self.__result = self.cur.fetchall()
            return self.__result
        except Exception as e:
            self.__error = e.args[0]
            return self.__error

    def select_name_dir_subDir(self, id_dir):
        try:
            query = 'SELECT directory.`name`, sub_directory.name FROM sub_directory, `directory` WHERE sub_directory.id_directory=directory.id_directory and directory.id_directory=%s' %(id_dir)
            self.cur.execute(query)
            self.__result = self.cur.fetchall()
            return self.__result
        except Exception as e:
            self.__error = e.args[0]
            return self.__error

    def select_id_dir_by_name(self, name):
        try:
            query = 'SELECT `id_directory` FROM directory WHERE name ="%s"'%(name)
            self.cur.execute(query)
            self.__result = self.cur.fetchall()
            return self.__result
        except Exception as e:
            self.__error = e.args[0]
            return self.__error

    def search(self, key, order):
        try:
            select = "SELECT directory.name as loc, directory.id_directory, file.name as file, file.permision, file.Size, file.date"
            frm = " FROM directory, file"
            where = " WHERE directory.id_directory=file.id_directory AND file.name like'%"+key+"%'"+" OR file.date like'%"+key+"%'"+" OR directory.name like'%"+key+"%' GROUP BY id_file"+" ORDER BY "+order+" DESC"
            self.cur.execute(select+frm+where)
            self.__result = self.cur.fetchall()
            return self.__result
        except Exception as e:
            self.__error = e.args[0]
            return self.__error


import json
import boto3
from smart_open import smart_open, codecs
from ConfigParser import ConfigParser
import psycopg2

def publish_message(producerInstance, topic_name, key, value):
    "Function to send messages to the specific topic"
    try:
        producerInstance.produce(topic_name,key=key,value=value)
        producerInstance.flush()
        print('Message published successfully.')
    except Exception as ex:
        print('Exception in publishing message')
        print(str(ex))

def config(filename='database.ini', section='postgresql'):
    # create a parser
    parser = ConfigParser()
    # read config file
    parser.read(filename)

    # get section, default to postgresql
    db = {}
    if parser.has_section(section):
        params = parser.items(section)
        for param in params:
            db[param[0]] = param[1]
    else:
        raise Exception('Section {0} not found in the {1} file'.format(section, filename))

    return db

def insert_data(finaldict,tablename):
    conn = None
    try:
        params = config()
        # connect to the PostgreSQL database
        conn = psycopg2.connect(**params)
        # create a new cursor
        curs = conn.cursor()
        query = curs.mogrify("INSERT INTO {} ({}) VALUES {}".format(
            tablename,
            ', '.join(finaldict[0].keys()),
            ', '.join(["%s"] * len(finaldict))
        ), [tuple(v.values()) for v in finaldict])
        print(query)
        curs.execute(query)
        conn.commit()
        curs.close()
    except (Exception, psycopg2.DatabaseError) as error:
        print(error)
    finally:
        if conn is not None:
            conn.close()

def get_event_files(tableprefix):
    return list(my_bucket.objects.filter(Prefix=tableprefix))

client = boto3.client('s3')
resource = boto3.resource('s3')
my_bucket = resource.Bucket('gdelt-sample-data')
events_files = get_event_files("events")
gkg_files = get_event_files("gkg")
mentions_files = get_event_files("mentions")
gkg_obj = codecs.getreader('utf-8')(gkg_files[0].get()['Body'])
event_obj = codecs.getreader('utf-8')(events_files[0].get()['Body'])
mention_obj = codecs.getreader('utf-8')(mentions_files[0].get()['Body'])
events_columns = ['GlobalEventID', 'Day', 'MonthYear', 'Year', 'FractionDate',
                  'Actor1Code', 'Actor1Name', 'Actor1CountryCode',
                  'Actor1KnownGroupCode', 'Actor1EthnicCode',
                  'Actor1Religion1Code', 'Actor1Religion2Code',
                  'Actor1Type1Code', 'Actor1Type2Code', 'Actor1Type3Code',
                  'Actor2Code', 'Actor2Name', 'Actor2CountryCode',
                  'Actor2KnownGroupCode', 'Actor2EthnicCode',
                  'Actor2Religion1Code', 'Actor2Religion2Code',
                  'Actor2Type1Code', 'Actor2Type2Code', 'Actor2Type3Code',
                  'IsRootEvent', 'EventCode', 'EventBaseCode',
                  'EventRootCode', 'QuadClass', 'GoldsteinScale',
                  'NumMentions', 'NumSources', 'NumArticles', 'AvgTone',
                  'Actor1Geo_Type', 'Actor1Geo_Fullname',
                  'Actor1Geo_CountryCode', 'Actor1Geo_ADM1Code',
                  'Actor1Geo_ADM2Code', 'Actor1Geo_Lat', 'Actor1Geo_Long',
                  'Actor1Geo_FeatureID', 'Actor2Geo_Type',
                  'Actor2Geo_Fullname', 'Actor2Geo_CountryCode',
                  'Actor2Geo_ADM1Code', 'Actor2Geo_ADM2Code',
                  'Actor2Geo_Lat', 'Actor2Geo_Long', 'Actor2Geo_FeatureID',
                  'ActionGeo_Type', 'ActionGeo_Fullname',
                  'ActionGeo_CountryCode', 'ActionGeo_ADM1Code',
                  'ActionGeo_ADM2Code', 'ActionGeo_Lat', 'ActionGeo_Long',
                  'ActionGeo_FeatureID', 'DATEADDED', 'SOURCEURL']

gkg = ["recordid","date" , "srccollectionidentifier","srccommonname","documentid","counts","countsv1","themes","enhancedthemes",
	"locations", "enhancedlocation","persons","enhancedpersons","organizations","enhancedorganizations","tone","enhanceddates",
	"gcam","sharingimage","relatedimages", "socialimageembeds", "socialvideoembeds", "quotations", "allnames", "amounts","translationinfo",
	"extrasxml"]

mentions = ["GlobalEventID","EventTimeDate","MentionTimeDate","MentionType","MentionSourceName","MentionIdentifier","SentenceID",
            "Actor1CharOffset","Actor2CharOffset","ActionCharOffset","InRawText","Confidence","MentionDocLen","MentionDocTone"]

gkg_finaldict=[]
for record in gkg_obj:
    features = record.strip().split("\t")
    if(len(features)==27):
        tmpDict = dict()
        tmpDict = dict({gkg[i]:features[i].encode("utf-8") for i in range(len(gkg))})
        gkg_finaldict.append(tmpDict)

for i in range(0,len(gkg_finaldict),1000):
    insert_data(gkg_finaldict[i:i+1000],"public.gkg")

event_finaldict=[]
for record in event_obj:
    features = record.strip().split("\t")
    if(len(features)==61):
        tmpDict = dict()
        tmpDict = dict({events_columns[i]: features[i].encode("utf-8") for i in range(len(events_columns))})
        event_finaldict.append(tmpDict)

for i in range(0,len(event_finaldict),1000):
    insert_data(event_finaldict[i:i+1000],"public.events")

mentions_finaldict=[]
for record in mention_obj:
    features = record.strip().split("\t")
    print(record)
    if(len(features)==14):
        tmpDict = dict()
        tmpDict = dict({mentions[i]: features[i].encode("utf-8") for i in range(len(mentions))})
        mentions_finaldict.append(tmpDict)

for i in range(0,len(mentions_finaldict),1000):
    insert_data(mentions_finaldict[i:i+1000],"public.mentions")


from datetime import datetime, time
import sys
from time import sleep

import datefunc

def choose_date(now):
    datefunc.clear_terminal()
    option = input("Choose counter:\n 1 - time to pay,\n 2 - time to vacation,\n 3 - time to end of working day \n")
    datefunc.clear_terminal()\

    if option == '1' or option == 1:

        return datefunc.time_to_pay(now)

    if option == '2' or option == 2:

        return datefunc.time_to_vacation()

    if option == '3' or option == 3:

       return datefunc.time_end_workingday()

    else:
        print('fuck yourself')
        sys.exit()

def main():
    now = datetime.now()
    # print(now.today().weekday())
    req = choose_date(now)

    while req>now:
        print("%dd %dh %dm %ds" % datefunc.daysHoursMinutesSecondsFromSeconds(datefunc.dateDiffInSeconds(now, req)))
        datefunc.clear_terminal()
        now = datetime.now()

    print("Thank you")

if __name__ == "__main__":
    main()


from Folder_de_Testes.base import Fox_HEIGHT, Fox_WIDTH
import pygame
import random

#Parametros gerais

WIDTH = 880
HEIGHT = 400
gravity = 1

def randon_sizes_for_walls(xpos):
    protection = 200
    altura = random.randint(200, 400)
    wall = Wall(False, xpos, altura)
    inversal_wall = Wall(True, xpos,HEIGHT - altura - protection)
    return (wall,  inversal_wall)

class Fox(pygame.sprite.Sprite):

    def __init__(self):

        pygame.sprite.Sprite.__init__(self)
        count_fox = 0
        Fox_WIDTH = 170
        Fox_HEIGHT = 100
        self.gravity = 1
        Fox1 = pygame.image.load('Folder_de_Testes/assets/img/raposa 1.png').convert_alpha()
        Fox1 = pygame.transform.scale(Fox1, (Fox_WIDTH, Fox_HEIGHT))
        Fox2 = pygame.image.load('Folder_de_Testes/assets/img/raposa2.png').convert_alpha()
        Fox2 = pygame.transform.scale(Fox2, (Fox_WIDTH, Fox_HEIGHT))
        Fox3 = pygame.image.load('Folder_de_Testes/assets/img/raposa 3.png').convert_alpha()
        Fox3 = pygame.transform.scale(Fox3, (Fox_WIDTH, Fox_HEIGHT))
        self.flying_one = pygame.image.load('Folder_de_Testes/assets/img/raposafinal.png').convert_alpha()
        self.flying_one = pygame.transform.scale(self.flying_one, (100, 100))

        self.images = [Fox1,Fox2,Fox3]

        self.count_fox = count_fox
        self.image = Fox1

        self.rect = self.image.get_rect()
        self.rect.centerx = WIDTH / 4
        self.rect.bottom = HEIGHT - 100
        self.speedy = 1

        self.now_on_windon = 0

        self.speed_modifier = 0.0

    def update(self):

        self.rect.y += self.speedy

        self.speedy += self.gravity + 0.1 * (-self.speedy)

        self.mask = pygame.mask.from_surface(self.image)

        self.count_fox += 1

        #print(self.speed_modifier)

        if self.speed_modifier > -12:
            self.speed_modifier -= 0.0024

        if self.count_fox >= 10 and  self.rect.bottom > HEIGHT:

            self.now_on_windon = (self.now_on_windon + 1) % 3
            self.image = self.images[self.now_on_windon]
            self.count_fox = 0

        elif self.speedy <0 :
            self.image = self.flying_one
            #print(self.speedy)
            #print(self.count_fox)

        # Mantem dentro da tela
        if self.rect.bottom > HEIGHT:

            self.rect.bottom = HEIGHT
            #self.speedy = 1
            #game = False
        if self.rect.top < 0:

           self.rect.top = 0

    def pulo(self):

        self.speedy = -16 + self.speed_modifier

fox_group = pygame.sprite.Group()
fox = Fox()
fox_group.add(fox)

class Wall_meteor_fisic(pygame.sprite.Sprite):

    def __init__(self, img):
        # Construtor da classe mãe (Sprite).
        pygame.sprite.Sprite.__init__(self)

        Wall_WIDTH = 50
        Wall_HEIGHT = random.randint(50, 250)
        self.image = img
        self.rect = self.image.get_rect()
        self.rect.x = (WIDTH-Wall_WIDTH)
        self.rect.y = random.randint(10,300)
        self.speedx = random.randint(-5, -3)
        Wall_HEIGHT = random.randint(50, 250)

    def update(self):
        # Atualizando a posição do meteoro
        self.rect.x += self.speedx
        Wall_WIDTH = 50
        # Se o meteoro passar do final da tela, volta para cima e sorteia
        # novas posições e velocidades
        if self.rect.top > HEIGHT or self.rect.right < 0 or self.rect.left > WIDTH:
            self.rect.x = (WIDTH-Wall_WIDTH)
            self.rect.y = random.randint(10,300)
            self.speedx = random.randint(-5, -3)

class Invible_wall:

    def __init__(self,img):

        pygame.sprite.Sprite.__init__(self)

        self.image = img
        self.rect = self.image.get_rect()

class Wall(pygame.sprite.Sprite):
    def __init__(self, inversal,posx, posy):
        # Construtor da classe mãe (Sprite).
        pygame.sprite.Sprite.__init__(self)

        wall_HEIGHT = 80
        wall_WIDTH = 80

        self.image = pygame.image.load('Folder_de_Testes/assets/img/Tree.png').convert_alpha()
        self.image = pygame.transform.scale(self.image, (wall_WIDTH, wall_HEIGHT))
        self.rect = self.image.get_rect()
        self.rect[0] = posx

        if inversal:

            self.image = pygame.transaform.flip(self.image,False, True)
            self.rect[1] = (self.rect[3] - posy)
        else:
            self.rect[1] = HEIGHT - posy

        self.mask = pygame.mask.from_surface(self.image)

        self.speedx = random.randint(-5, -3)

    def update(self):
        self.rect[0] += self.speedx

class Coin(pygame.sprite.Sprite):
    def __init__(self):
        # Construtor da classe mãe (Sprite).
        pygame.sprite.Sprite.__init__(self)
        coin_HEIGHT = 50
        coin_WIDTH = 50
        self.image = pygame.image.load('Folder_de_Testes/assets/img/coin.png').convert_alpha()
        self.mask = pygame.mask.from_surface(self.image)
        self.rect = self.image.get_rect()
        self.rect.x = (WIDTH-coin_WIDTH)
        self.rect.y = (HEIGHT  - coin_HEIGHT)
        self.speedx = random.randint(-5, -3)
        METEOR_HEIGHT = random.randint(50, 250)

    def update(self):
        # Atualizando a posição do meteoro
        METEOR_HEIGHT = random.randint(50, 250)
        self.rect.x += self.speedx
        coin_WIDTH = 50
        # Se o meteoro passar do final da tela, volta para cima e sorteia
        # novas posições e velocidades
        if self.rect.top > HEIGHT or self.rect.right < 0 or self.rect.left > WIDTH:
            self.rect.x = (WIDTH-coin_WIDTH)
            self.rect.y = (HEIGHT - METEOR_HEIGHT)
            self.speedx = random.randint(-5, -3)

class Predator(pygame.sprite.Sprite):
    def __init__(self):
        # Construtor da classe mãe (Sprite).
        pygame.sprite.Sprite.__init__(self)
        coin_HEIGHT = 50
        coin_WIDTH = 50
        self.image = pygame.image.load('Folder_de_Testes/assets/img/piranha.png').convert_alpha()
        self.image = pygame.transform.scale(self.image, (coin_WIDTH, coin_HEIGHT))
        self.mask = pygame.mask.from_surface(self.image)
        self.rect = self.image.get_rect()
        self.rect.x = (WIDTH-coin_WIDTH)
        self.rect.y = random.randint(10, 300)
        self.speedx = random.randint(-5, -3)
        METEOR_HEIGHT = random.randint(50, 250)

    def update(self):
        # Atualizando a posição do meteoro
        METEOR_HEIGHT = random.randint(50, 250)
        self.rect.x += self.speedx
        coin_WIDTH = 50
        # Se o meteoro passar do final da tela, volta para cima e sorteia
        # novas posições e velocidades
        if self.rect.top > HEIGHT or self.rect.right < 0 or self.rect.left > WIDTH:
            self.rect.x = (WIDTH-coin_WIDTH)
            self.rect.y = (HEIGHT - METEOR_HEIGHT)
            self.speedx = random.randint(-5, -3)


## import libraries

from tkinter import *
from gtts import gTTS
from playsound import playsound

################### Initialized window####################

root = Tk()
root.geometry('350x300')
root.resizable(0,0)
root.config(bg = 'light yellow')
root.title('DataFlair - TEXT_TO_SPEECH')

##heading
Label(root, text = 'HELIGA TEKST' , font='arial 20 bold' , bg ='white smoke').pack()
Label(root, text ='DataFlair' , font ='arial 15 bold', bg = 'blue').pack(side = BOTTOM)

#label
Label(root, text ='Sisesta Tekst', font ='arial 15 bold', bg ='white').place(x=20,y=60)

##text variable
Msg = StringVar()

#Entry
entry_field = Entry(root,textvariable =Msg, width ='50')
entry_field.place(x=20 , y=100)

###################define function##############################

def Tekst():
    Message = entry_field.get()
    speech = gTTS(text = Message, lang ='et', slow = True)
    speech.save('DataFlair.mp3')
    playsound('DataFlair.mp3')

def Exit():
    root.destroy()

def Reset():
    Msg.set("")

#Button
Button(root, text = "ESITA" , font = 'arial 15 bold', command = Tekst, bg = 'light blue', width =6).place(x=25, y=140)
Button(root,text = 'VÄLJU',font = 'arial 15 bold' , command = Exit, bg = 'green').place(x=100,y=140)
Button(root, text = 'UUESTI', font='arial 15 bold', command = Reset, bg = 'yellow' ).place(x=175 , y =140)

#infinite loop to run program
root.mainloop()


from langchain.document_loaders import WebBaseLoader
from langchain.document_loaders import PyPDFLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.embeddings import HuggingFaceEmbeddings
from langchain.vectorstores import Chroma
import os
from langchain.chat_models import JinaChat
from langchain.prompts.chat import (
    ChatPromptTemplate,
    SystemMessagePromptTemplate,
    HumanMessagePromptTemplate,
)
from langchain.chains import RetrievalQAWithSourcesChain
from langchain.llms import AI21

# create a new instance of chatbot and saves it as a JSON file
def createNewBot(name, fileType, path, url):
	loader = None

	if fileType == 'web':
		loader = WebBaseLoader(url)
	elif fileType == 'doc':
		loader = PyPDFLoader(path)

	data = loader.load()

	text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100)
	all_splits = text_splitter.split_documents(data)

	embeddings = HuggingFaceEmbeddings()

	persistentDir = "bots/" + name + "/vectorstore/"
	vectorstore = Chroma.from_documents(documents=all_splits, embedding=embeddings, persist_directory=persistentDir)
	# print(vectorstore)
	# jina_api_key = os.environ['JINA_API_KEY']
	# chat = JinaChat(temperature=0, jinachat_api_key=jina_api_key)

	# chat = ChatAnyscale(model_name='meta-llama/Llama-2-7b-chat-hf', temperature=1.0, anyscale_api_key=os.environ["ANYSCALE_API_KEY"])
	chat = AI21(ai21_api_key=os.getenv("AI21_API_KEY"))
	# memory = ConversationSummaryMemory(llm=chat,memory_key="chat_history",return_messages=True)

	retriever = vectorstore.as_retriever()

	template = (
    r"""You are a helpful English speaking assistant. Use the following pieces of context to answer the users question. If you cannot find the answer from the pieces of context, just say that you don't know, don't try to make up an answer.
	----------------
	{context}
	"""
	)
	system_message_prompt = SystemMessagePromptTemplate.from_template(template)
	human_template = "{question}"
	human_message_prompt = HumanMessagePromptTemplate.from_template(human_template)

	chat_prompt = ChatPromptTemplate.from_messages(
    [system_message_prompt, human_message_prompt]
	)

	# finalChain = ConversationalRetrievalChain.from_llm(chat, retriever=retriever, memory = memory, combine_docs_chain_kwargs={'prompt': chat_prompt})
	finalChain = RetrievalQAWithSourcesChain.from_chain_type(chat, retriever=retriever)

	# print(finalChain.retriever)
	# SAVING DOESNT WORK OUT BECAUSE LANGCHAIN HAS YET TO SUPPORT THIS
	chainSaveFolder = "bots/" + name + '/'

	botSavePath = chainSaveFolder + name + '.json'
	finalChain.save(botSavePath)

	# retrieverSavePath = chainSaveFolder + name + '_retriever.json'
	# with open(retrieverSavePath, "w") as f:
	# 	# json.dump(finalChain.retriever.to_json(), f, indent = 2)
	# 	json.dump(vectorstore, f, indent = 2)

	return finalChain


from datetime import datetime, timezone, timedelta

def stem(label: str, blacklist: list):
    '''
    This function stems a given event label.

    Inputs:
    - label: single label to stem
    - blacklist: list of terms, that should be excluded from the label

    Return: stemmed label
    '''
    parts = label.split(' ')
    parts = list(filter(lambda x: x not in blacklist, parts))
    return ' '.join(parts)

def time_dif(x: tuple, interval: str):
    '''
    Calculate the differences between two points in time.

    Inputs:
        - x: tuple of two datetime objects
        - interval: indicator of the return type; accepted values: 'd', 'h', 's'

    Return: interval in days, hours or seconds
    '''
    res = time_wrap(x[0], x[1])

    days = res.days
    hours = res.seconds//60//60
    seconds = res.seconds

    if interval is 'd':
        return days
    elif interval is 'h':
        return hours + (days * 24)
    elif interval is 's':
        return seconds + (days * 24 * 60 * 60)

def number_of_non_workdays(start, end):
    '''
    Compute the number of days between two points in time, excluding weekends.

    Input:
        - start: datetime object
        - end: datetime object

    Return:
        int: number of days
    '''
    # 0: Monday
    days = []

    while(start <= end):
        days.append(start.weekday())
        start = start + timedelta(days=1)

    days = len(list(filter(lambda x: x > 4, days)))

    return days

def time_wrap(start: datetime, end: datetime, s_hour = 8, e_hour = 18):
    '''
    Return the temporal difference between two points in time, adjusted to a given workschedule.

    Input:
        - start: datetime object
        - end: datetime object
        - s_hour: start of workschedule
        - e_hour: end of workschedule

    Return:
        - timedelta in seconds
    '''

    # worktime after start event
    e_time = datetime(start.year, start.month, start.day, e_hour)
    start = start.replace(tzinfo=None)
    t1 = (e_time - start).seconds

    # worktime before end event
    end = end.replace(tzinfo=None)
    s_time = datetime(start.year, start.month, start.day, s_hour)
    t3 = (end - s_time).seconds

    # calculate days between start and end exclusive non-working days
    days_total = (end - start).days
    non_workingdays = number_of_non_workdays(start, end)

    working_days = days_total - non_workingdays
    if working_days > 1:
        working_days -= 1 # consider only complete day in between
        total_hours_between = (e_hour - s_hour) * working_days

        # convert into seconds
        t2 = total_hours_between * 60 * 60
    else:
        # in this case, there is no full working day between start and end
        t2 = 0

    total_dif = t1 + t2 + t3

    return timedelta(seconds=total_dif)


from multiprocessing import Value, Queue, Process
from config import config
from spider.HtmlCrawl import IpCrawl
from usable.usable import usable
from db.db_select import save_data

def startProxyCrawl(queue,db_proxy_num):
    crawl = IpCrawl(queue,db_proxy_num)
    crawl.run()

def validator(queue1,queue2):
    pass

if __name__ == "__main__":
    DB_PROXY_NUM = Value('i', 0)
    q1 = Queue(maxsize=config.TASK_QUEUE_SIZE)
    q2 = Queue()
    p1 = Process(target=startProxyCrawl, args=(q1, DB_PROXY_NUM))
    p2 = Process(target=usable, args=(q1, q2))
    p3 = Process(target=save_data, args=(q2, DB_PROXY_NUM))
    p1.start()
    p2.start()
    p3.start()
    p1.join()
    p2.join()
    p3.join()