from __future__ import annotations

from asyncio import create_subprocess_shell
from asyncio.subprocess import PIPE
import json
from typing import Any, Dict, List

from songbird.helpers import ytdl
from songbird.songbird import Metadata

async def get_playlist(playlist: str) -> List[YoutubeVideo]:
    cmd = f"yt-dlp '{playlist}' --flat-playlist --dump-single-json"

    proc = await create_subprocess_shell(cmd, stdout=PIPE, stderr=PIPE)
    stdout, _ = await proc.communicate()

    out: List[YoutubeVideo] = []
    if stdout:
        out.extend(
            YoutubeVideo(entry) for entry in json.loads(stdout)["entries"]
        )

    return out

class YoutubeVideo:
    def __init__(self, video: Dict[str, Any]) -> None:
        self.metadata = Metadata(
            channel=video.get("channel"),
            duration=video.get("duration"),
            source_url=video.get("url"),
            title=video.get("title"),
            thumbnail=video.get("thumbnails", [{"url": None}])[0]["url"]
        )

    def __await__(self):
        return ytdl(self.metadata.source_url).__await__()


import copy
import csv
import subprocess
import numpy as np
from scipy.sparse.csgraph import minimum_spanning_tree
import matplotlib.pyplot as plt

class Graph:

    def __init__(self, points, size):
        self._matrix = np.zeros((size, size))
        self._size = len(points)

        for i, a in enumerate(points):
            for j, b in enumerate(points):
                if self._matrix[i][j] == 0.0:
                    dist = np.sqrt((a[0] - b[0]) ** 2 + (a[1] - b[1]) ** 2)
                    self._matrix[i][j] = dist
                    self._matrix[j][i] = dist

    def size(self):
        return self._size

    def get_distance(self, i, j):
        return self._matrix[i][j]

    def mst(self):
        temp_matrix = self._matrix.copy()

        for i in range(self._size):
            for j in range(self._size):
                if i >= j:
                    temp_matrix[i][j] = 0

        return minimum_spanning_tree(temp_matrix).toarray()

    def remove_node(self, i):
        self._matrix = np.delete(self._matrix, i, 1)
        self._matrix = np.delete(self._matrix, i, 0)
        self._size -= 1

def one_tree(graph):
    short_graph = copy.deepcopy(graph)
    short_graph.remove_node(0)

    cost = np.sum(np.sum(short_graph.mst()))

    min_cost = graph.get_distance(0, 1)
    second_min = min_cost

    for j in range(1, graph.size()):
        dist = graph.get_distance(0, j)
        if dist < min_cost:
            second_min = min_cost
            min_cost = dist

    cost += min_cost + second_min

    return cost

def get_lower_bound(file_name):
    uu = []
    with open(file_name) as file:
        i, spam_reader = 0, csv.reader(file, delimiter=' ')
        _ = next(spam_reader)
        for row in spam_reader:
            if '' in row:
                row.remove('')
            _, x, y = map(float, row)
            uu.append([x, y])
            i += 1

    lb = one_tree(Graph(points=uu, size=len(uu)))

    return lb, 3 * lb / 2

if __name__ == '__main__':

    while True:
        ini = 4
        tt, ss, rr = [], [], []
        for size in range(ini, 1000):

            process = subprocess.Popen(['python3', 'gen_tsp.py', str(size), str(2)])
            process.communicate()

            file_name = 'data_{}.txt'.format(size)

            ot, lb = get_lower_bound(file_name)

            process = subprocess.Popen(['./h_pro', file_name, 'tour_{}.txt'.format(size)], stdout=subprocess.PIPE)
            out, err = process.communicate()
            ds = float(out.decode())

            print(ds / lb, ds, lb)

            print(80 * '-')
            print('SIZE : {}'.format(size))

            tt.append(ds)
            ss.append(lb)
            rr.append(ot)

            plt.figure(figsize=(10, 10))
            plt.title('(3 / 2) 1-Tree (RED)\nH Algorithm Tour Length (BLUE)\n1-Tree (GREEN)\nhttp://twitter.com/maxtuno')
            plt.plot(range(ini, ini + len(ss)), ss, 'r-', label='(3/2) 1-Tree')
            plt.plot(range(ini, ini + len(tt)), tt, 'b-', label='H Algorithm')
            plt.plot(range(ini, ini + len(rr)), rr, 'g-', label='1-Tree')
            plt.savefig('graph.png')
            plt.close()


#!/usr/bin/env python
import rospy
import tf

from std_msgs.msg import String
from nav_msgs.msg import Odometry
from geometry_msgs.msg import Twist, PoseStamped
from actionlib_msgs.msg import GoalStatusArray
from drones_move.srv import *
from move_commands import MoveCommands as MC

import config

class Velocity:

    def __init__(self, linear=[0, 0, 0], angular=[0, 0, 0]):
        self.linear = linear
        self.angular = angular

class ReadyState:
    READY = 'READY'
    BUSY = 'BUSY'

class BotState:
    STATIONARY = 'STATIONARY'
    RPREPARING = 'RPREPARING'
    RSTARTING  = 'RSTARTING'
    ROTATING   = 'ROTATING'
    MOVING     = 'MOVING'

class BotCache:

    def __init__(self):
        self.odom = None
        self.move = None
        self.rstart_pose = None
        self.state = BotState.STATIONARY
        self.velocity = Velocity()
        self.command = MC.STOP

class Publishers:

    def __init__(self, cmdvel=None, movegoal=None, readystate=None):
        self.cmdvel = cmdvel
        self.movegoal = movegoal
        self.readystate = readystate

class Subscribers:

    def __init__(self, subscriptions=[]):
        for s in subscriptions:
            rospy.Subscriber(s[0], s[1], s[2])

class Services:

    def __init__(self, services):
        for s in services:
            rospy.Service(s[0], s[1], s[2])

class TurtlebotMover:

    def __init__(self):
        rospy.init_node(config.bot_mover['name'], anonymous=True)
        self.rate = rospy.Rate(10)
        self.cache = BotCache()
        self.pub = None

    def run(self):
        Subscribers(subscriptions = [ (config.turtlebot['pub']['odom'], Odometry, self.odom_callback )
                                    , (config.move_base['pub']['status'], GoalStatusArray, self.goal_status_callback ) ])
        Services(services = [ (config.bot_mover['srv']['move_bot'], MoveTurtlebot, self.handle_move) ])
        self.pub = Publishers( cmdvel     = rospy.Publisher(config.bot_mover['pub']['cmd_vel'], Twist, queue_size=10)
                             , movegoal   = rospy.Publisher(config.bot_mover['pub']['move_goal'], PoseStamped, queue_size=10)
                             , readystate = rospy.Publisher(config.bot_mover['pub']['ready_state'], String, queue_size=10) )
        self.loop()

    def goal_status_callback(self, data):
        if self.cache.command == MC.MOVETO and self.is_movement_complete(data):
            self.log('I: {0} | {1}'.format(self.cache.state, self.cache.command))
            self.complete_movement()
        else:
            self.log('E: {0} | {1}'.format(self.cache.command, self.is_movement_complete(data)))

    def is_movement_complete(self, data):
        result = False

        if len(data.status_list) == 1:
            if data.status_list[0].status in [2, 3, 4, 5, 8]:
                result = True

        return result

    def complete_movement(self):
        self.cache.command = MC.STOP
        self.cache.state = BotState.STATIONARY

    def handle_move(self, request):
        if request.command == MC.THREE60:
            self.cache.command = MC.THREE60
            self.cache.state = BotState.RPREPARING
        elif request.command == MC.MOVETO:
            self.cache.command = MC.MOVETO
            self.cache.state = BotState.MOVING
            self.publish10(self.pub.movegoal, self.get_pose_stamped(request.pose))
        return MoveTurtlebotResponse('ok')

    def odom_callback(self, data):
        self.cache.odom = data

        if self.cache.state in [BotState.RPREPARING, BotState.RSTARTING, BotState.ROTATING]:
            if self.cache.state == BotState.RPREPARING:
                self.prepare_rotation(data)
            elif self.cache.state == BotState.RSTARTING:
                self.begin_rotation(data)
            elif self.cache.state == BotState.ROTATING:
                self.complete_rotation(data)

    def prepare_rotation(self, data):
        self.cache.rstart_pose = data.pose.pose
        self.cache.state = BotState.RSTARTING

    def begin_rotation(self, data):
        if self.is_rotation_started(data.pose.pose, self.cache.rstart_pose):
            self.cache.state = BotState.ROTATING
        else:
            self.cache.velocity.angular[2] = 0.5

    def complete_rotation(self, data):
        if self.is_rotation_complete(data.pose.pose, self.cache.rstart_pose):
            self.cache.velocity.angular[2] = 0.0
            self.cache.state = BotState.STATIONARY
            self.cache.command = MC.STOP

    def is_rotation_started(self, current, target):
        current_e = tf.transformations.euler_from_quaternion(( current.orientation.x
                                                             , current.orientation.y
                                                             , current.orientation.z
                                                             , current.orientation.w ))
        target_e = tf.transformations.euler_from_quaternion(( target.orientation.x
                                                            , target.orientation.y
                                                            , target.orientation.z
                                                            , target.orientation.w ))
        return False if self.are_angles_almost_equal(current_e[2], target_e[2]) else True

    def is_rotation_complete(self, current, target):
        current_e = tf.transformations.euler_from_quaternion(( current.orientation.x
                                                             , current.orientation.y
                                                             , current.orientation.z
                                                             , current.orientation.w ))
        target_e = tf.transformations.euler_from_quaternion(( target.orientation.x
                                                            , target.orientation.y
                                                            , target.orientation.z
                                                            , target.orientation.w ))
        return True if self.are_angles_almost_equal(current_e[2], target_e[2]) else False

    def are_angles_almost_equal(self, one, two):
        return True if abs(two - one) < 0.005 else False

    def loop(self):
        while not rospy.is_shutdown():
            if self.cache.command == MC.THREE60:
                self.publish( [ (self.pub.cmdvel
                                , self.make_twist( linear  = self.cache.velocity.linear
                                                 , angular = self.cache.velocity.angular )) ] )
            self.publish( [ (self.pub.readystate, self.get_ready_state()) ] )
            self.rate.sleep()

    def get_ready_state(self):
        return ReadyState.READY if self.cache.state == BotState.STATIONARY else ReadyState.BUSY

    def publish(self, actions):
        for a in actions:
            a[0].publish(a[1])

    def publish10(self, publisher, message):
        for i in range(0, 10):
            publisher.publish(message)
            self.rate.sleep()

    def make_twist(self, linear=[0, 0, 0], angular=[0, 0, 0]):
        t = Twist()
        t.linear.x = linear[0]
        t.linear.y = linear[1]
        t.linear.z = linear[2]

        t.angular.x = angular[0]
        t.angular.y = angular[1]
        t.angular.z = angular[2]

        return t

    def log(self, message):
        rospy.loginfo(message)

    def get_pose_stamped(self, pose):
            p = PoseStamped()
            p.header.stamp = rospy.Time.now()
            p.header.frame_id = 'map'
            p.pose = pose
            return p

def main():
    TurtlebotMover().run()

if __name__ == '__main__':
    main()


from urllib.request import urlopen
import re
import pyproteins.container.customCollection
import pyproteins.container.Core
import pyproteinsext.pfam as pfam
import json

import gzip

from xml.etree.ElementTree import parse, dump, fromstring, register_namespace, ElementTree, tostring

from os.path import expanduser

PfamEntrySet = None
uniprotEntrySet = None

PfamCache = None

from json import JSONEncoder

import pyproteins.utils.make_json_serializable
#def _default(self, obj):
#    return getattr(obj.__class__, "toJSON", _default.default)(obj)
#
#_default.default = JSONEncoder.default
#JSONEncoder.default = _default

## Returns a list of all keyword in collection, along with the list of uniprotObj featuring them
def keyWordChart(uniprotObjIter, kwType='GO'):
    def kwMapper(obj, _type) :
        if _type == 'GO':
            return obj.GO
        raise TypeError("implement other KW plz")

    kwChart = {}
    for uniprotObj in uniprotObjIter:
        for kwObj in  kwMapper(uniprotObj, kwType):
            if kwObj not in kwChart:
                kwChart[kwObj] = []
            kwChart[kwObj].append(uniprotObj)

    return sorted([ (k,v) for k,v in kwChart.items() ], key=lambda x : len(x[1]), reverse=True)

# Give link to uniprot Collection to allow proxy settings
# and cache setting for it and pfam
def getUniprotCollection ():
    global uniprotEntrySet
    if not uniprotEntrySet:
        uniprotEntrySet = EntrySet()

    return uniprotEntrySet

#def setCache(location):
#    print location
#    uniprotEntrySet.setCache(location=location)

#def proxySetting(**kwargs):
#    proxySetting(**kwargs)

def getPfamCollection ():
    global PfamEntrySet
    if not PfamEntrySet:
        home = expanduser("~")
        PfamCacheDefault = home
        PfamEntrySet = pfam.EntrySet(PfamCacheDefault)

    return PfamEntrySet

def proxySetting(**param):
    pyproteins.container.Core.proxySetting(param)

'''
    TODO Isoform, minimal -> affects the fasta sequence
                 Need to check isoform data xml structure, sequence variant specs of Uniprot
'''

def strip(string):
    subString = re.search(".xml$", string)
    if subString:
        return string.split(".")[0]

    return None

def capture(string):
    subString = re.search("[OPQ][0-9][A-Z0-9]{3}[0-9]|[A-NR-Z][0-9]([A-Z][A-Z0-9]{2}[0-9]){1,2}", string)
    if subString:
        return subString.group()

    return None

def isValidID(string):
    if re.match("^[OPQ][0-9][A-Z0-9]{3}[0-9]|[A-NR-Z][0-9]([A-Z][A-Z0-9]{2}[0-9]){1,2}$", string):
        return True
    if re.match("^[A-Z]{3}[0-9]{5}$", string):
        return True
    return False

class fetchEntries():
    def __init__(self, list):
        self.list = list
        self.entrySet = [Entry(id = id) for id in self.list]
    def __iter__(self):
        for entry in self.entrySet:
            yield entry

''' Collection of uniprot entries
customary cache location is "/Users/guillaumelaunay/work/data/uniprot"
'''
class EntrySet(pyproteins.container.customCollection.EntrySet):

    def __init__(self, **kwargs):
        self.ns = '{http://uniprot.org/uniprot}'

        home = expanduser("~")
        cachePath = home
        self.isXMLCollection = False
        self.isRedisCollection = False
        if 'streamXML' or 'collectionXML' in kwargs:
            self.isXMLCollection = True
            if 'streamXML' in kwargs:
                self.etree = parse(kwargs['streamXML'])
            elif 'collectionXML' in kwargs:
                if kwargs['collectionXML'].endswith('.gz'):
                    with gzip.open(kwargs['collectionXML'], 'r') as gz:
                        self.etree = parse(gz)
                else:
                    self.etree = parse(kwargs['collectionXML'])

            self.etree_root = self.etree.getroot()
            self.index = None
          #  print(f"==> {type(self.etree_root)} {type(self.etree)} <==")

        super().__init__(collectionPath=cachePath, constructor=Entry, typeCheck=isValidID, indexer=strip)
        if 'collectionXML' in kwargs:
            print (f"Acknowledged {len(self)} entries {kwargs['collectionXML']}")

    def __len__(self):
        if self.isXMLCollection:
            return len(list([ _ for _ in self ]))
        return super().__len__()

    def keys(self):
        """ Returns uniprot identifiers within collection as a generator """
        if self.isXMLCollection:
            for entry in self.etree_root.findall(f"{self.ns}entry"):
                for alt_acc in entry.findall(f"{self.ns}accession"):
                    yield(alt_acc.text)
        else :
            return super().keys()

    def has(self, id):
        if self.isXMLCollection:
            if self.index is None:
                self.index = {}
                for acc in self.keys():
                    self.index[acc] = True
            return id in self.index
        else :
            return super().has()

    def __iter__(self):
        if self.isXMLCollection:
            for entry in self.etree_root.findall(f"{self.ns}entry"):
                uniprotID = entry.find(f"{self.ns}accession").text
                yield Entry(uniprotID, xmlEtreeHandler = entry, xmlNS = self.ns)

    def get(self, uniprotID):
        if self.isXMLCollection:
            for entry in self.etree_root.findall(f"{self.ns}entry"):
                for acc in entry.findall(f"{self.ns}accession"):
                    if acc.text == uniprotID: # entry is the node matching provided UNIPROT accessor
                        return Entry(uniprotID, xmlEtreeHandler = entry, xmlNS = self.ns)
            return None
        else :
            print(f"Looking in XML files/dir collection for {uniprotID}")
            return super().get(uniprotID, xmlNS = self.ns)

    def serialize(self, ext=''):
        global PfamCache
        print ("serializing uniprot collection")
        super().serialize(ext=ext)
        if PfamCache:
            print ("serializing pfam collection")
            getPfamCollection().serialize(ext=ext)

    @property
    def taxids(self):
        taxids = set()
        for e in self:
            taxids.add(e.taxid)
        return list(taxids)

class Entry(pyproteins.container.Core.Container):
    ## Wrap and split kwargs
    # Re-encode etree xpath like search
    # https://docs.python.org/3/library/xml.etree.elementtree.html#example
    # //div[@id='..' and @class='...]

    def __repr__(self):
        asStr = f"{self.id}:{self.AC}\n"
        asStr += f"{self.name}:{self.fullName}({self.geneName})\n"

        if self.STRING_ID:
            asStr += f"STRING_ID:{self.STRING_ID}\n"

        asStr += f"taxid:{self.taxid}:{self.lineage}\n"
        asStr += f"KW:{self.KW}\n"
        asStr += f"GO:{self.GO}\n"

        return asStr

    def _buildXpath(self, xpath, **kwargs):

        _xpath_ = xpath.replace("/",f"/{self._ns}")
        _xpath_ += ' and '.join( [ f"[@{k}='{v}']" for k,v in kwargs.items() ] )

        if not ( _xpath_.startswith('/') or _xpath_.startswith('./') ) :
            _xpath_ = f"{self._ns}{_xpath_}"

        #print(f"building xpath from {xpath} to {_xpath_}")
        return _xpath_

    def _xmlAttrib(self, key, elem=None):
        if elem is None:
            elem = self.xmlHandler
        return elem.attrib[key]

    def _xmlFind(self, tag, elem=None, **kwargs):
        if elem is None:
            elem = self.xmlHandler
        if len(kwargs.keys()) > 1:
            raise(f"Asking for Too many attributes ({len(kwargs.keys())}) in tag xpath search")

        xpathStr = self._buildXpath(tag, **kwargs)
        return elem.find(xpathStr)

    def _xmlFindAll(self, tag, elem=None, **kwargs):
        if elem is None:
            elem = self.xmlHandler

        xpath = self._buildXpath(tag, **kwargs)
        return elem.findall(xpath)

    def __init__(self, id, baseUrl="http://www.uniprot.org/uniprot/", fetchable= True, fileName=None, xmlEtreeHandler=None, xmlNS=None):
        if not id:
            raise TypeError('identifier is empty')
        super().__init__(id, url=baseUrl + str(id) + '.xml', fileName=fileName)
        #pyproteins.container.Core.Container.__init__(self, id, url=baseUrl + str(id) + '.xml', fileName=fileName)

        if not xmlNS is None:
            self._ns = xmlNS

        if not xmlEtreeHandler is None:
            self.xmlHandler = xmlEtreeHandler
        else:
            print("Search for", f"{xmlNS}entry")
            self.xmlHandler = self.getXmlHandler(fetchable=fetchable).find(f"{xmlNS}entry")

        if self.xmlHandler is None:
            return None

        self.name = self._xmlFind("./name").text

        if not self._xmlFind("./protein/recommendedName/fullName") is None:
            self.fullName = self._xmlFind("./protein/recommendedName/fullName").text
        else:
            self.fullName = self.name

        self.geneName =  None
        e = self._xmlFind("gene")
        if not e is None:
           self.geneName = self._xmlFind("./name", elem=e).text

        self.STRING_ID = None
        e = self._xmlFind("./dbReference", type="STRING")
        if not e is None:
            self.STRING_ID = self._xmlAttrib('id', elem=e)

        self.parseAC()
        self.parseLineage()
        self.parseKW()
        self.parseGO()
        self.parseSequence()
        self.parse_subcellular_location()

# Following oarsing stages are disabled since we got rid of bs4
# Need to port them  to lxml, making use of xpath syntax
    def PARSER_TO_PORT_TO_ETREE(self):

        self.parseSse()
        self.Ensembl = self.parseEnsembl()
        self.GeneID = self.parseGeneID()
        self.parseSequence()
        self.parsePDB()
        self.parseMIM()
        self.parseDI()
        self.parseORPHA()
        self.xref = self.get_xref()
        self.parseInterpro()

    def __hash__(self):
        return hash(self.id)

    def __copy__(self):
        return self

    def __deepcopy__(self, memo):
        return self

    def __eq__ (self, other):
        return self.id == other.id

    def toJSON(self):
        #print ('toJSON')
        #asDict = {}
        #for k in self.__dict__.keys():
        #    if k == 'xmlHandler':
        ##        continue
        #    if k != 'GO':
        #        continue
        #    asDict[str(k)] = getattr(self, k)
        #    print(asDict)

        container = {}
        for k, v in self.__dict__.items():
            if k == 'name':
                container[k] = v
            if k == 'GO':
                container[k] = [ go.__dict__ for go in v ]
            if k == 'id':
                container[k] = v
            if k == 'geneName':
                container[k] = v
            if k == 'fullName':
                container[k] = v
            if k == 'taxid':
                container[k] = v
        return container

    def parseAC(self):
        self.AC = [ e.text for e in  self._xmlFindAll("./accession") ]

    def parseLineage(self):
        self.lineage = [ e.text for e in  self._xmlFindAll("./organism/lineage/taxon") ]
        e = self._xmlFind('organism/dbReference', type='NCBI Taxonomy')
        self.taxid = self._xmlAttrib('id', elem=e)

    def parseMIM(self):
        self.MIM = []
        for e in self.xmlHandler.find_all("dbReference", type="MIM"):
            if str(e.parent.name) == 'entry':
                self.MIM.append(MimKW(e))

    def parseDI(self):
        self.DI = []
        for e in self.xmlHandler.find_all("disease"):
            self.DI.append(DI(e))

    def parseGO(self):
        self.GO = []
        for e in self._xmlFindAll("dbReference", type="GO"):
            self.GO.append(GoKW(
                self._xmlAttrib('id', elem=e),
                self._xmlAttrib('value', elem = self._xmlFind("property", elem=e, type="term") ),
                self._xmlAttrib('value', elem = self._xmlFind("property", elem=e, type="evidence") )
            ))

    def parseORPHA(self):
        self.ORPHA = []
        for e in self.xmlHandler.find_all("dbReference", type="Orphanet"):
            if str(e.parent.name) == 'entry':
                self.ORPHA.append(OrphaKW(e))

    def parsePDB(self):
        self.pdbRef = []
        for e in self.xmlHandler.find_all("dbReference", type="PDB"):
            self.pdbRef.append(PDBref(e))

    def parseDomain(self):
        try:
            self.domains=getPfamCollection().map(uniprotID=self.id)
        except:
            self.domains=[]

        #self.domains = []
        #for e in self.xmlHandler.find_all("feature", type="domain"):
        #    buf = Domain(e, self.id)
        #    if buf.description:
        #        self.domains.append(Domain(e, self.id))
        #for e in self.xmlHandler.find_all("feature", type="repeat"):
        #    buf = Domain(e, self.id)
        #    if buf.description:
        #        self.domains.append(Domain(e, self.id))
        #if not self.domains:
        #    print "No domain data found for " + self.id + ", attempting pfam"
        #    try :
        #        self.domains = getPfamCollection().map(uniprotID=self.id)
        #    except ValueError as msg:
        #        print ("Could not bind uniprot to its pfam ressources reason\n" + str(msg))

    def parseSse(self):
        self.sse = []
        for e in self.xmlHandler.find_all("feature", type="strand"):
            self.sse.append(Sse(e))
        for e in self.xmlHandler.find_all("feature", type="turn"):
            self.sse.append(Sse(e))
        for e in self.xmlHandler.find_all("feature", type="helix"):
            self.sse.append(Sse(e))

    def parseKW(self):
        self.KW = []

        for e in self._xmlFindAll("./keyword"):
            self.KW.append( UniprotKW(self._xmlAttrib('id', elem=e), e.text) )

    def parseSequence(self):
        self.sequence = self._xmlFind("./sequence").text.replace("\n", "").replace(" ", "")
        #self.sequence = Sequence(self.xmlHandler.find("sequence", {"length" : True}))
    #    pass

    def parseEnsembl(self):
        #Search for Ensembl id : ENSGXXXXXXXXXXX
        Ensembl_id = []
        for e in self.xmlHandler.find_all("dbReference", type="Ensembl"):
            for e_gene_id in e.find_all('property',type='gene ID'):
                if e_gene_id["value"] not in Ensembl_id:
                    Ensembl_id.append(e_gene_id["value"])
        return Ensembl_id

    def parseGeneID(self):
        GeneID= []
        for e_gene_id in self.xmlHandler.find_all("dbReference", type="GeneID"):
            if e_gene_id["id"] not in GeneID:
                GeneID.append(e_gene_id["id"])
        return GeneID

    def parse_subcellular_location(self):
        self.subcellular_location = []
        for citation in self._xmlFindAll('comment', type="subcellular location"):
            for sl in self._xmlFindAll('subcellularLocation', elem = citation):
                for location in self._xmlFindAll('location', elem = sl):
                    self.subcellular_location.append(location.text)
        # except :
        #     print("WARNING", self.AC, "subcellular location can't be parsed")
        #     print(citation, sl, location)

        # self.subcellular_location = []
        # citations = self._xmlFindAll("comment", type="subcellular location")
        # for citation in citations:
        #     annotation = self._xmlFind('subcellularLocation/location', elem=citation)

        #     # if not annotation:
        #     #     print("no annotation", self.AC, annotation)
        #     # else:
        #     try:
        #         self.subcellular_location.append(annotation.text)
        #     except:
        #         print("WARNING", self.AC, "subcellular location can't be parsed")
        #         self.subcellular_location = ['WARNING']

    def get_xref(self):
        # print("GET XREF")
        dic_xref = {'EMBL': {}, 'RefSeq': {}}
        # Search EMBL
        for e in self.xmlHandler.find_all("dbReference", type="EMBL"):
            if str(e.parent.name) == 'entry':
                for e_prot_id in e.find_all('property',type='protein sequence ID'):
                    dic_xref["EMBL"][e["id"]] = e_prot_id["value"]
        # Search RefSeq
        for e in self.xmlHandler.find_all("dbReference", type="RefSeq"):
            if str(e.parent.name) == 'entry':
                for e_prot_id in e.find_all('property',type='nucleotide sequence ID'):
                    dic_xref["RefSeq"][e_prot_id["value"]] = e["id"]
        return dic_xref

    def parseInterpro(self):
        self.Interpro = []
        for e in self.xmlHandler.find_all("dbReference", type = "InterPro"):
            for name in e.find_all("property", type = "entry name"):
                self.Interpro.append(Interpro(e))

    @property
    def fasta(self):
        return '>' + str(self.id) + ' ' + str(self.name) + '\n' + str(self.sequence)

    def peptideSeed(self):
        return {
            'id' : self.id,
            'desc' : self.name,
            'seq' : str(self.sequence)
        }

        #self.parseIsoform()

    def hasKW(self, keyword):
        if keyword.upper() in (kw.id.upper() for kw in self.KW):
            return True
        return False

    @property
    def isGOannot(self):
        return not len(self.GO) == 0

    def hasGO(self, keyword):
        if keyword.upper() in (kw.id.upper() for kw in self.GO):
            return True
        return False

    def hasMIM(self, keyword):
        if keyword.upper() in (kw.id.upper() for kw in self.MIM):
            return True
        return False

    def hasORPHA(self, keyword):
        if keyword.upper() in (kw.id.upper() for kw in self.ORPHA):
            return True
        return False

    def hasDI(self, keyword):
        if keyword.upper() in (kw.id.upper() for kw in self.DI):
            return True
        return False

    # returns a position information container
    def pos(self, i, lookup=False):
        if  i < 1 or i >= len(self.sequence):
            raise IndexError("sequence index \""  + str(i) + "\" is out of range (" + str(len(self.sequence)) + ")")
        d = self._domainFlyCast(position=i)
        if len(d) > 1:
            raise ValueError("\"" + self.id + "\" lays more than one domain \"" + str(len(d)) +
                             "\" at position " + str(i)  )
        domain="SomeDefault"
        if d:
            domain = d[0]
        else:
            l = self._domainFlyCast(NterLookup=i)
            r = self._domainFlyCast(CterLookup=i)
            if l and r:
                domain = { 'outOfBounds' : 'hinge' }
            elif l:
                domain = { 'outOfBounds' : 'Cter' }
            elif r:
                domain = { 'outOfBounds' : 'Nter' }
            else:
                print ("Not a hinge a Cter or a Nter at " + str(i) + " in " + self.id)
        return Position(i, domain=domain, sse=self._getSse(i), letter=self.sequence[i])

    def _domainFlyCast(self, **kwargs):
        if not self.domains:
            raise ValueError("No domain data available for " + self.id)
            #return None
        if 'position' in kwargs:
            results = [ d._dict for d in self.domains if d.owns(kwargs['position']) ]
            return results
        if 'NterLookup' in kwargs:
            for i in range (int(kwargs['NterLookup']), 0, -1):
                d = self._domainFlyCast(position=i)
                if d:
                    return d#d[0]
        if 'CterLookup' in kwargs:
            for i in range (int(kwargs['CterLookup']), len(self.sequence) + 1):
                d = self._domainFlyCast(position=i)
                if d:
                    return d#d[0]

        return None

    def _getSse(self, position):
        i = int(position)
        if not self.sse:
            return None
        for d in self.sse:
            if d.begin <= i <= d.end:
                return d.type
        return "coil"

# Custom encoder for uniprot entity
class EntryEncoder(json.JSONEncoder):
    def default(self, entryObj):
        if isinstance(entryObj, pyproteins.container.Core.Container):
            container = {}
            for k, v in entryObj.__dict__.items():
                if k == 'name':
                    container[k] = v
                if k == 'GO':
                    container[k] = [ go.__dict__ for go in v ]
                if k == 'id':
                    container[k] = v
                if k == 'geneName':
                    container[k] = v
                if k == 'fullName':
                    container[k] = v
            return container
        # Error
        return json.JSONEncoder.default(self, entryObj)

class Position ():
    def __init__(self, number, **kwargs):
        self.number = number
        self.domain = kwargs['domain'] if 'domain' in kwargs else None

        self.sse = kwargs['sse'] if 'sse' in kwargs else None
        self.letter = kwargs['letter'] if 'letter' in kwargs else None

    def __repr__(self):
        return self.letter + str(self.number) + "(" + str(self.sse) + ") : " + str(self.domain)

class Sequence():
    def __init__(self, e):
        if not e:
            raise ValueError("Cant parse sequence")
        self.string = e.text.replace("\n", "").replace(" ", "")
        self.mass =e._xmlAttrib('mass')
    def __len__(self):
        return len(self.string)
    def __getitem__(self, i):
        if isinstance(i, slice):
            newSlice = slice(i.start - 1, i.stop - 1)
            return self.string[newSlice]
        else:
            if i < 1 or i > len(self):
                raise IndexError(str(i) + " out of range [1-" + self(len(self)) + "]")
            return self.string[i - 1]

    def __repr__(self):
        return self.string

class Domain(object):
    def __init__(self, xmlHandler, id):
        self.begin = [int(e['position']) for e in xmlHandler.find_all('begin')]
        self.end = [int(e['position']) for e in xmlHandler.find_all('end')]
        if len(self.begin) != len(self.end):
            raise ValueError("Number of end/stop differs " + str(self.begin) + "/" + str(self.end))
        if 'description' not in xmlHandler:
            #print "Warning Domain w/in \"" + id + "\" does not feature any description"
            self.description = None
        else:
            self.description = xmlHandler['description']
        self.carriedBy = id

    def __eq__(a, b):
        if a.id != b.id:
            return False
        if a.begin != b.begin:
            return False
        if a.end != b.end:
            return False
        if a.description != b.description:
            return False

        return True

    def __repr__(self):
        b = [ str (self.begin[i]) + "-" + str(self.end[i]) for i,e  in enumerate (self.begin) ]
        return "\"" + self.description + "\"\t" + ",".join(b) + "\n"

    def owns(self, position):
        try :
            j = int(position)
        except ValueError as err:
            print ("Improper amino acid position \"" + str(position) + "\"")
            return False

        for i, e  in enumerate (self.begin):
            if self.begin[i] <= j <= self.end[i]:
                return True

        return False

    @property
    def _dict(self):
        return self.__dict__

class Sse():
    def __init__(self, e):
        self.begin = int(e.find('begin')['position'])
        self.end =  int(e.find('end')['position'])
        self.type = e['type']
    def __repr__(self):
        return "SSE:" + self.type + " " + str(self.begin) + "-" + str(self.end)

class annotTerm:
    def __init__(self):
        pass
    def __hash__(self):
        return hash(str(self.id))
    def __eq__(self, other):
        return hash(self) == hash(other)

class GoKW(annotTerm):
    def __init__(self, id,term, evidence):
        self.id = id
        self.term = term
        self.evidence = evidence
    def __repr__(self):
        return self.id + ":" + self.term + "{" + self.evidence + "}"

class MimKW(annotTerm):
    def __init__(self, e):
        self.id = e['id']
        self.value = e.find('property', type='type')['value']
    def __repr__(self):
        return self.id + ":" + self.value

class OrphaKW(annotTerm):
    def __init__(self, e):
        self.id = e['id']
        self.type = e.find("property")['type']
        self.value = e.find('property')['value']

    def __repr__(self):
        return self.id + ": (" + self.type + ")" + self.value

class DI(annotTerm):
    def __init__(self, e):
        self.id = e['id']
        self.name = e.find('name').string
        acronym = e.find('acronym')
        self.acronym = acronym.string if acronym else 'NA'
        self.description = e.find('description').string
    def __repr__(self):
        return self.id + ":" + self.name + " (" + self.acronym + ") {" + self.description + "}"

class PDBref():
    def __init__(self, e):
        self.id = e['id']
        self.method = e.find('property', type='method')['value'] if e.find('property', type='method') else None
        self.resolution = e.find('property',type='resolution')['value'] if e.find('property', type='resolution') else None
        self.chains = e.find('property',type='chains')['value'] if e.find('property', type='chains') else None

    def __repr__(self):
        string = "PDB_id: " + self.id
        elements = []
        if self.method:
            elements.append("method: " + self.method)
        if self.resolution :
            elements.append("resolution: " + self.resolution)
        if self.chains:
            elements.append("chains: " + self.chains)
        if elements:
            return string + ':{' + ','.join(elements)  + '}'

        return string

class UniprotKW():
    def __init__(self, id, text):
        self.id = id
        self.term = text
    def __repr__(self):
        return self.id + ":" + self.term

class Interpro():
    def __init__(self,e):
        self.id = e["id"]
        self.getName(e)

    def __repr__(self):
        return self.id + ":" + self.name

    def getName(self,e):
        for name in e.find_all("property", type = "entry name"):
           self.name = name["value"]

class Genome():
    def __init__(self,xmlHandler):
        self.searchEMBL(xmlHandler)
        self.searchRefSeq(xmlHandler)

    def searchEMBL(self,xmlHandler):
        self.EMBLRef=[]
        self.EMBLProteinRef=[]
        for e in xmlHandler.find_all("dbReference", type="EMBL"):
            if str(e.parent.name) == 'entry':
                self.EMBLRef.append(e['id'])
                for e_prot_id in e.find_all('property',type='protein sequence ID'):
                    self.EMBLProteinRef.append(e_prot_id['value'])

    def searchRefSeq(self,xmlHandler):
        self.RefSeqRef=[]
        self.RefSeqProteinRef=[]
        for e in xmlHandler.find_all("dbReference", type="RefSeq"):
            if str(e.parent.name) == 'entry':
                self.RefSeqProteinRef.append(e['id'])
                for e_prot_id in e.find_all('property',type='nucleotide sequence ID'):
                    self.RefSeqRef.append(e_prot_id['value'])


import time
from typing import Any, Dict, Optional

import requests

from .exceptions import BirdPlanError

PeeringDBInfo = Dict[str, Any]

# Keep a cache for results returned while loaded into memory
#
# Example:
#  peeringdb_cache = {
#    'objects': {
#        'asn:174': {  # ASN
#            '_timestamp': 0000000000,
#            'value': xxxxxx,
#        }
#    }
#  }
peeringdb_cache: Dict[str, Dict[str, Any]] = {}

PEERINGDB_16BIT_LOWER = 64512
PEERINGDB_16BIT_UPPER = 65534
PEERINGDB_32BIT_LOWER = 4200000000
PEERINGDB_32BIT_UPPER = 4294967294

class PeeringDB:  # pylint: disable=too-few-public-methods
    """PeeringDB support class."""

    def __init__(self) -> None:
        """Initialize object."""

    def get_prefix_limits(self, asn: int) -> PeeringDBInfo:
        """Return our peeringdb info entry, if there is one."""
        # We cannot do lookups on private ASN's
        if (PEERINGDB_16BIT_LOWER <= asn <= PEERINGDB_16BIT_UPPER) or (PEERINGDB_32BIT_LOWER <= asn <= PEERINGDB_32BIT_UPPER):
            return {"info_prefixes4": None, "info_prefixes6": None}

        # Try pull result from our cache
        result = self._cache(f"asn:{asn}")
        # If we can't, grab the result from PeeringDB live
        if not result:
            # Request the PeeringDB info for this AS
            response = requests.get(f"https://www.peeringdb.com/api/net?asn__in={asn}", timeout=10)
            # Check the result is not empty
            if not response:  # pragma: no cover
                raise BirdPlanError("PeeringDB returned and empty result")
            # Decode response
            result = response.json()["data"][0]
            # Cache the result we got
            self._cache(f"asn:{asn}", result)

        # Total cluster .... just to get typing happy
        peeringdb_info = {"info_prefixes4": 1, "info_prefixes6": 1}
        if result and "info_prefixes4" in result:
            peeringdb_info["info_prefixes4"] = result["info_prefixes4"]
        if result and "info_prefixes6" in result:
            peeringdb_info["info_prefixes6"] = result["info_prefixes6"]

        # Lastly return it
        return peeringdb_info

    def _cache(self, obj: str, value: Optional[PeeringDBInfo] = None) -> Optional[Any]:  # noqa: CFQ004
        """Retrieve or store value in cache."""

        if "objects" not in peeringdb_cache:
            peeringdb_cache["objects"] = {}

        if not value:
            # If the cached obj does not exist, return None
            if obj not in peeringdb_cache["objects"]:
                return None
            # Grab the cached object
            cached = peeringdb_cache["objects"][obj]
            # Make sure its timestamp is within 60s of being retrieved, if not, return None
            if cached["_timestamp"] + 60 < time.time():  # pragma: no cover
                return None
            # Else its valid, return the cached value
            return cached["value"]

        # Set the cached value
        peeringdb_cache["objects"][obj] = {
            "_timestamp": time.time(),
            "value": value,
        }

        return value


# https://web.dio.me/course/seguranca-da-informacao-com-python/learning/daf1d8ac-bfcd-48a0-bad8-e277a9642c44

import itertools

texto = input('Digite a palavra a ser permutada: ')

resultado = itertools.permutations(texto, len(texto))

for i in resultado:
    print(''.join(i))


# coding: utf-8
import tensorflow as tf
import numpy as np
import json
from sklearn import metrics
from tqdm import tqdm
import os
import re
from bert_layer import *

CONTINUE_TRAIN = False
MAX_SEQLEN = 16
MAX_TEXTLEN = 25
BATCH_SIZE = 512
EPOCHS = 100
LEARNING_RATE = 1e-5
ATT_SIZE = 256
MODEL_PATH = 'model/'
DROPOUT_RATE = 0.5
DISPLAY_ITER = 10
BERT_EMBEDDING_SIZE = 768
TRAINSET_PATH = '/bdata/jiayunz/Foursquare/100w/train_1_sentence_emb.json'
TESTSET_PATH = '/bdata/jiayunz/Foursquare/100w/test_1_sentence_emb.json'
BERT_MODEL_PATH = '../../embedding/models/multi_cased_L-12_H-768_A-12'
VENUE_EMBEDDING_PATH = 'venue_embedding.json'
class GenerateData():
    def __init__(self, rpath):
        self.rpath = rpath
        # read data
        self.read_data()
        # shuffle
        self.shuffle()
        # initialize batch id
        self.batch_id = 0

    def read_data(self):
        self.texts = []
        self.venues = []
        self.labels = []
        self.mask = []
        with open(VENUE_EMBEDDING_PATH) as rf:
            venue_embedding = json.load(rf)
        #bc = BertClient()
        with open(self.rpath, 'r') as rf:
            lines = rf.readlines()
            for line in tqdm(lines, total=len(lines)):
                user = json.loads(line.strip())
                #tips = [t['text'] for t in user['fsq']['tips']['tips content'][:MAX_SEQLEN]]
                #emb_tips = bc.encode(tips)
                emb_tips = user['fsq']['tips']['tips embedding'][:MAX_SEQLEN]
                emb_venues = [venue_embedding[t['category']] for t in user['fsq']['tips']['tips content'][:MAX_SEQLEN]]
                self.mask.append([True] * len(emb_tips) + [False] * (MAX_SEQLEN - len(emb_tips)))
                if len(emb_tips) < MAX_SEQLEN:
                    emb_tips = np.concatenate((emb_tips, [[0.] * 768 for _ in range(MAX_SEQLEN - len(emb_tips))]))
                self.texts.append(emb_tips)
                self.venues.append(emb_venues)
                self.labels.append(user['label'])

        self.labels = np.eye(2)[self.labels]

    def next(self):
        if BATCH_SIZE <= len(self.texts) - self.batch_id:
            batch_texts = self.texts[self.batch_id:(self.batch_id + BATCH_SIZE)]
            batch_venues = self.venues[self.batch_id:(self.batch_id + BATCH_SIZE)]
            batch_mask = self.mask[self.batch_id:(self.batch_id + BATCH_SIZE)]
            batch_labels = self.labels[self.batch_id:(self.batch_id + BATCH_SIZE)]
            self.batch_id = self.batch_id + BATCH_SIZE
        else:
            batch_texts = self.texts[self.batch_id:]
            batch_venues = self.venues[self.venues:]
            batch_mask = self.mask[self.batch_id:]
            batch_labels = self.labels[self.batch_id:]

            # shuffle
            self.shuffle()
            # reset batch id
            self.batch_id = 0

        return batch_texts, batch_venues, batch_mask, batch_labels

    def shuffle(self):
        np.random.seed(1117)
        np.random.shuffle(self.texts)
        np.random.seed(1117)
        np.random.shuffle(self.venues)
        np.random.seed(1117)
        np.random.shuffle(self.mask)
        np.random.seed(1117)
        np.random.shuffle(self.labels)

class Model():
    def __init__(self):
        self.build_inputs()
        self.build_model()
        self.build_loss()
        self.build_optimizer()

    def build_inputs(self):
        self.texts = tf.placeholder(shape=[None, MAX_SEQLEN, BERT_EMBEDDING_SIZE], dtype=tf.float32)
        self.venues = tf.placeholder(shape=[None, MAX_SEQLEN, BERT_EMBEDDING_SIZE], dtype=tf.float32)
        self.mask = tf.placeholder(tf.int32, [None, MAX_SEQLEN])
        self.targets = tf.placeholder(tf.float32, (None, 2))
        self.dropout_rate = tf.placeholder(tf.float32)

    def venue_embedding(self, inputs, vocabulary_size, embedding_size):
        embeddings = tf.Variable(tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
        embedded = tf.nn.embedding_lookup(embeddings, inputs)
        return embedded

    def attention(self, inputs, mask, return_alphas=False):
        # hidden_size = inputs.shape[2].value  # D value - hidden size of the RNN layer
        # Trainable parameters
        w_omega = tf.Variable(tf.variance_scaling_initializer(scale=1., mode='fan_in')((int(inputs.shape[-1]), ATT_SIZE)))
        b_omega = tf.Variable(tf.zeros([ATT_SIZE]))
        u_omega = tf.Variable(tf.variance_scaling_initializer(scale=1., mode='fan_in')((ATT_SIZE,)))

        # Applying fully connected layer with non-linear activation to each of the B*T timestamps;
        # the shape of `v` is (B,T,D)*(D,A)=(B,T,A), where A=attention_size
        v = tf.tanh(tf.tensordot(inputs, w_omega, axes=1) + b_omega)

        # For each of the timestamps its vector of size A from `v` is reduced with `u` vector
        vu = tf.tensordot(v, u_omega, axes=1)  # (B,T) shape
        # padding部分减去一个很大的整数，使其softmax后接近于0
        vu = tf.where(tf.equal(mask, True), vu, vu - float('inf'))

        alphas = tf.nn.softmax(vu)  # (B,T) shape

        # Output of (Bi-)RNN is reduced with attention vector; the result has (B,D) shape
        output = tf.reduce_sum(inputs * tf.expand_dims(alphas, -1), 1)

        if not return_alphas:
            return output
        else:
            return output, alphas

    def build_model(self):
        #embedded_venue = self.venue_embedding(tf.reshape(self.inputs['tip_venue'], (-1,)), 11, config['embedding_size']['tip_venue'])
        #embedded_venue = tf.reshape(embedded_venue, (-1, config['max_seqlen'], config['embedding_size']['tip_venue']))
        hidden_venue, alpha_venue = self.attention(self.venues, self.mask, return_alphas=True)
        hidden_text, alpha_text = self.attention(self.texts, self.mask, return_alphas=True)
        hidden = tf.concat([hidden_text, hidden_venue], axis=1)
        hidden = tf.keras.layers.Dense(units=512, activation='relu')(hidden)
        hidden = tf.keras.layers.Dense(units=128, activation='relu')(hidden)
        hidden = tf.keras.layers.Dense(units=32, activation='relu')(hidden)
        # dropout
        drop_outputs = tf.nn.dropout(hidden, rate=self.dropout_rate)
        self.logits = tf.keras.layers.Dense(units=2, activation=None)(drop_outputs)
        self.probs = tf.nn.softmax(self.logits)
        self.predictions = tf.argmax(self.probs, 1)

    def build_loss(self):
        cross_entropy = tf.nn.softmax_cross_entropy_with_logits_v2(logits=self.logits, labels=self.targets)
        self.loss = tf.reduce_mean(cross_entropy)

    def build_optimizer(self):
        self.optimizer = tf.train.AdamOptimizer(learning_rate=LEARNING_RATE).minimize(self.loss)

    def initialize_variables(self, sess):
        sess.run(tf.local_variables_initializer())
        sess.run(tf.global_variables_initializer())
        sess.run(tf.tables_initializer())
        K.set_session(sess)

    def train(self, trainset, testset):
        with tf.Session() as sess:
            if CONTINUE_TRAIN:
                try:
                    saver = tf.train.Saver()
                    saver.restore(sess, tf.train.latest_checkpoint(MODEL_PATH))
                    print('Successfully load model!')
                except:
                    print('Fail to load model!')
                    self.initialize_variables(sess)
            else:
                self.initialize_variables(sess)
            step = 1

            saver = tf.train.Saver()
            while step <= EPOCHS:
                self.train_one_epoch(sess, step, trainset)
                saver.save(sess, MODEL_PATH + 'model')
                print('----- Validation -----')
                self.test(sess, testset)
                print('---------------------')
                step += 1

            print("Optimization Finished!")

    def train_one_epoch(self, sess, step, dataset):
        iter = 0
        while iter * BATCH_SIZE < len(dataset.texts):
            iter += 1
            self.current_epoch = step
            batch_texts, batch_venues, batch_mask, batch_labels = dataset.next()
            feed = {
                self.texts: batch_texts,
                self.venues: batch_venues,
                self.mask: batch_mask,
                self.targets: batch_labels,
                self.dropout_rate: DROPOUT_RATE
            }

            _, y_pred, loss = sess.run([self.optimizer, self.predictions, self.loss], feed_dict=feed)
            y_true = np.argmax(batch_labels, 1)

            if iter % DISPLAY_ITER == 0:
                print("Epoch " + str(step) + ", Iter " + str(iter) +
                      ", Minibatch Loss = " + "{:.5f}".format(loss) +
                      ", Accuracy = " + "{:.5f}".format(metrics.accuracy_score(y_true, y_pred)) +
                      ", f1 score = " + "{:.5f}".format(metrics.f1_score(y_true, y_pred)) +
                      ", AUC = " + "{:.5f}".format(metrics.roc_auc_score(y_true, y_pred))
                      )

    def test(self, sess, dataset):
        #with tf.Session() as sess:
        #    saver = tf.train.Saver()
        #    saver.restore(sess, tf.train.latest_checkpoint(MODEL_PATH))

        predictions = []
        ground_truth = []

        feed = {
            self.texts: dataset.texts,
            self.venues: dataset.venues,
            self.mask: dataset.mask,
            self.targets: dataset.labels,
            self.dropout_rate: 0.
        }

        y_pred, loss = sess.run([self.predictions, self.loss], feed_dict=feed)
        y_true = np.argmax(dataset.labels, 1)

        predictions.extend(list(y_pred))
        ground_truth.extend(list(y_true))

        print("AUC:", metrics.roc_auc_score(ground_truth, predictions))
        print(metrics.classification_report(ground_truth, predictions, digits=4))

def main(_):
    #start_bert_server()
    trainset = GenerateData(TRAINSET_PATH)
    testset = GenerateData(TESTSET_PATH)
    model = Model()
    #BertServer.shutdown(port=5555)
    model.train(trainset, testset)

if __name__ == '__main__':
    tf.app.run()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import json
import sys
import urllib.parse
from termcolor import colored

sStandardInput = ""

for x in sys.stdin:
    sStandardInput = sStandardInput + x

#print (sStandardInput)
try:
    json_data = json.loads(sStandardInput)
except:
    pass

for sUrl in json_data["urls"]:
    print (colored("URL = " + sUrl + " (HTTP status "+ str(json_data['urls'][sUrl]['status']) + ")","green"))
#    print (colored("URL = " + sUrl["status"],"green"))

for app in json_data["applications"]:
    if app["version"]:
        sVersion1 = " " + str(app["version"])
        sVersion2 = " \"" + str(app["version"]) + "\""
    else:
        sVersion1 = ""
        sVersion2 = ""

    sExploitdbUrl = "https://www.exploit-db.com/search?q=" + urllib.parse.quote(app["name"])
    sGoogleUrl = "https://www.google.com/search?q=" + urllib.parse.quote("\"" + app["name"] + "\"" + sVersion2 + " cve |exploit |vulnerability |updates |changelog |risk |advisory |cvss")
    print (app["name"] + sVersion1 + " (" + app["confidence"] + "%)")
    print (" - " + app["website"])
    print (" - " + sExploitdbUrl)
    print (" - " + sGoogleUrl)


import matplotlib.pyplot as plt
import numpy,MetaDex,math

compressedtiers = []
tiers = MetaDex.getTiers()
for i in range(0,len(tiers),4):
    compressedtiers.append(tiers[i].split("-")[0])

#compressedtiers=["gen7ou"]

quarters = []
halves = []
thirdquarters =[]

for i in range(len(compressedtiers)):
    plt.figure(i)
    plt.subplot(121)
    plt.title("Usage for "+compressedtiers[i])
    plt.xlabel("Usage")
    plt.ylabel("Frequency")
    plt.axis([-0.05, 0.48, 0, 20])

    plt.subplot(122)
    plt.title("Usage vs. Raw Count for "+compressedtiers[i])
    plt.xlabel("Usage")
    plt.ylabel("Raw Count")

    numbers = ["0","1500","1630","1760"]
    #TODO: interesting...
    if "ou" in compressedtiers[i] and "battlespot" not in compressedtiers[i]:
        numbers = ["0","1500","1695","1825"]
    for n in numbers:
        tier=compressedtiers[i]+"-"+n
        tierfile = tier+".json"
        usageList = []
        rawcountList = []

        data = MetaDex.findTierData(tierfile)
        for poke in MetaDex.findTierData(tierfile):
            usageList.append(data[poke]["usage"])
            rawcountList.append(data[poke]["Raw count"])

        quarterUsage = sorted(usageList)[round(len(usageList)/4)]
        quarters.append(quarterUsage)
        halfUsage = sorted(usageList)[round(len(usageList)/2)]
        halves.append(halfUsage)
        thirdquarterUsage = sorted(usageList)[round(3*len(usageList)/4)]
        thirdquarters.append(thirdquarterUsage)

        usageNP = numpy.array(usageList)

        plt.figure(i)
        plt.subplot(121)
        if n == "0":
            plt.hist(usageNP, 500, color="red",alpha=0.5)
        elif n == "1500":
            plt.hist(usageNP, 500, color="yellow",alpha=0.5)
        elif n in ["1630","1695"]:
            plt.hist(usageNP, 500, color="green",alpha=0.5)
        elif n in ["1760","1825"]:
            plt.hist(usageNP, 500, color="blue",alpha=0.5)
        #plt.plot([quarterUsage, quarterUsage], [0, 20], c="black")
        #plt.plot([halfUsage, halfUsage], [0, 20], c="black")
        #plt.plot([thirdquarterUsage, thirdquarterUsage], [0, 20], c="black")

        plt.subplot(122)
        if n == "0":
            plt.scatter(usageList, rawcountList, s=2,color="red",alpha=0.75)
        elif n == "1500":
            plt.scatter(usageList, rawcountList, s=2,color="yellow",alpha=0.75)
        elif n in ["1630","1695"]:
            plt.scatter(usageList, rawcountList, s=2,color="green",alpha=0.75)
        elif n in ["1760","1825"]:
            plt.scatter(usageList, rawcountList, s=2,color="blue",alpha=0.75)

quarterav = sum(quarters)/len(compressedtiers)
quarterdeviation = 0
for i in quarters:
    quarterdeviation+=(i-quarterav)**2
quarterdeviation = math.sqrt(quarterdeviation/len(compressedtiers))

halfav = sum(halves)/len(compressedtiers)
halfdeviation = 0
for i in halves:
    halfdeviation+=(i-halfav)**2
halfdeviation = math.sqrt(halfdeviation/len(compressedtiers))

thirdquarterav = sum(thirdquarters)/len(compressedtiers)
thirdquarterdeviation = 0
for i in thirdquarters:
    thirdquarterdeviation+=(i-thirdquarterav)**2
thirdquarterdeviation = math.sqrt(thirdquarterdeviation/len(compressedtiers))

#for q in quarters:
#    print("data: %s" % q)
print("Quarter Average: %s" % quarterav)
print("Quarter Standard Deviation: %s" % quarterdeviation)
print()
#for h in halves:
#    print("data: %s" % h)
print("Half Average: %s" % halfav)
print("Half Standard Deviation: %s" % halfdeviation)
print()
#for t in thirdquarters:
#    print("data: %s" % t)
print("Third Quarter Average: %s" % thirdquarterav)
print("Third Quarter Standard Deviation: %s" % thirdquarterdeviation)
plt.show()


from collections import namedtuple

Algorithm = namedtuple("Algorithm", "name modelname options")

# define your ML algorithm here
analysis = [
Algorithm('NN',
          'DNN_ADAM_leakyReLU_layer256_batch32_GlorotNormalInitializer_l1-0p01_lr0p0001_decay0p0',
          {
          'layers':[256],
          'ncycles':1000,
          'batchSize':32,
          'dropout':0.5,
          'optimizer':'adam',
          'activation':'linear',
          'initializer':'glorot_normal',
          'regularizer':0.01,
          'classWeight':'balanced',
          'learningRate':0.0001,
          'decay':0.0,
          'momentum':0.0,
          'nesterov':False,
          'multiclassification':False
          }
),
Algorithm('BDT',
          'XGBoost_d3_nTrees1000_lr0p1',
          {
           'classifier':'xgboost',
           'learning_rate':0.1,
           'max_depth':3,
           'n_estimators':1000,
           'lambda':1, # Default=1, L2 regularization term on weights (analogous to Ridge regression)
           'alpha':0, # Default=0, L1 regularization term on weight (analogous to Lasso regression)
           'gamma':0, # A node is split only when the resulting split gives a positive reduction in the loss function. Gamma specifies the minimum loss reduction required to make a split.
          'scale_pos_weights':1 # Default=1, A value greater than 0 should be used in case of high class imbalance as it helps in faster convergence.
          }
),
Algorithm('RNN',
          'RNN_jetOnly_ADAM_leakyReLU_LSTM32_128NNlayer_batch32_BatchNorm_GlorotNormalInitializer_l2-0p01_lr0p001_decay0p0',
          {
          'collection':['jet'],
          'removeVar':['_m', '_mv2c10', '_id', '0_pt', '0_eta', '0_phi', '0_e', '1_pt', '1_eta', '1_phi', '1_e'],
          'unit_type':'LSTM',
          'n_units':32,
          'combinedDim':[128],
          'epochs':25,
          'batchSize':32,
          'dropout':0.5,
          'optimizer':'ADAM',
          'activation':'linear',
          'initializer':'glorot_normal',
          'regularizer':0.01,
          'learningRate':0.001,
          'decay':0.0,
          'momentum':0.0,
          'nesterov':False,
          'mergeModels':True,
          'multiclassification':False,
          'classWeight':'balanced'
          }
),
]


"""
    Natural Language Processing
"""

import nltk
#nltk.download_shell()
nltk.download("stopwords")

messages = [line.rstrip() for line in open("smsspamcollection/SMSSpamCollection")]

import pandas as pd
messages = pd.read_csv("smsspamcollection/SMSSpamCollection", sep="\t", names=["label","message"])
describe=messages.groupby("label").describe()

#for mess_no, messages in enumerate(messages)
messages["length"] = messages["message"].apply(len)

#import matplotlib.pyplot as plt
#import seaborn as sb
#messages["length"].plot.hist(bins=150)
#plt.show()

#spam ve olmayan mesajların uzunlukı dağılımları.
#messages.hist(column="length", by="label",bins=60,figsize=(12,4))

from nltk.corpus import stopwords as sw
stopwords = sw.words("english")

import string
def text_process(str):
    nopunc = [ch for ch in str if ch not in string.punctuation]
    nopunc = "".join(nopunc)
    ret=  [word for word in nopunc.split() if word.lower() not in stopwords]
    #return ret;
    #print(" ".join(ret))
    return " ".join(ret)

messages["message"] = messages["message"].apply(text_process)

#creatinf bag of words model
from sklearn.feature_extraction.text import CountVectorizer
import numpy as np
cv = CountVectorizer()# CountVectorizer(max_features=1500)#☼kolon sayısı
X = cv.fit_transform(messages["message"]).toarray()#matrix. sparse matrix yaptık (o' ı çok olan matrix)
y = messages["label"] == "ham"
y = np.array(y, dtype=int)

# Splitting the dataset into the Training set and Test set
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = .2, random_state = 0)

def f1_score(cm):
    tp = cm[0][0]
    tn = cm[1][1]
    fp = cm[1][0]
    fn = cm[0][1]

    accuracy = (tp+tn)/(tp+tn+fp+fn)
    precision = tp / (tp+fp)
    recall = tp/(tp+fp)
    score = 2*precision*recall/(precision+recall)
    return (accuracy, score)

from sklearn.metrics import confusion_matrix, classification_report
def test(classifier):
    classifier.fit(X_train, y_train)
    y_pred = classifier.predict(X_test)
    cm = confusion_matrix(y_test, y_pred)
    cr = classification_report(y_test, y_pred)
    return f1_score(cm), cr

from sklearn.naive_bayes import MultinomialNB
f1, cr = test(MultinomialNB())

#başka bir yöntem
#from sklearn.feature_extraction.text import TfidfTransformer
#from sklearn.pipeline import Pipeline
#pipeline = Pipeline([
#        ("bow",CountVectorizer(analyzer=text_process)),
#        ("tfidf", TfidfTransformer()),
#        ("classifier", MultinomialNB())
#        ])
#pipeline.fit(X_train,y_train)
#pred = pipeline.predict(X_test)
#cr_pipeline = classification_report(y_test)


"""
put the test data evaluation to this file
"""

from __future__ import absolute_import, division, print_function

import random, os, time, shutil, sys, datetime, pickle

import torch
import torch.nn as nn
import torch.backends.cudnn as cudnn
import torch.utils.data
import torch.utils.data.distributed

import glob
import logging

import time, os, random, json, math
import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import transformers
from transformers import (WEIGHTS_NAME, BertConfig, BertModel, BertTokenizer,
                          XLMConfig, XLMModel, XLMTokenizer,
                          XLMRobertaConfig, XLMRobertaModel, XLMRobertaTokenizer)
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler

from model_arch import ModelArch

import imp
config = imp.load_source('config', 'config.py')
args = config.config()

args['output_dir'] = args['output_dir'] + args['distname'] + '/'

print(args)

def seed_all(seed_value):
    random.seed(seed_value)  # Python
    np.random.seed(seed_value)  # cpu vars
    torch.manual_seed(seed_value)  # cpu  vars

    if torch.cuda.is_available():
        torch.cuda.manual_seed(seed_value)
        torch.cuda.manual_seed_all(seed_value)  # gpu vars
        torch.backends.cudnn.deterministic = True  # needed
        torch.backends.cudnn.benchmark = False

MODEL_CLASSES = {
    'bert': (BertConfig, BertModel, BertTokenizer),
    'xlm': (XLMConfig, XLMModel, XLMTokenizer),
    'xlmr': (XLMRobertaConfig, XLMRobertaModel, XLMRobertaTokenizer)
}

# 2. load_data

def load_test(args):
    print(f"  Model will be saved in {args['output_dir']}")
    if not os.path.exists(args['output_dir']):
        os.system('mkdir -p ' + args['output_dir'])
    test_path = args['data_dir'] + "test.cleaned.text.csv"
    # test_path = args['data_dir'] + "/test.hs.clean.csv"
    testdf = pd.read_csv(test_path)
    # testdf = pd.read_csv(test_path, nrows = 1000)
    test_text, test_labels = testdf.comment_text, testdf.label
    return test_text, test_labels

def load_model(args):
    args = args
    model_path = args['model_path']
    config_class, model_class, tokenizer_class = MODEL_CLASSES[args['model_type']]
    # config
    config = config_class.from_pretrained(model_path + 'config.json')
    config.output_hidden_states = False
    # model
    hugg_model = model_class.from_pretrained(model_path, config = config)
    # tokenizer
    tokenizer = tokenizer_class.from_pretrained(model_path)
    model = ModelArch(hugg_model, args)
    return tokenizer, model

def test_data_loader(tokenizer, args):

    test_text, test_labels = load_test(args)

    tokens_test = tokenizer.batch_encode_plus(
        test_text.tolist(),
        max_length = args['max_seq_length'],
        pad_to_max_length=True,
        truncation=True
    )

    test_seq = torch.tensor(tokens_test['input_ids'])
    test_mask = torch.tensor(tokens_test['attention_mask'])
    test_y = torch.tensor(test_labels.tolist())

    test_data = TensorDataset(test_seq, test_mask, test_y)

    return test_data, test_y


from abc import ABC, abstractmethod
import nltk
from nltk import sent_tokenize, word_tokenize
from nltk.corpus import stopwords

def bootstrap_nltk() -> None:
    """
    run once to initialize nltk resources

    (developed iteratively)
    """
    import nltk

    nltk.download("punkt")
    nltk.download("stopwords")

class AbsCorpus(ABC):
    @abstractmethod
    def __init__(self, file: str) -> None:
        raise NotImplementedError(
            "Corpus method must implement init method which should init 'file' and 'corpus'"
        )

    @abstractmethod
    def load_corpus(self) -> None:
        raise NotImplementedError("Corpus methods must implement load corpus method")

    @abstractmethod
    def clean_corpus(self) -> None:
        raise NotImplementedError("Corpus methods must implement clean corpus method")

    # @abstractmethod
    # def some_method(self):
    #     raise NotImplementedError("Corpus methods must implement ____ corpus method")

class SimpleCorpus(AbsCorpus):
    def __init__(self, file):
        self.file = file  # for now, a string indicating a full filepath
        self.load_corpus()
        self.clean_corpus()

    def load_corpus(self) -> None:
        self.corpus = ""
        with open(self.file, "rb") as some_txt:
            for line in some_txt.readlines():
                self.corpus = "".join([self.corpus, str(line)])
        self.raw_words = word_tokenize(self.corpus)
        self.sentences = sent_tokenize(self.corpus)

    def clean_corpus(self) -> None:
        sws = stopwords.words("english")
        self.clean_words = []
        no_punc = []
        for w in self.raw_words:
            if w.isalpha():  # only punctuation
                no_punc.append(w.lower())
        for w in no_punc:
            if w not in sws:  # not a stopword
                self.clean_words.append(w)

    def examine_corpus(self):
        print(f"there are {len(self.sentences)} sentences")
        print(f"there are {len(self.clean_words)} words")

        self.fdist = nltk.probability.FreqDist(self.clean_words)
        # print(self.fdist.tabulate())
        for word_count in self.fdist.most_common(250):
            print(word_count)


# taken from https://matplotlib.org/stable/gallery/widgets/slider_demo.html

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.widgets import Slider, Button

# The parametrized function to be plotted

def f(S_w, Lw, Ew, Tw):

    return kOrw*((S_w**Lw)/((S_w**Lw)+Ew*((1-S_w)**Tw)))

#krw = kOrw*((S_w**Lw)/((S_w**Lw)+Ew*((1-S_w)**Tw)))

t = np.linspace(0, 1, 1000)

Swir = 0.1
Sorw = 0.3
kOro = 1
kOrw = 0.6

Sw = np.linspace(Swir, 1-Sorw, 50)

S_w = (Sw - Swir)/(1- Sorw - Swir)

# Define initial parameters

init_Lw = 3
init_Ew = 3
init_Tw = 5

# Create the figure and the line that we will manipulate
fig, ax = plt.subplots()

#ax.plot(Sw, krw, color='b')
#ax.plot(Sw, kro, color='r')

line, = plt.plot(S_w, f(Sw, init_Lw, init_Ew, init_Tw), lw=2)
ax.set_xlabel('Saturation [frac]')

# adjust the main plot to make room for the sliders
plt.subplots_adjust(left=0.25, bottom=0.25)

# Make a horizontal slider to control the frequency.
axLw = plt.axes([0.05, 0.25, 0.0225, 0.63])
Lw_slider = Slider(
    ax=axLw,
    label='L',
    valmin=1,
    valmax=6,
    valinit=init_Lw,
    orientation="vertical"
)
#[0.25, 0.1, 0.65, 0.03],

# Make a vertically oriented slider to control the amplitude
axEw = plt.axes([0.1, 0.25, 0.025, 0.63])
Ew_slider = Slider(
    ax=axEw,
    label="E",
    valmin=1,
    valmax=6,
    valinit=init_Ew,
    orientation="vertical"
)

# Make a vertically oriented slider to control the amplitude
axTw = plt.axes([0.15, 0.25, 0.025, 0.63])
Tw_slider = Slider(
    ax=axTw,
    label="T",
    valmin=1,
    valmax=6,
    valinit=init_Tw,
    orientation="vertical"
)

# The function to be called anytime a slider's value changes
def update(val):
    line.set_ydata(f(S_w, Lw_slider.val, Ew_slider.val, Tw_slider.val))
    fig.canvas.draw_idle()

# register the update function with each slider
Lw_slider.on_changed(update)
Ew_slider.on_changed(update)
Tw_slider.on_changed(update)

# Create a `matplotlib.widgets.Button` to reset the sliders to initial values.
resetax = plt.axes([0.8, 0.05, 0.1, 0.04])
button = Button(resetax, 'Reset', hovercolor='0.975')

def reset(event):
    Lw_slider.reset()
    Ew_slider.reset()
    Tw_slider.reset()
button.on_clicked(reset)

plt.show()


import requests
from bs4 import BeautifulSoup
from itertools import cycle
import argparse
import io
import math

import pandas as pd
from pandas import DataFrame
from sklearn import linear_model
import statsmodels.api as sm
from sklearn.svm import SVR
import matplotlib.pyplot as plt
import numpy as np
import math
import sys

teamWinsByYear = []
overallTeamStats = []
covTeamStats = []
header = ''

def clean(stng):

    output = ''

    for letter in stng:
        if letter != '[' and letter != ']' and letter != "'": #and letter != ' ':
            output += letter

    return output

def getProxies(inURL):

    page = requests.get(inURL)
    soup = BeautifulSoup(page.text, 'html.parser')
    terms = soup.find_all('tr')
    IPs = []

    for x in range(len(terms)):

        term = str(terms[x])

        if '
' in str(terms[x]):
            pos1 = term.find('d>') + 2
            pos2 = term.find('	')

            pos3 = term.find('	') + 9
            pos4 = term.find('	US<')

            IP = term[pos1:pos2]
            port = term[pos3:pos4]

            if '.' in IP and len(port) < 6:
                IPs.append(IP + ":" + port)
                #print(IP + ":" + port)

    return IPs

proxyURL = "https://www.us-proxy.org/"
pxs = getProxies(proxyURL)
proxyPool = cycle(pxs)

def progress(count, total, status=''):

    bar_len = 60
    filled_len = int(round(bar_len * count / float(total)))

    percents = round(100.0 * count / float(total), 1)
    bar = '=' * filled_len + '-' * (bar_len - filled_len)

    sys.stdout.write('[%s] %s%s ...%s\r' % (bar, percents, '%', status))
    sys.stdout.flush()

def getTeamNames(year):

    teams = []

    page = requests.get('https://www.basketball-reference.com/leagues/NBA_' + str(year) + '_standings.html', proxies = {"http": next(proxyPool)})
    soup = str(BeautifulSoup(page.text, 'html.parser'))

    for x in range(1, 31):

        indexTo = soup.find('data-stat="ranker" csk="' + str(x) + '"')

        rawTeamStats = soup[indexTo + 80 : indexTo + 500]

        rawTeamSymbol = rawTeamStats[rawTeamStats.find('/teams') + 7 : rawTeamStats.find('/teams') + 15]
        teamSymbol = rawTeamSymbol[0 : rawTeamSymbol.find('/')]

        teamName = rawTeamStats[rawTeamStats.find('>') + 1 : rawTeamStats.find('</')]

        teams.append([teamName, teamSymbol])

    return teams

def generateDataset(year, seasons, datasetName):

    global overallTeamStats
    global covTeamStats
    global header

    overallTeamStats = []
    progressCounter = 0

    for x in range(30):

        yr = year

        for y in range(seasons):

            rawTeams = getTeamNames(yr)
            teams = [sorted(sublist) for sublist in rawTeams]
            teams.sort()

            page = requests.get('https://www.basketball-reference.com/teams/' + teams[x][0] + '/' + str(yr) + '.html', proxies = {"http": next(proxyPool)})
            soup = str(BeautifulSoup(page.text, 'html.parser'))

            teamName = teams[x][1]

            try:

                #Wins
                rawWins = soup[soup.find('data-stat="wins" >') + 15 : soup.find('data-stat="wins" >') + 30]
                wins = float(rawWins[rawWins.find('>') + 1 : rawWins.find('<')])
                #print('Wins', wins)

                #Losses
                rawLosses = soup[soup.find('data-stat="losses" >') + 15 : soup.find('data-stat="losses" >') + 30]
                losses = float(rawLosses[rawLosses.find('>') + 1 : rawLosses.find('<')])
                #print('Losses', losses)

                #Points / Game
                rawPts = soup[soup.find('data-stat="pts_per_g" >') + 15 : soup.find('data-stat="pts_per_g" >') + 30]
                ptsPerGame = float(rawPts[rawPts.find('>') + 1 : rawPts.find('<')])
                #print('Points / Game', ptsPerGame)

                #Offensive Rating
                rawOffRating = soup[soup.find('data-stat="off_rtg" >') + 16 : soup.find('data-stat="off_rtg" >') + 27]
                offRating = float(rawOffRating[rawOffRating.find('>') + 1 : rawOffRating.find('<')])
                #print('Off Rating', offRating)

                #Defensive Rating
                rawDefRating = soup[soup.find('data-stat="def_rtg" >') + 16 : soup.find('data-stat="def_rtg" >') + 27]
                defRating = float(rawDefRating[rawDefRating.find('>') + 1 : rawDefRating.find('<')])
                #print('Def Rating', defRating)

                #Pace
                rawPace = soup[soup.find('data-stat="pace" >') + 16 : soup.find('data-stat="pace" >') + 27]
                pace = float(rawPace[rawPace.find('>') + 1 : rawPace.find('<')])
                #print('Pace', pace)

                #FG%
                rawFgPCT = soup[soup.find('data-stat="fg_pct" >') + 18 : soup.find('data-stat="fg_pct" >') + 25]
                fgPCT = float(rawFgPCT[rawFgPCT.find('>') + 1 : rawFgPCT.find('<')])
                #print('FG%', fgPCT)

                #3P%
                raw3PCT = soup[soup.find('data-stat="fg3_pct" >') + 16 : soup.find('data-stat="fg3_pct" >') + 27]
                fg3PCT = float(raw3PCT[raw3PCT.find('>') + 1 : raw3PCT.find('<')])
                #print('3P%', fg3PCT)

                #FT%
                rawFTPCT = soup[soup.find('data-stat="ft_pct" >') + 16 : soup.find('data-stat="ft_pct" >') + 27]
                ftPCT = float(rawFTPCT[rawFTPCT.find('>') + 1 : rawFTPCT.find('<')])
                #print('FT%', ftPCT)

                #Offensive Rebounds / Game
                rawORB = soup[soup.find('data-stat="orb_per_g" >') + 10 : soup.find('data-stat="orb_per_g" >') + 30]
                orbPerGame = float(rawORB[rawORB.find('>') + 1 : rawORB.find('<')])
                #print('Offensive Rebounds / Game', orbPerGame)

                #Defensive Rebounds / Game
                rawDRB = soup[soup.find('data-stat="drb_per_g" >') + 10 : soup.find('data-stat="drb_per_g" >') + 30]
                drbPerGame = float(rawDRB[rawDRB.find('>') + 1 : rawDRB.find('<')])
                #print('Defensive Rebounds / Game', drbPerGame)

                #Assists / Game
                rawAstPG = soup[soup.find('data-stat="ast_per_g" >') + 15 : soup.find('data-stat="ast_per_g" >') + 30]
                astPerGame = float(rawAstPG[rawAstPG.find('>') + 1 : rawAstPG.find('<')])
                #print('Assists / Game', astPerGame)

                #Steals / Game
                rawStl = soup[soup.find('data-stat="stl_per_g" >') + 15 : soup.find('data-stat="stl_per_g" >') + 30]
                stlPerGame = float(rawStl[rawStl.find('>') + 1 : rawStl.find('<')])
                #print('Steals / Game', stlPerGame)

                #Blocks / Game
                rawBlk = soup[soup.find('data-stat="blk_per_g" >') + 15 : soup.find('data-stat="blk_per_g" >') + 30]
                blkPerGame = float(rawBlk[rawBlk.find('>') + 1 : rawBlk.find('<')])
                #print('Blocks / Game', blkPerGame)

                #Turnovers / Game
                rawTO = soup[soup.find('data-stat="tov_per_g" >') + 15 : soup.find('data-stat="tov_per_g" >') + 30]
                toPerGame = float(rawTO[rawTO.find('>') + 1 : rawTO.find('<')])
                #print('Turnovers / Game', toPerGame)

                #Fouls / Game
                rawF = soup[soup.find('data-stat="pf_per_g" >') + 15 : soup.find('data-stat="pf_per_g" >') + 30]
                foulsPerGame = float(rawF[rawF.find('>') + 1 : rawF.find('<')])
                #print('Fouls / Game', foulsPerGame)

                overallTeamStats.append([teamName, yr, wins, losses, ptsPerGame, offRating, defRating, pace, fgPCT, fg3PCT, ftPCT, orbPerGame, drbPerGame, astPerGame, stlPerGame, blkPerGame, toPerGame, foulsPerGame])
                covTeamStats.append([teamName, yr, wins, losses, ptsPerGame, offRating, defRating, pace, fgPCT, fg3PCT, ftPCT, orbPerGame, drbPerGame, astPerGame, stlPerGame, blkPerGame, toPerGame, foulsPerGame])

            except:

                print('Error', teamName, yr)
                print()

            progress(progressCounter, len(teams) * seasons, status='Loading...')
            progressCounter += 1
            yr -= 1

    print('\nDone!')

    MyFile = open(saveTo + datasetName + '.csv','w')

    header = 'Team Name,Year,Wins,Losses,PPG,Off Rating,Def Rating,Pace,FG%,FG3%,FT%,Off RBG,Def RBG,Assists / Game,Steals / Game,Blocks / Game,Turnovers / Game,Fouls / Game' + '\n'

    MyFile.write(header)

    for row in overallTeamStats:
        MyFile.write(clean(str(row)))
        MyFile.write('\n')

    MyFile.close()

    print('\nSaved as ' + saveTo + datasetName + '.csv')

def applyModel(trainSet, testSet):

    #Load Training Set
    df = pd.read_csv(saveTo + trainSet + '.csv')

    X = df[['PPG', 'Off Rating', 'Def Rating', 'FG%', 'FG3%', 'Def RBG', 'Assists / Game']]
    Y = df['Wins']

    regr = linear_model.LinearRegression()
    regr.fit(X, Y)

    #print('Intercept: \n', regr.intercept_)
    #print('Coefficients: \n', regr.coef_)

    #Predict Team Wins
    testData = pd.read_csv(saveTo + testSet + '.csv')

    predictions = regr.predict(testData[['PPG', 'Off Rating', 'Def Rating', 'FG%', 'FG3%', 'Def RBG', 'Assists / Game']])

    plt.plot(testData['Wins'], label = 'Actual Wins')
    plt.plot(predictions, label = 'Predicted Wins')
    plt.title('NBA Team Win Predictions (' + str(testYear) + '-' + str(testYear - testSeasons + 1) + ' Seasons)')
    plt.legend()
    plt.show()

    teamWins = testData.values.tolist()

    errSum = 0

    print("\n\nTeam Name, Year, Actual Wins, Predicted Wins")

    for x in range(len(testData)):
        print(teamWins[x][0], teamWins[x][1], teamWins[x][2], math.ceil(predictions[x]), 'Error: (' + str(abs(teamWins[x][2] - math.ceil(predictions[x]))) + ')\n')
        errSum += abs(teamWins[x][2] - math.ceil(predictions[x]))

    print('Avg Error', errSum / len(testData))

def findCorrelations():

    global covTeamStats
    global header

    correlations = []
    stat = header[header.find('PPG') : ]
    print()

    for y in range(4, 17):

        wins = []
        stats = []

        for x in range(len(covTeamStats)):

            wins.append(covTeamStats[x][2])
            stats.append(covTeamStats[x][y])
            #print(covTeamStats[x][2], covTeamStats[x][5])

        wins = np.array(wins)
        stats = np.array(stats)
        corre = np.corrcoef(wins, stats)

        correlations.append([stat[0 : stat.find(',')], corre[0][1]])

        print(stat[0 : stat.find(',')])
        print(corre[0][1])
        print()

        stat = stat[stat.find(',') + 1 : ]

    correlations = sorted(correlations,key=lambda l:l[1], reverse=True)

    print(correlations)

saveTo = 'C:\\Users\\faiza\\OneDrive\\Desktop\\NBAData\\' #Change this to the location in which you want to save your files

#Generate Training Set
year = int(input("Enter starting season: "))
seasons = int(input("Enter the number of seasons to gather data from: "))

print('Gathering training data from the ' + str(year) + " - " + str(year - seasons + 1) + " NBA seasons\n")

trainingSet = 'NBAMLTrainingSet' + str(year) + str(seasons) + "Y"

generateDataset(year, seasons, trainingSet)

#findCorrelations()

#Generate Testing Set
testYear = int(input("\nEnter starting season for your testing set: "))
testSeasons = int(input("Enter how many seasons you want to test: "))

print('Gathering testing data from the ' + str(testYear) + " - " + str(testYear - testSeasons + 1) + " NBA seasons\n")

testingSet = "NBAMLTestingSet" + str(testYear) + str(testSeasons) + "Y"

generateDataset(testYear, testSeasons, testingSet)

#Apply Regression Model
applyModel(trainingSet, testingSet)