import sys
import numpy as np
import random
from collections import OrderedDict

molcounter = 0
molbeads = 0
allbeads = 0
nowmol = 0
data = np.array([])
minpositions = np.array([])
maxpositions = np.array([])
molrange = []
boxsize = np.array([])
boxdata = np.array([])
molsize = np.array([])
waku = np.array([])
np.set_printoptions(precision=3)
positions = OrderedDict()
molpositions = OrderedDict()
r = sys.argv[2]
d = sys.argv[3]

def rotatex(deg):
    r = np.radians(deg)
    C = np.cos(r)
    S = np.sin(r)
    R_x = np.matrix((
        (1, 0, 0),
        (0, C, -S),
        (0, S, C)
    ))

    return R_x
def rotatey(deg):
    r = np.radians(deg)
    C = np.cos(r)
    S = np.sin(r)
    R_y = np.matrix((
        (C, 0, -S),
        (0, 1, 0),
        (S, 0, C)
    ))

    return R_y
def rotatez(deg):
    r = np.radians(deg)
    C = np.cos(r)
    S = np.sin(r)
    R_z = np.matrix((
        (C, -S, 0),
        (S, C, 0),
        (0, 0, 1)
    ))

    return R_z

def kaitten():
    global data
    global molbeads
    global molcounter
    global maxpositions
    global minpositions
    global nowmol
    counter = 0

    xflag= 0
    yflag = 0
    zflag = 0
    kaku = random.randint(0,360)
    for i in range(len(data)):

        if xflag ==1:
            a = np.array([data[i][0],data[i][1],data[i][2]])
            Rx = rotatex(kaku)
            b = np.dot(Rx,a)
            #b = np.array(b,dtype=np.uint8)
            b = np.reshape(b,(1,3))
            data[i] = b

        if yflag ==1:
            a = np.array([data[i][0],data[i][1],data[i][2]])
            Ry = rotatey(kaku)
            b = np.dot(Ry,a)
            #b = np.array(b,dtype=np.uint8)
            b = np.reshape(b,(1,3))
            data[i] = b

        if zflag == 1:
            a = np.array([data[i][0],data[i][1],data[i][2]])
            Rz = rotatez(kaku)
            b = np.dot(Rz,a)
            #b = np.array(b,dtype=np.uint8)
            b = np.reshape(b,(1,3))
            data[i] = b
        counter = counter + 1
        if counter == molbeads:
            kaku = random.randint(0,90)
            xflag = random.randint(0,1)
            yflag = random.randint(0,1)
            zflag = random.randint(0,1)
            #print(xflag,yflag,zflag)
            counter = 0

def maketmp():
    global molbeads
    np.set_printoptions(precision=3)
    f = open(sys.argv[1],"r")
    ft = open("tmp.pdb","w")
    global molcounter
    for line in f.readlines():
        l = line.split()
        for i in range(len(l)):
            if l[0] == "END":
                molcounter = molcounter +1
                if molcounter == 1:
                    tmp = molbeads

            else:
                ft.write(l[i])
                if i < len(l)-1:
                    ft.write(" ")
                else:
                    ft.write("\n")

        molbeads = molbeads + 1

    ft.write("END")
    ft.close()
    molbeads = tmp-1

def makedata():
    global allbeads
    global data
    global boxsize
    ftmp2 = open("tmp.pdb","r")
    for line in ftmp2.readlines():
        tmpline = line.split()
        if tmpline[0] == "ATOM":
            x,y,z  = float(tmpline[6]),float(tmpline[7]),float(tmpline[8])
            data =np.append(data,[x,y,z])
            allbeads = allbeads +1
        if tmpline[0] == "CRYST1":
            boxsize = np.append(boxsize,[float(tmpline[1])/2,(-1)*float(tmpline[1])/2])
            boxsize = np.append(boxsize,[float(tmpline[2])/2,(-1)*float(tmpline[2])/2])
            boxsize = np.append(boxsize,[float(tmpline[3])/2,(-1)*float(tmpline[3])/2])

    data = np.reshape(data,(allbeads,3))
    boxsize = np.reshape(boxsize,(3,2))
    ftmp2.close()
    #print(data)
def dictlize():
    global data
    global positions
    global d
    d = float(d)
    for i in range(len(data)):
        crd_i = [int(x/d) for x in data[i]]
        positions[tuple(crd_i)] = 1

    #print(positions)
def get_min_and_max_positions():
    global molbeads
    global molcounter
    global maxpositions
    global minpositions
    maxpositions = np.array([])
    minpositions = np.array([])
    minx,miny,minz = 10000,10000,10000
    maxx,maxy,maxz = -10000,-10000,-10000
    counter = 0

    for i in range(len(data)):
        if data[i][0] < minx:
            minx = data[i][0]
        if data[i][0] > maxx:
            maxx = data[i][0]
        if data[i][1] < miny:
            miny = data[i][1]
        if data[i][1] > maxy:
            maxy = data[i][1]
        if data[i][2] < minz:
            minz = data[i][2]
        if data[i][2] > maxz:
            maxz = data[i][2]
        counter = counter + 1
        if counter == molbeads and i>0:
            minpositions = np.append(minpositions,[minx,miny,minz])
            maxpositions = np.append(maxpositions,[maxx,maxy,maxz])
            minx,miny,minz = 10000,10000,10000
            maxx,maxy,maxz = -10000,-10000,-10000
            counter = 0

    minpositions = np.reshape(minpositions,(molcounter,3))
    maxpositions = np.reshape(maxpositions,(molcounter,3))
    #print("max",maxpositions)
    #print("min",minpositions)

def randomize():
    global molcounter
    global molbeads
    global data
    global boxsize
    global r
    global minpositions
    get_min_and_max_positions()
    kaitten()
    get_min_and_max_positions()
    mol = 0
    r = float(r)
    randx = random.uniform(boxsize[0][1]+r,boxsize[0][0]-r)
    randy = random.uniform(boxsize[1][1]+r,boxsize[1][0]-r)
    randz = random.uniform(boxsize[2][1]+r,boxsize[2][0]-r)
    #print(randx,randy,randz)
    counter = 0
    for i in range(len(data)):
        #print(mol,len(data))
        data[i][0] = data[i][0] + (randx-minpositions[mol][0])
        data[i][1] = data[i][1] + (randy-minpositions[mol][1])
        data[i][2] = data[i][2] + (randz-minpositions[mol][2])
        counter = counter +1
        #print(data[i][0],data[i][1],data[i][2])
        if i > 0 and counter == molbeads:
            randx = random.uniform(boxsize[0][1]+r,boxsize[0][0]-r)
            randy = random.uniform(boxsize[1][1]+r,boxsize[1][0]-r)
            randz = random.uniform(boxsize[2][1]+r,boxsize[2][0]-r)
            #print(randx,randy,randz)
            #print("w",mol,i)
            mol = mol+1
            counter = 0

    get_min_and_max_positions()

def makeoutput():
    global nowmol
    global data
    global molcounter
    global molbeads
    mols = 0
    flag =0
    counter = 0
    #print(data)

    fp = open("allEND_input.pdb","w")
    ft = open("tmp.pdb","r")
    fpp = open("withoutEND_input.pdb","w")
    fs = open("short.pdb","w")
    np.set_printoptions(precision=3)
    for line in ft.readlines():
        l = line.split()
        if l[0] == "CRYST1":
            for i in range(len(l)):
                fp.write(str(l[i]))
                fpp.write(str(l[i]))
                fs.write(str(l[i]))
                if i <len(l)-1:
                    if i > 2 and i <6:
                        fp.write("  ")
                        fpp.write("  ")
                        fs.write("  ")
                    elif i <3:
                        fp.write(" ")
                        fpp.write(" ")
                        fs.write(" ")
                    elif i > 5 and i <8:
                        fp.write(" ")
                        fpp.write(" ")
                        fs.write(" ")
                    elif i == 8:
                        fp.write("           ")
                        fpp.write("           ")
                        fs.write("           ")

                else:
                    fp.write("\n")
                    fpp.write("\n")
                    fs.write("\n")
        if l[0] == "ATOM":
            flag = 0
            for k in range(6,9):
                l[k] = format(round(float(data[mols*molbeads+counter][k-6]),5),'.5f')
                if float(l[6]) == 0:
                    l[k] = format(round(float(data[mols*molbeads+counter][k-6]),3),'.5f')
                if float(l[k]) >0:
                    if float(l[k]) < 10:
                        l[k] = format(round(float(data[mols*molbeads+counter][k-6]),5),'.5f')
                    elif float(l[k]) <100.0:
                        l[k] = format(round(float(data[mols*molbeads+counter][k-6]),4),'.4f')
                    else:
                        l[k] = format(round(float(data[mols*molbeads+counter][k-6]),3),'.3f')

                if float(l[k]) <0:
                    if float(l[k]) >-10.0:
                        l[k] = format(round(float(data[mols*molbeads+counter][k-6]),4),'.4f')
                    elif float(l[k]) > -100.0:
                        l[k] = format(round(float(data[mols*molbeads+counter][k-6]),3),'.3f')
                    elif float(l[k]) >-1000.0:
                        l[k] = format(round(float(data[mols*molbeads+counter][k-6]),2),'.2f')

            #l[6] = format(round(float(data[mols*molbeads+counter][0]),3),'.3f')
            counter = counter +1
            for i in range(len(l)):
                if i == 1:
                    if int(l[i]) <10:
                        fp.write("      ")
                        fpp.write("      ")
                        if mols == 0:
                            fs.write("      ")

                    elif int(l[i]) <100:
                        if int(l[i]) >9:
                            fp.write("     ")
                            fpp.write("     ")
                            if mols == 0:
                                fs.write("     ")

                    elif int(l[i]) <1000:
                        if int(l[i]) > 99:
                            fp.write("    ")
                            fpp.write("    ")
                            if mols == 0:
                                fs.write("    ")

                    elif int(l[i]) <10000:
                        if int(l[i]) > 999:
                            fp.write("   ")
                            fpp.write("   ")
                            if mols == 0:
                                fs.write("   ")

                    elif int(l[i]) <100000:
                        if int(l[i]) > 9999:
                            fp.write("  ")
                            fpp.write("  ")
                            if mols == 0:
                                fs.write("  ")

                if i == 2 or i == 3 or i == 9 or i == 10:
                    fp.write("  ")
                    fpp.write("  ")
                    if mols == 0:
                        fs.write("  ")

                if i == 4 or i == 7 or i == 8:
                    fp.write(" ")
                    fpp.write(" ")
                    if mols == 0:
                        fs.write(" ")

                if i == 5:
                    if int(l[i]) <10:
                        fp.write("   ")
                        fpp.write("   ")
                        if mols == 0:
                            fs.write("   ")

                    elif int(l[i]) <100:
                        if int(l[i]) >9:
                            fp.write("  ")
                            fpp.write("  ")
                            if mols == 0:
                                fs.write("  ")

                    elif int(l[i]) <1000:
                        if int(l[i]) > 99:
                            fp.write(" ")
                            fpp.write(" ")
                            if mols == 0:
                                fs.write(" ")

                if i == 6:
                    fp.write("     ")
                    fpp.write("     ")
                    if mols == 0:
                        fs.write("     ")

                if i == 11:
                    fp.write("      ")
                    fpp.write("      ")
                    if mols == 0:
                        fs.write("      ")

                fp.write(str(l[i]))
                fpp.write(str(l[i]))
                if mols == 0:
                    fs.write(str(l[i]))

                if i  ==len(l)-1:
                    fp.write("\n")
                    fpp.write("\n")
                    if mols == 0:
                        fs.write("\n")

        if counter == molbeads:
            if mols == 0:
                fs.write("END")
            mols = mols+1
            if mols == molcounter:
                fpp.write("END")
                fp.write("END")
            else:
                fp.write("END\n")

            counter = 0

    ft.close()

def changedata(mol):
    global molbeads
    global data

    xflag = random.randint(0,1)
    yflag = random.randint(0,1)
    zflag = random.randint(0,1)
    randx = random.uniform(-10,10)
    randy = random.uniform(-10,10)
    randz = random.uniform(-10,10)

    for i in range(mol*molbeads,(mol+1)*molbeads):
        if xflag == 1:
            data[i][0] = data[i][0] + randx
        if yflag == 1:
            data[i][1] = data[i][1] + randy
        if zflag == 1:
            data[i][2] = data[i][2] + randz

def getmolpositions(mol):
    global molbeads
    global molcounter
    global molpositions
    global data
    global d
    molpositions = OrderedDict()

    for i in range(mol*molbeads,(mol+1)*molbeads):
        crd_i = [int(x/d) for x in data[i]]
        molpositions[tuple(crd_i)] = 1

def checkdata():
    global data
    global molbeads
    global molcounter
    global positions
    global boxsize
    global molpositions
    global d
    flag = 1

    while flag == 1:
        flag = 0
        for i in range(molcounter-1):
            getmolpositions(i)
            for j in range(i+1,molcounter):
                for k in range(molbeads):
                    tmp = np.array([int(data[j*molbeads+k][0]/d),int(data[j*molbeads+k][1]/d),int(data[j*molbeads+k][2]/d)])
                    tmp = tuple(tmp)

                    if tmp in molpositions:
                        flag =1
                        mol = j
                        print("modifying",i, mol)
                        changedata(mol)
                        dictlize()

        for i in range(0,len(data)):
            if data[i][0] > boxsize[0][0] or data[i][0] < boxsize[0][1]:
                randomize()
                print("error in x")
                print("data,max,data,min",data[i][0],boxsize[0][0],data[i][0],boxsize[0][1])
                dictlize()
                flag = 1
                break
            if data[i][1] > boxsize[1][0] or data[i][1] < boxsize[1][1]:
                randomize()
                print("error in y")
                dictlize()
                flag = 1
                break
            if data[i][2] > boxsize[2][0] or data[i][2] < boxsize[2][1]:
                randomize()
                print("error in z")
                dictlize()
                flag = 1
                break

def make_atom_groups():
    global molbeads
    global molcounter
    ft = open("tmp.pdb","r")
    fa = open("atom_groups.inp","w")

    for i in range(molcounter):
        fa.write("mol")
        fa.write(str(i+1))
        fa.write(" ")
        fa.write(str(i*molbeads+1))
        fa.write("-")
        fa.write(str((i+1)*molbeads))
        fa.write("\n")
    fa.write("all")
    fa.write(" ")
    fa.write("1-")
    fa.write(str((i+1)*molbeads))

if __name__ == '__main__':
    print("attention at the boxsize of input files, recommend you to set x:1000 y:1000 z:1000")
    maketmp()
    makedata()
    randomize()
    dictlize()
    checkdata()
    makeoutput()
    make_atom_groups()
    print("molcounter, molbeads",molcounter,molbeads)


from collections import defaultdict, OrderedDict
import re

from pycldf import iter_datasets, Sources
from clldutils.misc import lazyproperty, nfilter
from clld.db.models import common
from clld.db.fts import tsvector
from clld.db.meta import DBSession

from dictionaria.lib.ingest import BaseDictionary
from dictionaria import models

def read_media_table(cldf):
    media = {}

    if cldf.get('MediaTable'):
        tablename = 'MediaTable'
    elif cldf.get('media.csv'):
        tablename = 'media.csv'
    else:
        return media

    property_map = {
        cldf[tablename, prop].name: fallback
        for prop, fallback in (
            ('id', 'ID'),
            ('name', 'Filename'),
            ('description', 'Description'),
            ('languageReference', 'Language_ID'),
            ('mediaType', 'mimetype'),
            ('downloadUrl', 'URL'))
        if cldf.get((tablename, prop))}

    for row in cldf[tablename]:
        media_item = {property_map.get(k) or k: v for k, v in row.items()}
        media[media_item['ID']] = media_item

    return media

def get_foreign_keys(ds, from_table):
    """
    :param ds: A CLDF Dataset object.
    :param from_table: A csvw.metadata.Table object.
    :return: a `dict` mapping CLDF component names to `list`s of column names in `from_table`,
    which are foreign keys into the component.
    """
    res = defaultdict(list)
    for component in ['EntryTable', 'SenseTable', 'ExampleTable']:
        ref = ds.get(component)
        if not ref:
            continue
        for fk in from_table.tableSchema.foreignKeys:
            if fk.reference.resource == ref.url and \
                    fk.reference.columnReference == ref.tableSchema.primaryKey and \
                    len(fk.columnReference) == 1:
                res[component].append(fk.columnReference[0])
    return res

def get_labels(type_, table, colmap, submission, exclude=None):
    labels = OrderedDict(submission.props.get('labels', []))
    exclude = exclude or []
    exclude.extend([colmap.get('mediaReference', 'Media_IDs'), 'ZCom1'])
    res = OrderedDict()
    for col in table.tableSchema.columns:
        if col.name not in exclude and col.name not in colmap.values():
            res[col.name] = (col.titles.getfirst() if col.titles else col.name, False)
    map_name = '{0}_map'.format(type_)
    if submission.props.get(map_name):
        for key, label in labels.items():
            if key in submission.props[map_name]:
                res[submission.props[map_name][key]] = (
                    label, key in submission.props.get('process_links_in_labels', []))
    order_name = '{0}_custom_order'.format(type_)
    if submission.props.get(order_name):
        res = OrderedDict([(k, res[k]) for k in submission.props[order_name] if k in res])
    return res

class Dictionary(BaseDictionary):
    @lazyproperty
    def cldf(self):
        try:
            return next(iter_datasets(self.dir))
        except StopIteration:
            raise ValueError('no cldf metadata found in {}'.format(self.dir))

    def add_refs(self, data, table, row, obj, labels):
        if table == 'EntryTable':
            model, kw = models.WordReference, dict(word=obj)
        elif table == 'SenseTable':
            model, kw = models.MeaningReference, dict(meaning=obj)
        else:
            raise ValueError(table)
        refs_col = self.cldf.get((table, 'source'))
        if refs_col:
            for sid, context in map(self.cldf.sources.parse, row.get(refs_col.name, [])):
                if sid in data['DictionarySource']:
                    DBSession.add(model(
                        source=data['DictionarySource'][sid],
                        description=labels.get(context, (context, None))[0], **kw))

    def load(self,
             submission,
             data,
             vocab,
             lang,
             comparison_meanings,
             labels):
        def id_(oid):
            return '%s-%s' % (submission.id, oid)

        print('######\n a CLDF dict! \n####')

        media = read_media_table(self.cldf)

        metalanguages = submission.props.get('metalanguages', {})
        entries = self.cldf['EntryTable']
        colmap = {k: self.cldf['EntryTable', k].name
                  for k in [
                      'id', 'headword', 'partOfSpeech', 'languageReference',
                      'mediaReference', 'source']
                  if self.cldf.get(('EntryTable', k))}
        fks = get_foreign_keys(self.cldf, entries)
        elabels = get_labels('entry', entries, colmap, submission, exclude=fks['EntryTable'][:])

        for lemma in entries:
            if not lemma[colmap['headword']]:
                continue
            oid = lemma.pop(colmap['id'])
            word = data.add(
                models.Word,
                oid,
                id=id_(oid),
                name=lemma.pop(colmap['headword']),
                pos=lemma.pop(colmap['partOfSpeech']),
                dictionary=vocab,
                language=lang)
            DBSession.flush()

            media_ids = set(
                lemma.get(colmap.get('mediaReference', 'Media_IDs'))
                or [])
            files = [(md5, media[md5]) for md5 in media_ids if md5 in media]
            for md5, spec in sorted(
                files,
                key=lambda i: i[1].get(submission.props.get('media_order', 'Description')) or i[1]['ID']
            ):
                submission.add_file(None, md5, common.Unit_files, word, spec)

            self.add_refs(data, 'EntryTable', lemma, word, elabels)

            for index, (key, label) in enumerate(elabels.items()):
                label, with_links = label
                if lemma.get(key):
                    DBSession.add(common.Unit_data(
                        object_pk=word.pk,
                        key=label.replace('_', ' '),
                        value=lemma[key],
                        ord=index,
                        jsondata=dict(with_links=with_links)))

        DBSession.flush()

        #
        # Now that all entries are in the DB and have primary keys, we can create the
        # self-referential links:
        #
        fullentries = defaultdict(list)
        for lemma in entries:
            fullentries[lemma[colmap['id']]].extend(list(lemma.items()))
            word = data['Word'][lemma[colmap['id']]]
            for col in fks['EntryTable']:
                col = self.cldf['EntryTable', col]
                label = col.titles.getfirst() if col.titles else col.name
                if label == 'Entry_IDs':
                    label = 'See also'
                label = label.replace('_', ' ')
                for lid in lemma[col.name] or []:
                    if lid not in data['Word']:
                        print('missing entry ID: {0}'.format(lid))
                    else:
                        DBSession.add(models.SeeAlso(
                            source_pk=word.pk, target_pk=data['Word'][lid].pk, description=label))

        sense2word = {}
        colmap = {k: self.cldf['SenseTable', k].name
                  for k in [
                      'id', 'entryReference', 'description', 'mediaReference',
                      'source']
                  if self.cldf.get(('SenseTable', k))}
        fks = get_foreign_keys(self.cldf, self.cldf['SenseTable'])

        slabels = get_labels(
            'sense',
            self.cldf['SenseTable'],
            colmap,
            submission,
            exclude=['alt_translation1', 'alt_translation2'] + fks['EntryTable'][:])

        for sense_index, sense in enumerate(self.cldf['SenseTable']):
            fullentries[sense[colmap['entryReference']]].extend(list(sense.items()))
            sense2word[sense[colmap['id']]] = sense[colmap['entryReference']]
            try:
                w = data['Word'][sense[colmap['entryReference']]]
            except KeyError:
                print('missing entry: {0}'.format(sense[colmap['entryReference']]))
                continue
            dsc = sense[colmap['description']]
            if not isinstance(dsc, list):
                dsc = [dsc]
            kw = dict(
                id=id_(sense[colmap['id']]),
                name='; '.join(nfilter(dsc)),
                semantic_domain=sense.pop('Semantic_Domain', None),
                ord=sense_index,
                word=w)
            if 'alt_translation1' in sense and metalanguages.get('gxx'):
                kw['alt_translation1'] = sense['alt_translation1']
                kw['alt_translation_language1'] = metalanguages.get('gxx')
            if 'alt_translation2' in sense and metalanguages.get('gxy'):
                kw['alt_translation2'] = sense['alt_translation2']
                kw['alt_translation_language2'] = metalanguages.get('gxy')
            m = data.add(models.Meaning, sense[colmap['id']], **kw)
            DBSession.flush()

            self.add_refs(data, 'SenseTable', sense, m, slabels)

            for index, (key, label) in enumerate(slabels.items()):
                label, with_links = label
                if sense.get(key):
                    DBSession.add(models.Meaning_data(
                        object_pk=m.pk,
                        key=label.replace('_', ' '),
                        value=sense[key],
                        ord=index,
                        jsondata=dict(with_links=with_links)))

            concepticon_field = sense.get('Concepticon_ID') or ''
            concepticon_ids = [
                elem.strip()
                for elem in concepticon_field.split(';')
                if elem.strip()]
            for i, concepticon_id in enumerate(concepticon_ids):
                match = re.fullmatch(r'([^]]*)\s*\[(\d+)\]', concepticon_id)
                if not match:
                    continue
                _, cid = match.groups()
                if cid not in comparison_meanings:
                    continue
                concept = comparison_meanings[cid]

                vsid = '%s-%s' % (lang.id, concept)
                vs = data['ValueSet'].get(vsid)
                if not vs:
                    vs = data.add(
                        common.ValueSet, vsid,
                        id=vsid,
                        language=lang,
                        contribution=vocab,
                        parameter_pk=concept)

                DBSession.add(models.Counterpart(
                    id='{0}-{1}'.format(m.id, i), name=w.name, valueset=vs, word=w))

            DBSession.flush()
            media_ids = set(
                sense.get(colmap.get('mediaReference', 'Media_IDs'))
                or [])
            files = [(md5, media[md5]) for md5 in media_ids if md5 in media]
            for md5, spec in sorted(
                files,
                key=lambda i: i[1].get(submission.props.get('media_order', 'Description')) or i[1]['ID']
            ):
                submission.add_file(None, md5, models.Meaning_files, m, spec)

            for col in fks['EntryTable']:
                col = self.cldf['SenseTable', col]
                if col.name == colmap['entryReference']:
                    continue
                label = col.titles.getfirst() if col.titles else col.name
                label = label.replace('_', ' ')
                entry_ids = sense[col.name]
                if entry_ids:
                    if not isinstance(entry_ids, list):
                        entry_ids = [entry_ids]
                    for eid in entry_ids:
                        if eid not in data['Word']:
                            print('missing entry ID: {0}'.format(eid))
                        else:
                            DBSession.add(models.Nym(
                                source_pk=m.pk, target_pk=data['Word'][eid].pk, description=label))

        colmap = {k: self.cldf['ExampleTable', k].name
                  for k in ['id', 'primaryText', 'translatedText']
                  if self.cldf.get(('ExampleTable', k))}
        for ex in self.cldf.get('ExampleTable', ()):
            #
            # FIXME: Detect the column with sense IDs by looking at the foreign keys!
            #
            mids = ex.get('Senses') or ex.get('Sense_IDs', [])
            if not isinstance(mids, list):
                mids = mids.split(' ; ')
            for mid in mids:
                if mid not in data['Meaning']:
                    continue
                if mid in sense2word:
                    fullentries[sense2word[mid]].extend(list(ex.items()))
                    models.MeaningSentence(
                        meaning=data['Meaning'][mid],
                        sentence=data['Example'][ex[colmap['id']]])
                else:
                    print('missing sense: {0}'.format(mid))

        for wid, d in fullentries.items():
            if wid in data['Word']:
                data['Word'][wid].fts = tsvector(
                    '; '.join('{0}: {1}'.format(k, v) for k, v in d if v))


import textwrap
from pathlib import Path

from pyfomod import fomod, parser

PACKAGE_PATH = Path(__file__).parent / "package_test"
INFO_PATH = Path(PACKAGE_PATH, "fomod", "info.xml")
CONF_PATH = Path(PACKAGE_PATH, "fomod", "moduleconfig.xml")

def test_preserve_data(tmp_path):
    with INFO_PATH.open() as info_file:
        orig_info = info_file.read()
    with CONF_PATH.open() as conf_file:
        orig_conf = conf_file.read()
    root = parser.parse(str(PACKAGE_PATH))
    parser.write(root, str(tmp_path))
    info_path = tmp_path / "fomod" / "info.xml"
    with info_path.open() as info_file:
        new_info = info_file.read()
    conf_path = tmp_path / "fomod" / "moduleconfig.xml"
    with conf_path.open() as conf_file:
        new_conf = conf_file.read()
    assert orig_info == new_info
    assert orig_conf == new_conf

def test_parse(tmp_path):
    root = parser.parse(str(PACKAGE_PATH))
    tuple_root = parser.parse((str(INFO_PATH), str(CONF_PATH)))
    assert root.to_string() == tuple_root.to_string()
    assert root._info.to_string() == tuple_root._info.to_string()
    content = textwrap.dedent(
        """\
            <config>
            <!-- comment -->
            </config>"""
    )
    (tmp_path / "fomod").mkdir()
    conf_path = tmp_path / "fomod" / "moduleconfig.xml"
    with conf_path.open("w") as conf_file:
        conf_file.write(content)
        conf_file.write("\n")
    expected = [
        fomod.ValidationWarning(
            "Syntax Error",
            "Element 'config': Missing child element(s). "
            "Expected is ( moduleName ).",
            None,
            critical=True,
        ),
        fomod.ValidationWarning(
            "Comment Detected",
            "There are comments in this fomod, they will be ignored.",
            None,
            critical=True,
        ),
        fomod.ValidationWarning(
            "Missing Info",
            "Info.xml is missing from the fomod subfolder.",
            None,
            critical=True,
        ),
    ]
    warnings = []
    parser.parse((None, str(conf_path)), warnings=warnings)
    assert warnings == expected


print("updating")
import urllib.request as url
def d(url,filename="cache.txt"):

    with open(filename,"wb") as f:

        vz=url.urlopen(url)

        #vz=vz.replace("\\n",'''\n''')
        print(vz)

        f.write(vz.read())

    return vz
try:
  d("https://raw.githubusercontent.com/AwesomeCatStudios/Escape-The-Building-Remix/master/Escape_The_Building_(Four_Rooms).py","Escape_The_Building_(Four_Rooms).py")
except:
    print("Get an internet connection dude!")


import argparse
import os
import time
from torch.backends import cudnn

from data import *
from trainer import *
import _pickle as pickle

def main(args):
	cuda.set_device(int(args.gpu_num))
	cudnn.benchmark = True

	start_time = time.time()

	# if os.path.isfile(args.data_path + 'vocab.pb'):
	# 	vocab = pickle.load(open(args.data_path, 'rb'))
	# else:
	# vocab = Vocab()
	# vocab.create_vocab_for_translation(args.train_path + 'train_all')
	# pickle.dump(vocab, open(args.data_path + 'vocab.pb', 'wb'))

	# Load dataset
	if os.path.isfile(args.data_path + 'prepro.pb'):
		dp = pickle.load(open(args.data_path + 'prepro.pb', 'rb'))
	else:
		dp = Preprocess(args.train_path + 'train.src', args.train_path + 'train.tgt', args.val_path + 'val.src', args.val_path + 'val.tgt', args.train_path + 'train_all')
		dp.preprocess_train()
		dp.preprocess_val()
		pickle.dump(dp, open(args.data_path + 'prepro.pb', 'wb'))

	print ("Elapsed Time: %1.3f \n"  %(time.time() - start_time))

	# print(dp.vocab.itos)
	# print(dp.train_src[0])
	# print(dp.train_tgt[0])
	# training_dataset = list(zip(dp.train_src, dp.train_tgt, dp.train_srclng, dp.train_tgtlng))
	# import random; random.seed(1234);
	# random.shuffle(training_dataset)
	MAX_SAMPLES = 40040
	# dp.train_src[:], dp.train_tgt[:], dp.train_srclng[:], dp.train_tgtlng[:] = zip(*training_dataset)
	dp.train_src, dp.train_tgt, dp.train_srclng, dp.train_tgtlng = dp.train_src[:MAX_SAMPLES], dp.train_tgt[:MAX_SAMPLES], dp.train_srclng[:MAX_SAMPLES], dp.train_tgtlng[:MAX_SAMPLES]

	print(dp.vocab.itos)
	print(dp.train_src[0])
	print(dp.train_tgt[0])
	print ("=========== Data Stat ===========")
	print ("Train: ", len(dp.train_src))
	print ("val: ", len(dp.val_tgt))
	print ("val: ", len(set(dp.train_src_sentlen)))
	print ("=================================")

	trainer = Trainer(dp, args)
	trainer.train()

if __name__ == '__main__':

	parser = argparse.ArgumentParser()

	# Language setting
	parser.add_argument('--dataset', type=str, default='europarl')
	parser.add_argument('--src-lang', action="append", type=str, default=[], dest='src_lang',
	                    help='list of type of lexical relations to capture. Options | syn | hyp | mer')
	parser.add_argument('--trg_lang', type=str, default='en')
	parser.add_argument('--mode', type=str, default='probe', help='probe|train')
	parser.add_argument('--max_len', type=int, default=20)
	parser.add_argument('--cuda', action='store_true',
	                    help='use CUDA')

	parser.add_argument('--gpu', type=int, default=0,
	                    help='use gpu x')
	parser.add_argument('--log-interval', type=int, default=100)
	# Model hyper-parameters
	parser.add_argument('--lr', type=float, default=0.001)
	parser.add_argument('--grad_clip', type=float, default=1)
	parser.add_argument('--num_layer', type=int, default=2)
	parser.add_argument('--embed_dim',  type=int, default=256)
	parser.add_argument('--hidden_dim', type=int, default=512)

	# Training setting
	parser.add_argument('--batch_size', type=int, default=32)
	parser.add_argument('--num_epoch', type=int, default=100)

	# Path
	parser.add_argument('--data_path', type=str, default='./data/')
	parser.add_argument('--train_path', type=str, default='./data/training/')
	parser.add_argument('--val_path', type=str, default='./data/dev/')

	# Dir.
	parser.add_argument('--log', type=str, default='log')
	parser.add_argument('--sample', type=str, default='sample')

	# Misc.
	parser.add_argument('--gpu_num', type=int, default=0)

	args = parser.parse_args()
	print (args)
	main(args)


import json

import requests
from flask import request
from flask_restful import Resource
import pika

class Orders(Resource):

    def post(self):
        record_to_be_created = request.get_json(force=True)
        # Make a GET request to the Inventory Service. URI is dynamically created based on the data in the received
        # message
        r = requests.get('http://172.17.0.3:5000/products/' + record_to_be_created['product type'] + '/quantity')
        ava_quantity = r.json()['quantity']

        # If quantify available > quantity requested
        if ava_quantity > record_to_be_created['quantity']:
            # to establish a connection with RabbitMQ server
            connection = pika.BlockingConnection(pika.ConnectionParameters('172.17.0.5'))
            channel = connection.channel()
            # Create an exchange of type topic
            channel.exchange_declare(exchange='order', exchange_type='topic')
            # Messages sent to a topic exchange can't have an arbitrary routing_key - it must be a list of words,
            # delimited by dots.  A message sent with a particular routing key will be delivered to all the queues
            # that are bound with a matching binding key.
            channel.basic_publish(exchange='order',
                                  routing_key='order.create.inventory.update',
                                  body=json.dumps(record_to_be_created))
            print("[x] Sent 'order_reqd!'")
            connection.close()
            return {"message": "[x] Sent 'order_reqd!'"}, 200

        else:
            return {"message": "Order cannot be accepted at the moment :  out-of-stock"}, 200


from datetime import datetime, timedelta

def to_datetime(now, string):
    if len(string) != 6:
        return None

    # if any field is '*' then make it equal to current time
    if string[0] == '*':
        secs = now.second
    else:
        secs = string[0]
    if string[1] == '*':
        mins = now.minute
    else:
        mins = string[1]
    if string[2] == '*':
        hours = now.hour
    else:
        hours = string[2]
    if string[3] == '*':
        dom = now.day
    else:
        dom = string[3]
    if string[4] == '*':
        mon = now.month
    else:
        mon = string[4]
    if string[5] == '*':
        dow = now.weekday()
    else:
        dow = string[5]

    return (datetime(now.year,mon,dom,hours,mins,secs))

def crontime_in_range(now, start, end):
    dt_start = to_datetime(now, start)
    dt_end = to_datetime(now, end,)
    if dt_start == None or dt_end == None:
        return False

    # take care of weekday
    dw_start = start[5]
    dw_end = end[5]
    if dw_start != '*' and dw_end != '*':
        if dw_start == dw_end != now.weekday():
            return False
        elif dw_start > dw_end:
            dt_end = dt_end + timedelta(weeks=1)
            if now.weekday() < dw_start:
                now = now + timedelta(weeks=1)
        else:
            if dw_start <= now.weekday() <= dw_end:
                dt_end = dt_end + timedelta(days=dw_end - dw_start)

    if dt_start > dt_end:
        dt_end = dt_end + timedelta(days=365)
        if now < dt_start:
            now = now + timedelta(days=365)

    if dt_start <= now <= dt_end:
        return True

    return False


import pexpect, sys

# Add the client folder path to the sys.path list
sys.path.append('/var/www/ton/my_project/client/')

import client

def ton_starter():
	path = client.__file__[:-11]
	#print("path= ", path)
	bash_command = path + "ton/./test-lite-client -C " + path + "ton/ton-lite-client-test1.config.json"
	#print(bash_command)
	p = pexpect.spawn(bash_command)
	p.setecho(False)
	return p


#!/usr/bin/python3

"""
Augur.py

APP: Inquisition
DESC: A library used to fetch and receive OSINT data used for further learning
CREATION_DATE: 2017-05-14

"""

# MODULES
# | Native

# | Third-Party

# | Custom
from lib.inquisit.Inquisit import Inquisit

# METADATA
__author__ = 'Joshua Carlson-Purcell'
__copyright__ = 'Copyright 2017, CarlsoNet'
__license__ = 'MIT'
__version__ = '1.0.0-alpha'
__maintainer__ = 'Joshua Carlson-Purcell'
__email__ = 'jcarlson@carlso.net'
__status__ = 'Development'

class Augur(Inquisit):
    """
    OSINT aggreegation utility
    """

    def __init__(self, cfg):
        Inquisit.__init__(self, cfg, lgrName=__name__)


# !usr/bin/env python
# -*- coding: utf-8 -*-
#
# Licensed under a 3-clause BSD license.
#
# @Author: Brian Cherinka
# @Date:   2017-08-06 22:12:44
# @Last modified by:   Brian Cherinka
# @Last Modified time: 2017-08-30 15:10:16

from __future__ import print_function, division, absolute_import
from functools import wraps
from sciserver.exceptions import SciServerError, SciServerAPIError
from sciserver import config
import requests

def checkAuth(func):
    ''' Decorator that checks if a token has been generated

    Function Decorator to check if a token has already been generated.
    If not it raises an error and tells you to log in.  Otherwise it
    returns the function and proceeds as normal.

    Returns:
        The decorated function

    Raises:
        SciServerError: User token is not defined. First log into SciServer.

    Example:
        >>>
        >>> @checkauth
        >>> def my_function():
        >>>     return 'I am working function'
        >>>

    '''

    @wraps(func)
    def wrapper(*args, **kwargs):
        token = config.get_token()
        if not token:
            raise SciServerError('User token is not defined. First log into SciServer.')
        else:
            return func(*args, **kwargs)
    return wrapper

def check_response(response, errmsg='Error'):
    ''' Checks the response

    Checks the response from a given request.  Raises an error
    for any response with a non 200 status_code .

    Parameters:
        response:
            the Python requests response object
        errmsg (str):
            A custom error message for the type of request being sent

    Returns:
        the HTTP response

    Raises:
        SciServerApiError: upon any HTTPError caused by a bad status code

    '''

    try:
        isbad = response.raise_for_status()
    except requests.HTTPError as http:
        err = response.content.decode()
        raise SciServerAPIError('{0}\n {1}: {2}'.format(http, errmsg, err))
    else:
        assert isbad is None, 'Http status code should not be bad'
        assert response.ok is True, 'Ok status should be true'
        return response

def make_header(content_type='application/json', accept_header='text/plain'):
    ''' Makes a request header

    Makes a request header to be passed along.  Includes the Content-Type,
    Accept arguments.  Also includes an X-Auth-Token is a token is present.

    Parameters:
        content_type (str):
            The request header Content-Type (default: application/json)
        accept_header (str):
            The request header Accept (default: application/json)

    Returns:
        headers (dict):
            The dictionary to be used as a request header

    '''

    headers = {'Content-Type': content_type, 'Accept': accept_header}

    # check for auth token
    token = config.get_token()
    if token is not None and token != "":
        headers['X-Auth-Token'] = token

    return headers

def send_request(url, reqtype='get', data=None, content_type='application/json',
                 acceptHeader='text/plain', errmsg='Error', stream=None):
    ''' Sends a request to the server

    Parameters:
        url (str):
            The url path for the request
        reqtype (str):
            The type of request to perform.  Default is get.  Choices are get, post, put, delete
        data ({str|dict}):
            Optional data to send in the request
        content_type (str):
            the header Content-Type argument.  Default is application/json
        acceptHeader (str):
            the header Accept argument. Default is text/plain
        errmsg (str):
            custom error message in case of faults
        stream (bool):
            optional.  if False, the response content will be immediately downloaded

    Returns:
        response:
            The HTTP response object

    Raises:
        SciServerError: upon any error to occur when sending the requests

    Example:
        >>> # to send a simple get request
        >>> response = send_request(url, errmsg='Error when executing a sql query.')
        >>>
        >>> # to send a post request
        >>> response = send_request(jobsURL, reqtype='post', data=data, content_type='application/json',
        >>>        acceptHeader='application/json', errmsg='Error when submitting job on queue')

    '''

    headers = make_header(content_type=content_type, accept_header=acceptHeader)

    # send the request
    try:
        if reqtype == 'get':
            response = requests.get(url, headers=headers, stream=stream)
        elif reqtype == 'post':
            response = requests.post(url, data=data, headers=headers, stream=stream)
        elif reqtype == 'put':
            response = requests.put(url, data=data, headers=headers, stream=stream)
        elif reqtype == 'delete':
            response = requests.delete(url, headers=headers, stream=stream)
    except Exception as e:
        raise SciServerError("A requests error occurred attempting to send: {0}".format(e))
    else:
        resp = check_response(response, errmsg=errmsg)
        return resp

class Task(object):
    ''' This class describes a SciServer Task '''

    def __init__(self, name=None, use_base=None, component='SciServer'):
        self.use_base = use_base
        self.component = component
        self.set_name(name)

    @property
    def base_name(self):
        if config.isSciServerComputeEnvironment():
            base = "Compute.SciScript-Python.{0}".format(self.component)
        else:
            base = "SciScript-Python.{0}".format(self.component)
        return base

    def set_name(self, name):
        if name:
            if self.use_base:
                self.name = '{0}.{1}'.format(self.base_name, name)
            else:
                self.name = name
        else:
            self.name = self.base_name


import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk

class newDissector(Gtk.Window):
    def __init__(self):
        Gtk.Window.__init__(self, title="Dissector Script")

        mainBox = self.box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL,spacing=10)
        self.add(mainBox)

        topBox = self.box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL,spacing=10)
        topBox.set_homogeneous(False)

        midOneBox = self.box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL,spacing=10)
        midOneBox.set_homogeneous(False)

        midTwoBox = self.box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL,spacing=10)
        midTwoBox.set_homogeneous(False)

        midThreeBox = self.box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL,spacing=10)
        midThreeBox.set_homogeneous(False)

        botBox = self.box = Gtk.Box(orientation=Gtk.Orientation.HORIZONTAL,spacing=10)
        botBox.set_homogeneous(False)

        mainBox.pack_start(topBox, True, True, 10)
        mainBox.pack_start(midOneBox, True, True, 10)
        mainBox.pack_start(midTwoBox, True, True, 10)
        mainBox.pack_start(midThreeBox, True, True, 10)
        mainBox.pack_start(botBox, True, True, 10)

        label = Gtk.Label("Generate a custom dissector script from a selected project.")
        label.set_justify(Gtk.Justification.CENTER)
        topBox.pack_start(label, True, True, 10)

        nameLabel = Gtk.Label("Project")
        nameLabel.set_justify(Gtk.Justification.CENTER)
        midOneBox.pack_start(nameLabel, True, True, 10)

        nameEntry = Gtk.Entry()
        nameEntry.set_text("Name")
        midOneBox.pack_start(nameEntry, True, True, 10)

        self.browse = Gtk.Button(label="Browse")
        self.browse.connect("clicked", self.browse_clicked)
        midOneBox.pack_start(self.browse, True, True, 10)

        nameLabel = Gtk.Label("Dissector Format")
        nameLabel.set_justify(Gtk.Justification.LEFT)
        midTwoBox.pack_start(nameLabel, True, True, 10)

        format_store = Gtk.ListStore(int,str)
        format_store.append([1, "format1"])
        format_store.append([2, "format2"])
        format_store.append([3, "format3"])
        format_store.append([4, "format4"])
        format_store.append([5, "format5"])

        format_combo = Gtk.ComboBox.new_with_model_and_entry(format_store)
        format_combo.connect("changed", self.on_name_combo_changed)
        format_combo.set_entry_text_column(1)
        midTwoBox.pack_start(format_combo, False, False, 20)

        nameLabel = Gtk.Label("Save Location")
        nameLabel.set_justify(Gtk.Justification.CENTER)
        midThreeBox.pack_start(nameLabel, True, True, 10)

        nameEntry = Gtk.Entry()
        nameEntry.set_text("System Path")
        midThreeBox.pack_start(nameEntry, True, True, 10)

        self.browse2 = Gtk.Button(label="Browse")
        self.browse2.connect("clicked", self.browse2_clicked)
        midThreeBox.pack_start(self.browse2, True, True, 10)

        self.generate = Gtk.Button(label="Generate")
        self.generate.connect("clicked", self.generate_clicked)
        botBox.pack_start(self.generate, True, True, 10)

        self.cancel = Gtk.Button(label="Cancel")
        self.cancel.connect("clicked", self.cancel_clicked)
        botBox.pack_start(self.cancel, True, True, 10)

    def browse_clicked(self,widget):
        print("Browsing n shit")
        dialog = Gtk.FileChooserDialog("Please choose a file", self,
            Gtk.FileChooserAction.OPEN,
            (Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL,
             Gtk.STOCK_OPEN, Gtk.ResponseType.OK))

        self.add_filters(dialog)

        response = dialog.run()
        if response == Gtk.ResponseType.OK:
            print("Open clicked")
            print("File selected: " + dialog.get_filename())
        elif response == Gtk.ResponseType.CANCEL:
            print("Cancel clicked")

        dialog.destroy()

    def browse2_clicked(self,widget):
        print("Browsing n shit")
        dialog = Gtk.FileChooserDialog("Please choose a folder", self,
            Gtk.FileChooserAction.SELECT_FOLDER,
            (Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL,
             "Select", Gtk.ResponseType.OK))
        dialog.set_default_size(800, 400)

        response = dialog.run()
        if response == Gtk.ResponseType.OK:
            print("Select clicked")
            print("Folder selected: " + dialog.get_filename())
        elif response == Gtk.ResponseType.CANCEL:
            print("Cancel clicked")

        dialog.destroy()

    def generate_clicked(self,widget):
        print("generating")
        self.destroy()

    def cancel_clicked(self,widget):
        print("canceling")
        self.destroy()

    def on_name_combo_changed(self, combo):
        tree_iter = combo.get_active_iter()
        if tree_iter is not None:
            model = combo.get_model()
            row_id, name = model[tree_iter][:2]
            print("Selected: ID=%d, name=%s" % (row_id, name))
        else:
            entry = combo.get_child()
            print("Entered: %s" % entry.get_text())

    def add_filters(self, dialog):
        filter_text = Gtk.FileFilter()
        filter_text.set_name("Text files")
        filter_text.add_mime_type("text/plain")
        dialog.add_filter(filter_text)

        filter_py = Gtk.FileFilter()
        filter_py.set_name("Python files")
        filter_py.add_mime_type("text/x-python")
        dialog.add_filter(filter_py)

        filter_any = Gtk.FileFilter()
        filter_any.set_name("Any files")
        filter_any.add_pattern("*")
        dialog.add_filter(filter_any)


import sys

input = sys.stdin.readline

n, m = map(int, input().split())

arr = [[0] * (n+1)]

for _ in range(n):
    arr.append([0]+list(map(int, input().split())))

temp = 0

for x in range(1, n+1):
    for y in range(1, n):
        arr[x][y+1] += arr[x][y]

for y in range(1, n+1):
    for x in range(1, n):
        arr[x+1][y] += arr[x][y]

for _ in range(m):
    x1, y1, x2, y2 = map(int, input().split())
    print(arr[x2][y2] - arr[x1-1][y2] - arr[x2][y1-1] + arr[x1-1][y1-1])


# coding=utf-8
import pandas as pd
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np

data = pd.read_csv('/home/drproduck/Downloads/ibm.csv')
for c in data.columns:
    s = {x for x in data[c]}
    print(c, len(s))
data.Attrition = [1 if x == 'Yes' else 0 for x in data.Attrition]
print(data.columns)
data.drop(['Over18', 'EmployeeCount', 'StandardHours'], axis=1, inplace=True)
for c in data.columns:
    if all(isinstance(n, str) for n in data[c]):
        s = {x for x in data[c]}
        t = list(s)
        dic = {}
        for z in range(len(s)):
            dic[t[z]] = z
        print(dic)
        data[c] = [dic[x] for x in data[c]]

plt.figure(1)
plt.hist([data.BusinessTravel[data.Attrition==1],data.BusinessTravel[data.Attrition==0]], color=['green','red'], label=['quit', 'stay'])

matrix = data.as_matrix()

attrition = matrix[:,1:2]
matrix = np.append(matrix, attrition, axis=1)
matrix = np.delete(matrix, 1, axis=1)
print(np.shape(matrix))
import collections
c = collections.Counter()
for x in matrix[:,31]:
    c[x] += 1
print(c)

np.save('matrix_weighted', matrix)


import pygame

def load_image(name):
    try:
        image = pygame.image.load(name)
    except (pygame.error, message):
        print("Cannot load image: " + name)
        raise (SystemExit, message)
    image = image.convert()
    return image, image.get_rect()

class Earth(pygame.sprite.Sprite):

    def __init__(self, gs):
        pygame.sprite.Sprite.__init__(self)
        self.gs = gs
        self.img_path = "./globe.png"
        self.red_globe = "./globe_red100.png"
        self.image, self.rect = load_image(self.img_path)
        self.x = self.gs.width-(self.rect.width/4)
        self.y = self.gs.height-(self.rect.height/4)
        self.rect.centerx = self.x
        self.rect.centery = self.y
        self.radius = self.rect.width/2
        self.max_hp = 500
        self.hp = 500
        self.exploding = 0
        self.exp_frame = 0

    def tick(self):
        self.explode()

    def hit(self):
        self.hp -= 100
        if self.hp <= 0:
            self.exploding = 1
        elif self.hp < .41*self.max_hp:
            self.image, self.rect = load_image(self.red_globe)
            self.rect.centerx = self.x
            self.rect.centery = self.y

    def explode(self):
        if self.exploding and self.exp_frame <= 16:
            if self.exp_frame < 10:
                self.image, self.rect = load_image("./explosion/frames00" + str(self.exp_frame) + "a.png")
            else:
                self.image, self.rect = load_image("./explosion/frames0" + str(self.exp_frame) + "a.png")

            self.rect.centerx = self.x
            self.rect.centery = self.y
            self.exp_frame += 1
            if self.exp_frame == 16:
                self.gs.end = 1


from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from database_setup import Base, Shoppingmall, Cloth, User
""" Import Login session """
from flask import session as login_session
import random
import string
""" imports for gconnect"""
from oauth2client.client import flow_from_clientsecrets
from oauth2client.client import FlowExchangeError
import httplib2
import json
from flask import make_response
import requests
""" import login decorator """
from functools import wraps
from flask import Flask, render_template
from flask import request, redirect, jsonify, url_for, flash
app = Flask(__name__)

CLIENT_ID = json.loads(open('client_secrets.json',
                            'r').read())['web']['client_id']
APPLICATION_NAME = "Catalog"

engine = create_engine('sqlite:///dresses.db')
Base.metadata.bind = engine

DBSession = sessionmaker(bind=engine)

@app.route('/')
@app.route('/login')
def showlogin():
    session = DBSession()
    state = ''.join(random.choice(string.ascii_uppercase + string.digits)
                    for x in range(32))
    login_session['state'] = state
    session.close()
    return render_template('login.html', STATE=state)

@app.route('/gconnect', methods=['POST'])
def gconnect():
    session = DBSession()
    """ validate state token """
    if request.args.get('state') != login_session['state']:
        response = make_response(json.dumps('Invalid state parameter.'), 401)
        response.headers['Content-Type'] = 'application-json'
        session.close()
        return response
    """ Obtain authorization code """
    code = request.data

    try:
        """ upgrade the authorization code in credentials object"""
        oauth_flow = flow_from_clientsecrets('client_secrets.json', scope='')
        oauth_flow.redirect_uri = 'postmessage'
        credentials = oauth_flow.step2_exchange(code)
    except FlowExchangeError:
        response = make_response(json.dumps('Failed to upgrade\
                                            the authorization code'), 401)
        response.headers['Content-Type'] = 'application-json'
        return response

    """ Check that the access token is valid."""
    access_token = credentials.access_token
    url = ('https://www.googleapis.com/oauth2/v1/tokeninfo?access_token=%s'
           % access_token)
    h = httplib2.Http()
    result = json.loads(h.request(url, 'GET')[1])
    """ If there was an error in the access token info, abort."""
    if result.get('error') is not None:
        response = make_response(json.dumps(result.get('error')), 500)
        response.headers['Content-Type'] = 'application/json'
        return response

    """ Verify that the access token is used for the intended user. """
    gplus_id = credentials.id_token['sub']
    if result['user_id'] != gplus_id:
        response = make_response(
            json.dumps("Token's user ID doesn't match given user ID."), 401)
        response.headers['Content-Type'] = 'application/json'
        return response

    """ Verify that the access token is valid for this app. """
    if result['issued_to'] != CLIENT_ID:
        response = make_response(
            json.dumps("Token's client ID does not match app's."), 401)
        print("Token's client ID does not match app's.")
        response.headers['Content-Type'] = 'application/json'
        return response
    """ Access token within the app """
    stored_access_token = login_session.get('access_token')
    stored_gplus_id = login_session.get('gplus_id')
    if stored_access_token is not None and gplus_id == stored_gplus_id:
        response = make_response(json.dumps('Current user\
                                            is already connected.'),
                                 200)
        response.headers['Content-Type'] = 'application/json'
        return response

    """ Store the access token in the session for later use. """

    login_session['access_token'] = credentials.access_token
    login_session['gplus_id'] = gplus_id

    """ Get user info """
    userinfo_url = "https://www.googleapis.com/oauth2/v1/userinfo"
    params = {'access_token': access_token, 'alt': 'json'}
    answer = requests.get(userinfo_url, params=params)

    data = answer.json()

    login_session['username'] = data['name']
    login_session['picture'] = data['picture']
    login_session['email'] = data['email']
    """ See if user exists or if it doesn't make a new one """
    user_id = getUserID(data["email"])
    if not user_id:
        user_id = createUser(login_session)
    login_session['user_id'] = user_id

    output = ''
    output += '
Welcome, '
    output += login_session['username']
    output += '!
'
    output += ''
    flash("you are now logged in as %s" % login_session['username'])
    print("done!")
    return output

""" Helper Functions """

def createUser(login_session):
    session = DBSession()

    newUser = User(name=login_session['username'],
                   email=login_session['email'],
                   picture=login_session['picture'])
    session.add(newUser)
    session.commit()
    user = session.query(User).filter_by(email=login_session['email']).one()
    session.close()
    return user.id

def getUserInfo(user_id):
    session = DBSession()
    user = session.query(User).filter_by(id=user_id).first()
    session.close()
    return user

def getUserID(email):
    session = DBSession()
    try:
        user = session.query(User).filter_by(email=email).one()
        session.close()
        return user.id
    except:
        return None

""" DISCONNECT - Revoke a current user's token and reset their login_session. """
@app.route('/gdisconnect')
def gdisconnect():
    session = DBSession()
    """ only disconnect a connected User """
    access_token = login_session.get('access_token')

    if access_token is None:
        response = make_response(json.dumps('Current user not connected'), 401)
        response.headers['Content-Type'] = 'application/json'
        return response

    url = 'https://accounts.google.com/o/oauth2/revoke?token=%s' % access_token
    h = httplib2.Http()
    result = h.request(url, 'GET')[0]
    if result['status'] == '200':
        response = make_response(json.dumps('Successfully disconnected.'), 200)
        response.headers['Content-Type'] = 'application/json'
        return response
    else:
        response = make_response(json.dumps('Failed to revoke\
                                            token for given user.', 400))
        response.headers['Content-Type'] = 'application/json'
        session.close()
        return response

@app.route('/logout')
def logout():
    if 'provider' in login_session:
        if login_session['provider'] == 'google':
            gdisconnect()
            del login_session['gplus_id']
            del login_session['access_token']
            del login_session['username']
            del login_session['email']
            del login_session['picture']
            del login_session['user_id']
            del login_session['provider']
            flash("You have succesfully been logout")
            return redirect(url_for('showShoppingmalls'))
    else:
        flash("You were not logged in")
        return redirect(url_for('showShoppingmalls'))

@app.route('/shoppingmall/<int:shoppingmall_id>/cloth/JSON')
def shoppingmallclothJSON(brand_id):
    session = DBSession()
    shoppingmall = session.query(Shoppingmall).filter_by(
                   id=shoppingmall_id).one()
    details = session.query(Cloth).filter_by(
               shoppingmall_id=shoppingmall_id).all()
    session.close()
    return jsonify(Cloth=[i.serialize for i in details])

@app.route('/shoppingmall/<int:shoppingmall_id>/details/<int:details_id>/JSON')
def clothesJSON(shoppingmall_id, details_id):
    session = DBSession()
    Clothes_Details = session.query(Cloth).filter_by(id=details_id).one()
    session.close()
    return jsonify(Cloth_Details=Cloth_Details.serialize)

@app.route('/shoppingmall/JSON')
def shoppingmallsJSON():
    session = DBSession()
    shoppingmalls = session.query(Shoppingmall).all()
    session.close()
    return jsonify(shoppingmalls=[r.serialize for r in shoppingmalls])
""" Show all shoppingmalls """

@app.route('/')
@app.route('/shoppingmall/')
def showShoppingmalls():
    session = DBSession()
    shoppingmalls = session.query(Shoppingmall).all()
    """ return "This page will show all my shoppingmalls" """
    session.close()
    return render_template('shoppingmalls.html', shoppingmalls=shoppingmalls)

""" Create a new shoppingmall"""
@app.route('/shoppingmall/new/', methods=['GET', 'POST'])
def newShoppingmall():
    session = DBSession()
    if 'username' not in login_session:
        return redirect('/login')
    if request.method == 'POST':
        newShoppingmall = Shoppingmall(name=request.form['name'],
                                       user_id=login_session['user_id'])
        session.add(newShoppingmall)
        flash('New shoppingmall %s Successfully\
              Created' % newShoppingmall.name)
        session.commit()
        session.close()
        return redirect(url_for('showShoppingmalls'))
    else:
        return render_template('newShoppingmall.html')
    """ return "This page will be for making a new shoppingmall"

 Edit a shoppingmall"""

@app.route('/shoppingmall/<int:shoppingmall_id>/edit/',
           methods=['GET', 'POST'])
def editShoppingmall(shoppingmall_id):
    session = DBSession()
    if 'username' not in login_session:
        return redirect('/login')
    editShoppingmall = session.query(Shoppingmall).filter_by(
                       id=shoppingmall_id).one()
    if editShoppingmall.user_id != login_session['user_id']:
        return ""
    if request.method == 'POST':
            editShoppingmall.name = request.form['name']
            session.add(editShoppingmall)
            flash('Shoppingmall Successfully Edited %s' % request.form['name'])
            session.commit()
            session.close()
            return redirect(url_for('showShoppingmalls',
                            shoppingmall_id=shoppingmall_id))
    else:
        return render_template('editShoppingmall.html',
                               shoppingmall_id=shoppingmall_id,
                               shoppingmall=editShoppingmall)

    """ return 'This page will be for editing shoppingmall %s' % shoppingmall_id

Delete a shoppingmall"""

@app.route('/shoppingmall/<int:shoppingmall_id>/delete/',
           methods=['GET', 'POST'])
def deleteShoppingmall(shoppingmall_id):
    session = DBSession()
    deleteShoppingmall = session.query(
        Shoppingmall).filter_by(id=shoppingmall_id).one()
    if 'username' not in login_session:
        return redirect('/login')
    deleteShoppingmall = session.query(Shoppingmall).filter_by(
                         id=shoppingmall_id).one()
    if deleteShoppingmall.user_id != login_session['user_id']:
        return ""
    if request.method == 'POST':
        session.delete(deleteShoppingmall)
        session.commit()
        session.close()
        return redirect(
            url_for('showShoppingmalls', shoppingmall_id=shoppingmall_id))
    else:
        return render_template('deleteShoppingmall.html',
                               shoppingmall_id=shoppingmall_id,
                               shoppingmall=deleteShoppingmall)

""" Show a shoppingmall clothes"""
@app.route('/shoppingmall/<int:shoppingmall_id>/')
@app.route('/shoppingmall/<int:shoppingmall_id>/cloth/')
def showCloth(shoppingmall_id):
    session = DBSession()
    shoppingmall = session.query(Shoppingmall).filter_by(
                   id=shoppingmall_id).one()
    creator = getUserInfo(shoppingmall.user_id)
    details = session.query(Cloth).filter_by(
              shoppingmall_id=shoppingmall_id).all()
    session.close()
    return render_template('cloth.html', details=details,
                           shoppingmall=shoppingmall, creator=creator)
    """ return 'This page is the product for shoppingmall %s' % shoppinmall_id

Create a new product details"""

@app.route('/shoppingmall/<int:shoppingmall_id>/shoppingmall/new/',
           methods=['GET', 'POST'])
def newCloth(shoppingmall_id):
    session = DBSession()
    if 'username' not in login_session:
        return redirect('/login')
    shoppingmall = session.query(Shoppingmall).filter_by(
                   id=shoppingmall_id).one()
    if login_session['user_id'] != shoppingmall.user_id:
        return ""
    if request.method == 'POST':
        newShoppingmall = Cloth(
            name=request.form['name'],
            description=request.form['description'],
            price=request.form['price'],
            type=request.form['type'],
            shoppingmall_id=shoppingmall_id,
            user_id=shoppingmall.user_id)
        session.add(newShoppingmall)
        flash('New clothes %s  Successfully Created' % (newShoppingmall.name))
        session.commit()
        session.close()

        return redirect(url_for('showCloth', shoppingmall_id=shoppingmall_id))
    else:
        return render_template('newCloth.html',
                               shoppingmall_id=shoppingmall_id)
""" return 'This page is for making a new cloth details for shoppingmall %s'

 Edit a cloth details"""

@app.route('/shoppingmall/<int:shoppingmall_id>/cloth/<int:cloth_id>/edit',
           methods=['GET', 'POST'])
def editCloth(shoppingmall_id, cloth_id):
    session = DBSession()
    if 'username' not in login_session:
        return redirect('/login')
    editCloth = session.query(Cloth).filter_by(id=cloth_id).one()
    shoppingmall = session.query(Shoppingmall).filter_by(
                   id=shoppingmall_id).one()
    if login_session['user_id'] != shoppingmall.user_id:
        return ""
    if request.method == 'POST':
        if request.form['name']:
            editCloth.name = request.form['name']
        if request.form['description']:
            editCloth.description = request.form['name']
        if request.form['price']:
            editCloth.price = request.form['price']
        if request.form['type']:
            editCloth.type = request.form['type']
        session.add(editCloth)
        session.commit()
        flash(' clothes Successfully Edited')
        session.close()
        return redirect(url_for('showCloth', shoppingmall_id=shoppingmall_id))
    else:
        return render_template('editCloth.html',
                               shoppingmall_id=shoppingmall_id,
                               cloth_id=cloth_id, details=editCloth)

    """ return 'This page is for editing cloth details %s' % cloth_id

 Delete a cloth details"""

@app.route('/shoppingmall/<int:shoppingmall_id>/cloth/<int:cloth_id>/delete',
           methods=['GET', 'POST'])
def deleteCloth(shoppingmall_id, cloth_id):
    session = DBSession()
    if 'username' not in login_session:
        return redirect('/login')
    shoppingmall = session.query(Shoppingmall).filter_by(
                   id=shoppingmall_id).one()
    deleteCloth = session.query(Cloth).filter_by(id=cloth_id).one()
    if login_session['user_id'] != shoppingmall.user_id:
        return ""
    if request.method == 'POST':
        session.delete(deleteCloth)
        session.commit()
        session.close()
        return redirect(url_for('showCloth', shoppingmall_id=shoppingmall_id))
    else:
        return render_template('deleteCloth.html',
                               shoppingmall_id=shoppingmall_id,
                               cloth_id=cloth_id, details=deleteCloth)
    """ return "This page is for deleting cloth details %s" % cloth_id"""

if __name__ == '__main__':
    app.secret_key = 'super_secret_key'
    app.debug = True
    app.run(host='0.0.0.0', port=8080)


import numpy as np
from scipy import misc
import os

test_file = "/home/didi/Documents/Codes/test.txt"
valid_file = "/home/didi/Documents/Codes/valid.txt"
train_file = "/home/didi/Documents/Codes/train.txt"
def read_data(filename):
    thefile = open(filename,"r")
    lines = thefile.readlines()
    #print lines
    content = [x.strip() for x in lines]
    images = []
    labels = []
    for item in content:
        #print eval(item)[0]
        image_name = "/home/didi/Documents/Codes/data/" + eval(item)[0]
        #print image_name
        image = misc.imread(image_name)
        image = np.array(image,dtype = np.float32)
        image = image.flatten()
        images.append(image)

        labels.append(int(eval(item)[1]))

    images = np.array(images,dtype = np.float32)
    labels = np.array(labels, dtype  = np.int32)
    return images, labels

def load_train_data():
    train_data, train_label = read_data(train_file)
    return train_data, train_label

def load_test_data():
    test_data, test_label = read_data(test_file)
    return test_data, test_label

def load_valid_data():
    valid_data, valid_label = read_data(valid_file)
    return valid_data, valid_label


import unittest
from contextlib import redirect_stdout
from io import StringIO

from ukoly.kapitola2.quotes import *

class MyTestCase(unittest.TestCase):
    def setUp(self):
        self.out = StringIO()

    def test_double_quotes(self):
        with redirect_stdout(self.out):
            double_quotes()
        self.assertEqual(self.out.getvalue(), 'Alenka rekla: "Bez mi koupit rohliky, Honzo."\n')

    def test_single_quotes(self):
        with redirect_stdout(self.out):
            single_quotes()
        self.assertEqual(self.out.getvalue(), "Alenka rekla: 'Bez mi koupit rohliky, Honzo.'\n")

    def test_mixed_quotes(self):
        with redirect_stdout(self.out):
            mixed_quotes()
        self.assertEqual(self.out.getvalue(), '"You\'ll be an IT specialist!\", that\'s what the guys at Viet Code said.\n')

if __name__ == '__main__':
    unittest.main()


import numpy as np
import re
import itertools
from collections import Counter
from os import listdir
from os.path import isfile, join
import  json

def clean_str(string):
    """
    Tokenization/string cleaning for all datasets except for SST.
    Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py
    """
    string = re.sub(r"[()\'\`]", " ", string)
    string = re.sub(r"\'s", " \'s", string)
    string = re.sub(r"\'ve", " \'ve", string)
    string = re.sub(r"n\'t", " n\'t", string)
    string = re.sub(r"\'re", " \'re", string)
    string = re.sub(r"\'d", " \'d", string)
    string = re.sub(r"\'ll", " \'ll", string)
    string = re.sub(r",", " comma", string)
    string = re.sub(r"!", " exclamationmark ", string)
    string = re.sub(r"\(", "", string)
    string = re.sub(r"\)", "", string)
    string = re.sub(r"\?", " questionmark ", string)
    string = re.sub(r"\.\.\.", " dotdotdot ", string)
    string = re.sub(r"\.", "", string)
    return string.strip().lower()

def match_text(ID):
    json_path=""
    mypath="rumoureval-data/"

    for d in listdir(mypath):
        if ID in listdir(mypath+d):
            dir=d
            json_path=mypath+dir+"/"+ID+"/source-tweet/"+ID+".json"
            break

    for d in listdir(mypath):#event
        for thread in listdir(mypath+d):
            reply_path=mypath+d+"/"+thread+"/replies/"
            if ID+".json" in listdir(reply_path):
                json_path=reply_path+ID+".json"
                break
        if json_path!="":#break outer loop
            break
    if (isfile(json_path)):
        source = json.load(open(json_path, 'r'))
    else:
        return False
    return source['text']

def load_data_and_labels(task,jsonfile):
    """
    Loads rumorID and stance from file
    """
    ID1=[]#these list for memory ID order
    ID2=[]
    ID3=[]
    ID4=[]
    truth_values = json.load(open(jsonfile, 'r'))
    category=0
    agreed_examples=[]
    disagreed_examples=[]
    appeal_examples=[]
    comment_examples=[]
    if (task=="taskA"):
        category=4#SDQC
        for id in truth_values.keys():
            if(match_text(id)):
                if (truth_values[id]=="support"):
                    agreed_examples.append(match_text(id))
                    ID1.append(id)
                if (truth_values[id]=="deny"):
                    disagreed_examples.append(match_text(id))
                    ID2.append(id)
                if (truth_values[id]=="query"):
                    appeal_examples.append(match_text(id))
                    ID3.append(id)
                if (truth_values[id]=="comment"):
                    comment_examples.append(match_text(id))
                    ID4.append(id)
    else:
        category=3#rumor or not

    #strip special char
    agreed_examples = [s.strip() for s in agreed_examples]
    disagreed_examples = [s.strip() for s in disagreed_examples]
    appeal_examples = [s.strip() for s in appeal_examples]
    comment_examples = [s.strip() for s in comment_examples]

    IDlist=ID1+ID2+ID3+ID4
    # Split by words
    x_text = agreed_examples + disagreed_examples+appeal_examples+comment_examples
    x_text = [clean_str(sent) for sent in x_text]
    # Generate labels
    agreed_labels = [[0, 0,0,1] for _ in agreed_examples]
    disagreed_labels = [[0,0,1,0] for _ in disagreed_examples]
    appeal_labels = [[0, 1,0,0] for _ in appeal_examples]
    comment_labels = [[1, 0,0,0] for _ in comment_examples]
    y = np.concatenate([agreed_labels,disagreed_labels,appeal_labels,comment_labels], 0)

    return [x_text, y,IDlist]

def batch_iter(data, batch_size, num_epochs, shuffle=True):
    """
    Generates a batch iterator for a dataset.
    """
    data = np.array(data)
    data_size = len(data)
    num_batches_per_epoch = int(len(data)/batch_size) + 1

    for epoch in range(num_epochs):
        # Shuffle the data at each epoch

        for batch_num in range(num_batches_per_epoch):
            shuffle_indices = np.random.permutation(np.arange(data_size))
            shuffled_data = data[shuffle_indices]
            y_=[]
            for batch in shuffled_data:
                #print(batch)
                x_batch, y_batch = zip(batch)
                y_.append(y_batch)

            new_data=[]#new data
            a1=0
            a2=0
            a3=0
            a4=0
            i=0

            for y in y_:

                if ((y[0][0]==1) and (a1<batch_size/4)):
                    a1+=1
                    new_data.append(shuffled_data[i])
                if ((y[0][1]==1) and (a2<batch_size/4)):
                    a2+=1
                    new_data.append(shuffled_data[i])
                if ((y[0][2]==1) and (a3<batch_size/4)):
                    a3+=1
                    new_data.append(shuffled_data[i])
                if ((y[0][3]==1) and (a4<batch_size/4)):
                    a4+=1
                    new_data.append(shuffled_data[i])
                i+=1
            # print(len(new_data))
            new_data=np.array(new_data)
            indices = np.random.permutation(np.arange(len(new_data)))
            out_data = new_data[indices]
            yield out_data


import sys
from unittest.mock import Mock
from pathlib import Path

import pytest
import jupytext
import nbformat
import nbconvert

from ploomber import DAG, DAGConfigurator
from ploomber.tasks import NotebookRunner
from ploomber.products import File
from ploomber.exceptions import DAGBuildError, DAGRenderError
from ploomber.tasks import notebook
from ploomber.executors import Serial

def fake_from_notebook_node(self, nb, resources):
    return bytes(42), None

def test_error_when_path_has_no_extension():
    with pytest.raises(ValueError) as excinfo:
        notebook.NotebookConverter('a')

    msg = 'Could not determine output format for product: "a"'
    assert msg in str(excinfo.value)

@pytest.mark.parametrize(
    'path, exporter',
    [('file.ipynb', None), ('file.pdf', nbconvert.exporters.pdf.PDFExporter),
     ('file.html', nbconvert.exporters.html.HTMLExporter),
     ('file.md', nbconvert.exporters.markdown.MarkdownExporter)])
def test_notebook_converter_get_exporter_from_path(path, exporter):
    converter = notebook.NotebookConverter(path)
    assert converter.exporter == exporter

@pytest.mark.parametrize(
    'exporter_name, exporter',
    [('ipynb', None), ('pdf', nbconvert.exporters.pdf.PDFExporter),
     ('html', nbconvert.exporters.html.HTMLExporter),
     ('md', nbconvert.exporters.markdown.MarkdownExporter),
     ('markdown', nbconvert.exporters.markdown.MarkdownExporter),
     ('slides', nbconvert.exporters.slides.SlidesExporter)])
def test_notebook_converter_get_exporter_from_name(exporter_name, exporter):
    converter = notebook.NotebookConverter('file.ext', exporter_name)
    assert converter.exporter == exporter

@pytest.mark.parametrize('output', [
    'file.ipynb', 'file.pdf',
    pytest.param(
        'file.html',
        marks=pytest.mark.xfail(
            sys.platform == 'win32',
            reason='nbconvert has a bug when exporting to HTML on windows')),
    'file.md'
])
def test_notebook_conversion(monkeypatch, output, tmp_directory):
    # we mock the method that does the conversion to avoid having to
    # intstall tex for testing
    monkeypatch.setattr(nbconvert.exporters.pdf.PDFExporter,
                        'from_notebook_node', fake_from_notebook_node)

    nb = nbformat.v4.new_notebook()
    cell = nbformat.v4.new_code_cell('1 + 1')
    nb.cells.append(cell)

    with open(output, 'w') as f:
        nbformat.write(nb, f)

    conv = notebook.NotebookConverter(output)
    conv.convert()

@pytest.mark.parametrize(
    'name, out_dir',
    [
        ['sample.py', '.'],
        ['sample.R', '.'],
        ['sample.ipynb', '.'],
        # check still works even if the folder does not exit yet
        ['sample.ipynb', 'missing_folder']
    ])
def test_execute_sample_nb(name, out_dir, tmp_sample_tasks):
    dag = DAG()

    NotebookRunner(Path(name),
                   product=File(Path(out_dir, name + '.out.ipynb')),
                   dag=dag)
    dag.build()

def _dag_simple(nb_params=True, params=None):
    path = Path('sample.py')

    if nb_params:
        path.write_text("""
# + tags=["parameters"]
a = None
b = 1
c = 'hello'
""")
    else:
        path.write_text("""
# + tags=["parameters"]
""")

    dag = DAG()
    NotebookRunner(path, product=File('out.ipynb'), dag=dag, params=params)
    return dag

def _dag_two_tasks(nb_params=True, params=None):
    root = Path('root.py')
    root.write_text("""
# + tags=["parameters"]
""")

    path = Path('sample.py')
    if nb_params:
        path.write_text("""
# + tags=["parameters"]
a = None
b = 1
c = 'hello'
""")
    else:
        path.write_text("""
# + tags=["parameters"]
""")

    dag = DAG()
    root = NotebookRunner(root, product=File('root.ipynb'), dag=dag)
    task = NotebookRunner(path,
                          product=File('out.ipynb'),
                          dag=dag,
                          params=params)
    root >> task
    return dag

def test_dag_r(tmp_directory):
    path = Path('sample.R')

    path.write_text("""
# + tags=["parameters"]
a <- NULL
b <- 1
c <- c(1, 2, 3)
""")

    dag = DAG()
    NotebookRunner(path, product=File('out.ipynb'), dag=dag, params=dict(z=1))

    # parameter extraction is not implemented but should not raise an error
    dag.render()

def test_render_error_on_syntax_error(tmp_directory):
    path = Path('sample.py')

    path.write_text("""
# + tags=["parameters"]
if
""")

    dag = DAG()
    NotebookRunner(path, product=File('out.ipynb'), dag=dag)

    with pytest.raises(DAGRenderError) as excinfo:
        dag.render()

    assert 'invalid syntax\n\nif\n\n  ^\n' in str(excinfo.value)

def test_render_error_on_undefined_name_error(tmp_directory):
    path = Path('sample.py')

    path.write_text("""
# + tags=["parameters"]

# +
df.head()
""")

    dag = DAG()
    NotebookRunner(path, product=File('out.ipynb'), dag=dag)

    with pytest.raises(DAGRenderError) as excinfo:
        dag.render()

    assert "undefined name 'df'" in str(excinfo.value)

def test_render_pass_on_missing_product_parameter(tmp_directory):
    path = Path('sample.py')

    path.write_text("""
# + tags=["parameters"]

# +
df = None
df.to_csv(product)
""")

    dag = DAG()
    NotebookRunner(path, product=File('out.ipynb'), dag=dag)

    # the render process injects the cell with the product variable so this
    # should not raise any errors, even if the raw source code does not contain
    # the product variable
    assert dag.render()

@pytest.mark.parametrize('code', [
    """
# + tags=["parameters"]

# +
import pandas as pd
df = pd.read_csv(upstream['root'])
""",
    """
# + tags=["parameters"]

# +
x

# +
import pandas as pd
df = pd.read_csv(upstream['root'])
""",
],
                         ids=['simple', 'multiple-undefined'])
def test_render_error_on_missing_upstream(tmp_directory, code):
    path = Path('sample.py')
    path.write_text(code)

    dag = DAG()
    NotebookRunner(path, product=File('out.ipynb'), dag=dag)

    with pytest.raises(DAGRenderError) as excinfo:
        dag.render()

    expected = ("undefined name 'upstream'. Did you forget"
                " to declare upstream dependencies?")
    assert expected in str(excinfo.value)

@pytest.mark.parametrize('factory', [_dag_simple, _dag_two_tasks])
def test_render_error_on_missing_params(tmp_directory, factory):
    dag = factory()

    with pytest.raises(DAGRenderError) as excinfo:
        dag.render()

    assert "Missing params: 'a', 'b', and 'c'" in str(excinfo.value)

@pytest.mark.parametrize('factory', [_dag_simple, _dag_two_tasks])
def test_render_error_on_unexpected_params(tmp_directory, factory):
    dag = factory(nb_params=False, params=dict(a=1, b=2, c=3))

    with pytest.raises(DAGRenderError) as excinfo:
        dag.render()

    assert "Unexpected params: 'a', 'b', and 'c'" in str(excinfo.value)

@pytest.mark.parametrize('factory', [_dag_simple, _dag_two_tasks])
def test_render_error_on_missing_and_unexpected_params(tmp_directory, factory):
    dag = factory(nb_params=True, params=dict(d=1, e=2, f=3))

    with pytest.raises(DAGRenderError) as excinfo:
        dag.render()

    assert "Unexpected params: 'd', 'e', and 'f'" in str(excinfo.value)
    assert "Missing params: 'a', 'b', and 'c'" in str(excinfo.value)

@pytest.mark.parametrize('code', [
    """
# + tags=["parameters"]
upstream = None
product = None
""", """
# + tags=["parameters"]
upstream = None
""", """
# + tags=["parameters"]
product = None
"""
])
def test_ignores_declared_product_and_upstream(tmp_directory, code):
    path = Path('sample.py')

    path.write_text(code)

    dag = DAG()
    NotebookRunner(path, product=File('out.ipynb'), dag=dag)
    dag.render()

@pytest.mark.xfail(
    sys.platform == 'win32',
    reason='nbconvert has a bug when exporting to HTML on windows')
def test_can_convert_to_html(tmp_sample_tasks):
    dag = DAG()

    NotebookRunner(Path('sample.ipynb'),
                   product=File(Path('out.html')),
                   dag=dag,
                   name='nb')
    dag.build()

def test_can_execute_with_parameters(tmp_directory):
    dag = DAG()

    code = """
# + tags=["parameters"]
var = None
    """

    NotebookRunner(code,
                   product=File(Path(tmp_directory, 'out.ipynb')),
                   dag=dag,
                   kernelspec_name='python3',
                   params={'var': 1},
                   ext_in='py',
                   name='nb')
    dag.build()

def test_can_execute_when_product_is_metaproduct(tmp_directory):
    dag = DAG()

    code = """
# + tags=["parameters"]
var = None

# +
from pathlib import Path

Path(product['model']).touch()
    """

    product = {
        'nb': File(Path(tmp_directory, 'out.ipynb')),
        'model': File(Path(tmp_directory, 'model.pkl'))
    }

    NotebookRunner(code,
                   product=product,
                   dag=dag,
                   kernelspec_name='python3',
                   params={'var': 1},
                   ext_in='py',
                   nb_product_key='nb',
                   name='nb')
    dag.build()

def test_raises_error_if_key_does_not_exist_in_metaproduct(tmp_directory):
    dag = DAG()

    product = {
        'some_notebook': File(Path(tmp_directory, 'out.ipynb')),
        'model': File(Path(tmp_directory, 'model.pkl'))
    }

    code = """
# + tags=["parameters"]
var = None

# +
    """

    with pytest.raises(KeyError) as excinfo:
        NotebookRunner(code,
                       product=product,
                       dag=dag,
                       kernelspec_name='python3',
                       params={'var': 1},
                       ext_in='py',
                       nb_product_key='nb',
                       name='nb')

    assert "Key 'nb' does not exist in product" in str(excinfo.value)

def test_failing_notebook_saves_partial_result(tmp_directory):
    dag = DAG()

    code = """
# + tags=["parameters"]
var = None

raise Exception('failing notebook')
    """

    # attempting to generate an HTML report
    NotebookRunner(code,
                   product=File('out.html'),
                   dag=dag,
                   kernelspec_name='python3',
                   params={'var': 1},
                   ext_in='py',
                   name='nb')

    # build breaks due to the exception
    with pytest.raises(DAGBuildError):
        dag.build()

    # but the file with ipynb extension exists to help debugging
    assert Path('out.ipynb').exists()

def test_error_if_wrong_exporter_name(tmp_sample_tasks):
    dag = DAG()

    with pytest.raises(ValueError) as excinfo:
        NotebookRunner(Path('sample.ipynb'),
                       product=File(Path('out.ipynb')),
                       dag=dag,
                       nbconvert_exporter_name='wrong_name')

    assert ('Could not find nbconvert exporter with name "wrong_name"'
            in str(excinfo.value))

def test_error_if_cant_find_exporter_name(tmp_sample_tasks):
    dag = DAG()

    with pytest.raises(ValueError) as excinfo:
        NotebookRunner(Path('sample.ipynb'),
                       product=File(Path('out.wrong_ext')),
                       dag=dag,
                       nbconvert_exporter_name=None)

    assert 'Could not find nbconvert exporter' in str(excinfo.value)

def test_skip_kernel_install_check(tmp_directory):
    dag = DAG()

    code = """
# + tags=["parameters"]
1 + 1
    """

    NotebookRunner(code,
                   product=File(Path(tmp_directory, 'out.ipynb')),
                   dag=dag,
                   kernelspec_name='unknown_kernel',
                   ext_in='py',
                   name='nb',
                   check_if_kernel_installed=False)
    dag.render()

# TODO: we are not testing output, we have to make sure params are inserted
# correctly
@pytest.fixture
def tmp_dag(tmp_directory):
    dag = DAG()

    code = """
# + tags=["parameters"]
var = None

# +
1 + 1
    """
    p = Path('some_notebook.py')

    p.write_text(code)

    NotebookRunner(p,
                   product=File(Path(tmp_directory, 'out.ipynb')),
                   dag=dag,
                   kernelspec_name='python3',
                   params={'var': 1},
                   name='nb')

    dag.render()

    return dag

def test_develop_saves_changes(tmp_dag, monkeypatch):
    def mock_jupyter_notebook(args, check):
        nb = jupytext.reads('2 + 2', fmt='py')
        # args: "jupyter" {app} {path} {other args, ...}
        nbformat.write(nb, args[2])

    monkeypatch.setattr(notebook.subprocess, 'run', mock_jupyter_notebook)
    monkeypatch.setattr(notebook, '_save', lambda: True)

    tmp_dag['nb'].develop()
    path = str(tmp_dag['nb'].source.loc)

    assert Path(path).read_text().strip() == '2 + 2'

@pytest.mark.parametrize('app', ['notebook', 'lab'])
def test_develop_with_custom_args(app, tmp_dag, monkeypatch):
    mock = Mock()

    monkeypatch.setattr(notebook.subprocess, 'run', mock)
    monkeypatch.setattr(notebook, '_save', lambda: True)

    tmp_dag['nb'].develop(app=app, args='--port=8888 --no-browser')

    # make sure params are quoted to prevent code injection
    mock.assert_called_once_with([
        'jupyter', app, 'some_notebook-tmp.ipynb', '--port=8888',
        '--no-browser'
    ],
                                 check=True)

def test_develop_unknown_app(tmp_dag):
    with pytest.raises(ValueError) as excinfo:
        tmp_dag['nb'].develop(app='unknown')

    assert '"app" must be one of' in str(excinfo.value)

def test_develop_workflow_with_hot_reload(tmp_directory, monkeypatch):
    cfg = DAGConfigurator()
    cfg.params.hot_reload = True
    dag = cfg.create()

    code = """
# + tags=["parameters"]
var = None

# +
1 + 1
    """
    p = Path('some_notebook.py')

    p.write_text(code)

    t = NotebookRunner(p,
                       product=File(Path(tmp_directory, 'out.ipynb')),
                       dag=dag,
                       kernelspec_name='python3',
                       params={'var': 1},
                       name='nb')

    def mock_jupyter_notebook(args, check):
        nb = jupytext.reads("""
# + tags=["parameters"]
var = None

# +
2 + 2
""",
                            fmt='py')
        # args: "jupyter" {app} {path} {others, ...}
        nbformat.write(nb, args[2])

    dag.render()

    monkeypatch.setattr(notebook.subprocess, 'run', mock_jupyter_notebook)
    monkeypatch.setattr(notebook, '_save', lambda: True)

    t.develop()

    # source code must be updated
    assert '2 + 2' in str(t.source).strip()

    nb = nbformat.reads(t.source.nb_str_rendered,
                        as_version=nbformat.NO_CONVERT)
    source = jupytext.writes(nb, fmt='py')

    assert '2 + 2' in source

def test_develop_error_if_r_notebook(tmp_sample_tasks):
    dag = DAG()

    t = NotebookRunner(Path('sample.R'), product=File('out.ipynb'), dag=dag)

    dag.render()

    with pytest.raises(NotImplementedError):
        t.develop()

    with pytest.raises(NotImplementedError):
        t.debug()

# TODO: make a more general text and parametrize by all task types
# but we also have to test it at the source level
# also test at the DAG level, we have to make sure the property that
# code differ uses (raw code) it also hot_loaded
def test_hot_reload(tmp_directory):
    cfg = DAGConfigurator()
    cfg.params.hot_reload = True

    dag = cfg.create()

    path = Path('nb.py')
    path.write_text("""
# + tags=["parameters"]
# some code

# +
1 + 1
    """)

    t = NotebookRunner(path,
                       product=File('out.ipynb'),
                       dag=dag,
                       kernelspec_name='python3')

    t.render()

    path.write_text("""
# + tags=["parameters"]
# some code

# +
2 + 2
    """)

    t.render()

    assert '2 + 2' in str(t.source)
    assert t.product._outdated_code_dependency()
    assert not t.product._outdated_data_dependencies()

    assert '2 + 2' in t.source.nb_str_rendered

    report = dag.build()

    assert report['Ran?'] == [True]

    # TODO: check task is not marked as outdated

@pytest.mark.parametrize('kind, to_patch', [
    ['ipdb', 'IPython.terminal.debugger.Pdb.run'],
    ['pdb', 'pdb.run'],
    ['pm', None],
])
def test_debug(monkeypatch, kind, to_patch, tmp_dag):
    if to_patch:
        mock = Mock()
        monkeypatch.setattr(to_patch, mock)

    tmp_dag['nb'].debug(kind=kind)

    if to_patch:
        mock.assert_called_once()

@pytest.mark.xfail(sys.platform == "win32",
                   reason="Two warnings are displayed on windows")
def test_warns_if_export_args_but_ipynb_output(tmp_sample_tasks):
    dag = DAG(executor=Serial(build_in_subprocess=False))

    NotebookRunner(Path('sample.ipynb'),
                   File('out.ipynb'),
                   dag,
                   nbconvert_export_kwargs=dict(exclude_input=True))

    with pytest.warns(UserWarning) as records:
        dag.build()

    # NOTE: not sure why sometimes two records are displayed, maybe another
    # library is throwing the warning
    assert any(
        "Output 'out.ipynb' is a notebook file" in record.message.args[0]
        for record in records)

def test_change_static_analysis(tmp_sample_tasks):
    dag = DAG(executor=Serial(build_in_subprocess=False))

    # static_analysis is True by default, this should fail
    t = NotebookRunner(Path('sample.ipynb'),
                       File('out.ipynb'),
                       dag,
                       params=dict(a=1, b=2))

    # disable it
    t.static_analysis = False

    # this should work
    dag.render()