import os, datetime, random
from PIL import Image
from PIL import ImageDraw
from PIL import ImageFont

print(r'''
    Welcome to Tomas Silva's armwrestling double-elimination brackets.

    For more information you can contact me at tsilvaebensperger@gmail.com
--------------------------------------------------------------------------

STEP1: INSERT NUMBER OF CATEGORIES AND CATEGORIES.

--------------------------------------------------------------------------
''')

''' Let's create the directory where all the created files will be.
'''

try:
    name = input('What do you want to call this competition: ')
    day_time = datetime.date.today()
    directorio = os.getcwd()+r'\Competitions'+'\\'+str(day_time)+' '+name
    os.mkdir(directorio)
except Exception as e:
    print(e)

input('Press any key to continue. ')

while True:
    try:
        cat = int(input('How many categories?: '))
        if cat <=0:
            print('You need at least one category.')
            continue
        break
    except Exception:
        print('Please input a valid number.')

#These are some lists, dicts and variables used.
dictionary = {}
n = 0
weights = list()

while n<cat-1:
    try:
        cat_weight = int(input('Insert weight category: '))
        n +=1
        weights.append(cat_weight)
    except Exception:
        print('Insert a valid weight.')
weights = sorted(weights)
weight_string = [str(x) for x in weights]
if cat>=2:
    print('\nThe categories are: '+' '.join(weight_string)+' +'+weight_string[-1])
else:
    print('\nOnly one category: OPEN')
print('\n--------------------------------------------------------------------------')

def count_athletes(a, b):
    count_weight = dict()
    dictionary_copy = dict(a)
    for i in b:
        count_weight[i]=0
        for j in list(dictionary_copy):
            if dictionary_copy[j] <= i:
                count_weight[i] += 1
                dictionary_copy.pop(j)
    count_weight['+'+str(b[-1])] = 0
    for i in dictionary_copy:
        if dictionary_copy[i] > b[-1]:
            count_weight['+'+str(b[-1])] += 1
    return count_weight

text_to_decide = ''
while text_to_decide != 'proceed':

    text_to_decide = input('''
     What do you want to do?
         - to add athlete: add
         - to remove athlete: del
         - to see the list of athletes: list
         - to proceed to competition: proceed

Action: ''')

    if text_to_decide == 'add':
        try:
            name = input('Name: ')
            weight = float(input('Weight: '))
            dictionary[name]=weight
        except Exception:
            print('Not a valid name or weight.')

    elif text_to_decide== 'del':
        try:
            name = input('Name: ')
            dictionary.pop(name)
        except Exception:
            print('Not a valid name.')

    elif text_to_decide== 'list':
        print('\n')
        for i in sorted(dictionary):
            print(i,':', dictionary[i])
        print('\n')
        provisional =count_athletes(dictionary, weights)
        for i in provisional:
            print(i,':', provisional[i] )

print(r'''
--------------------------------------------------------------------------

STEP2: PRESENT ALL THE ATHLETES AND SAVE IT TO A FILE.

--------------------------------------------------------------------------
''')
dictionary_of_athletes = dict()
for i in weights:
    dictionary_of_athletes[i]=list()
#print(dictionary_of_athletes)

d_of_a_keys = sorted(dictionary_of_athletes.keys())
dictionary_of_athletes['+'+str(d_of_a_keys[-1])]=list()

for i in d_of_a_keys:
    for j in list(dictionary):
        if dictionary[j] <= i:
            dictionary_of_athletes[i].append(j)
            dictionary.pop(j)

for i in dictionary:
    if dictionary[i] > d_of_a_keys[-1]:
        dictionary_of_athletes['+'+str(d_of_a_keys[-1])].append(i)

file = open(directorio +'\CATEGORIES.txt','w')
file.write('THE CATEGORIES ARE:\n\n')
for i in dictionary_of_athletes.keys():
    a= str(i)+'\n\n'
    file.write(a)
    for j in dictionary_of_athletes[i]:
        b = str(j)+'\n'
        file.write(b)
    file.write('----------------------------------------------------------\n')
file.close()

file = open(directorio +'\CATEGORIES.txt', 'r')
text = file.read()
print(text)
file.close()
print('\nYou can find the text file here:\n'+os.path.abspath('CATEGORIES.txt'))

print(r'''
--------------------------------------------------------------------------

STEP3: DRAWS AND SAVE IT TO A FILE

--------------------------------------------------------------------------
''')

coord = {
        3:[[70,142],[70,205],[190,50],[190, 175],[190, 245],[190, 335],[320, 100],[320,285],[425,175],[425, 375],[559,270]],
        4:[[70, 20],[70, 80],[70,142],[70,205],[70, 300], [70, 365],[190,50],[190, 175],[190, 245],[190, 335],[320, 100],[320,285],[425,175],[425, 375],[559,270]],
        5:[[35, 135],[35, 170],[105, 25],[105, 90],[105,150],[105, 210],[105, 250],[105, 295],[180, 55],[180, 180],[180, 265],[180, 395],[235, 235],[235, 325],[310, 105],[310, 275],[430, 170], [430, 365], [500, 265]],
        6:[],
        7:[],
}

for i in dictionary_of_athletes:
    random.shuffle(dictionary_of_athletes[i])

def create_brackets(dictt, path, coords):

    for i in dictt:
        largo = len(dictt[i])
        img = Image.open(os.getcwd()+r'\Files\Brackets'+'\\'+str(largo)+'.png')
        draw = ImageDraw.Draw(img)
        font = ImageFont.truetype('arial.ttf',10)
        for j in range(largo):
            draw.text((coords[largo][j][0],coords[largo][j][1]),dictt[i][j],(0,0,0), font=font)
        img.save(path+'\\'+str(i)+'.png')
        img.close()

create_brackets(dictionary_of_athletes, directorio, coord)

def update_brackets(bracket, lista, path, indice):
    for i in bracket:
        img = Image.open(path+'\\'+ str(indice) +'.png')
        draw = ImageDraw.Draw(img)
        font = ImageFont.truetype('arial.ttf',10)
        draw.text((bracket[i][0],bracket[i][1]),i,(0,0,0), font=font)
        img.save(path+'\\'+ str(indice) +'.png')
        img.close()

#We define a function of each number of possible competitors per category.

def cat3(lista, peso):
#        random.shuffle(lista)
        w1, w2, w3, w4, w5, l1, l2, l3, l4 = ' '*1, ' '*2, ' '*3, ' '*4, ' '*5, ' '*5, ' '*7, ' '*8, ' '*9
        bracket = {w1:[190, 175], l1:[190, 245], l2:[190, 335], w2:[320,100], w3:[320, 285], w4:[425,175], l4:[425, 375], w5:[559, 270]}

        temp_list = [lista[0], lista[1]]
        while w1 not in temp_list:
            print('\nMatch 1:', lista[0], 'vs', lista[1])
            w1 = input('Who won?: ')
        temp_list.remove(w1)
        l1 = temp_list[0]
        bracket = {w1:[190, 175], l1:[190, 245], l2:[190, 335], w2:[320,100], w3:[320, 285], w4:[425,175], l4:[425, 375], w5:[559, 270]}
        update_brackets(bracket, lista, directorio, peso)

        temp_list = [lista[2], w1]
        while w2 not in temp_list:
            print('\nMatch 2:', lista[2], 'vs', w1)
            w2 = input('Who won?: ')
        temp_list.remove(w2)
        l2 = temp_list[0]
        bracket = {w1:[190, 175], l1:[190, 245], l2:[190, 335], w2:[320,100], w3:[320, 285], w4:[425,175], l4:[425, 375], w5:[559, 270]}
        update_brackets(bracket, lista, directorio, peso )

        temp_list = [l1, l2]
        while w3 not in temp_list:
            print('\nMatch 3:', l1, 'vs', l2)
            w3 = input('Who won?: ')
        temp_list.remove(w3)
        l3 = temp_list[0]
        bracket = {w1:[190, 175], l1:[190, 245], l2:[190, 335], w2:[320,100], w3:[320, 285], w4:[425,175], l4:[425, 375], w5:[559, 270]}
        update_brackets(bracket, lista, directorio, peso )

        temp_list = [w2, w3]
        while w4 not in temp_list:
            print('\nMatch 4 FINAL:', w2, 'vs', w3)
            w4 = input('Who won?: ')
        temp_list.remove(w4)
        l4 = temp_list[0]
        bracket = {w1:[190, 175], l1:[190, 245], l2:[190, 335], w2:[320,100], w3:[320, 285], w4:[425,175], l4:[425, 375], w5:[559, 270]}
        update_brackets(bracket, lista, directorio, peso )

        if w4 == w2:
            first_place = w4
            second_place = l4
            third_place = l3
            bracket = {w1:[190, 175], l1:[190, 245], l2:[190, 335], w2:[320,100], w3:[320, 285], w4:[425,175], l4:[425, 375], w5:[559, 270]}
            update_brackets(bracket, lista, directorio, peso )
        else:
            temp_list = [w4, l4]
            while w5 not in temp_list:
                print('\nMatch 5 FINAL REMATCH:', w4, 'vs', l4)
                w5 = input('Who won?: ')
            temp_list.remove(w5)
            l5 = temp_list[0]

            first_place = w5
            second_place = l5
            third_place = l3
            bracket = {w1:[190, 175], l1:[190, 245], l2:[190, 335], w2:[320,100], w3:[320, 285], w4:[425,175], l4:[425, 375], w5:[559, 270]}
            update_brackets(bracket, lista, directorio, peso )

        places = [first_place, second_place, third_place]
        return places

def cat4(lista, peso):
#    random.shuffle(lista)
    l1, l2, w1, w2, l3, w4, w3, w5, w6, l6, w7 = ' '*1, ' '*2, ' '*3, ' '*4, ' '*5, ' '*5, ' '*7, ' '*8, ' '*9, ' '*10, ' '*11
    bracket = {l1:[70, 300],l2:[70, 365],w1:[190,50],w2:[190, 175],l3:[190, 245],w4:[190, 335],w3:[320, 100],w5:[320,285],w6:[425,175],l6:[425, 375],w7:[559,270]}
    temp_list = [lista[0], lista[1]]
    while w1 not in temp_list:
        print('\nMatch 1:', lista[0], 'vs', lista[1])
        w1 = input('Who won?: ')
    temp_list.remove(w1)
    l1 = temp_list[0]
    bracket = {l1:[70, 300],l2:[70, 365],w1:[190,50],w2:[190, 175],l3:[190, 245],w4:[190, 335],w3:[320, 100],w5:[320,285],w6:[425,175],l6:[425, 375],w7:[559,270]}
    update_brackets(bracket, lista, directorio, peso)

    temp_list = [lista[2], lista[3]]
    while w2 not in temp_list:
        print('\nMatch 2:', lista[2], 'vs', lista[3])
        w2 = input('Who won?: ')
    temp_list.remove(w2)
    l2 = temp_list[0]
    bracket = {l1:[70, 300],l2:[70, 365],w1:[190,50],w2:[190, 175],l3:[190, 245],w4:[190, 335],w3:[320, 100],w5:[320,285],w6:[425,175],l6:[425, 375],w7:[559,270]}
    update_brackets(bracket, lista, directorio, peso)

    temp_list = [w1, w2]
    while w3 not in temp_list:
        print('\nMatch 3:', w1, 'vs', w2)
        w3 = input('Who won?: ')
    temp_list.remove(w3)
    l3 = temp_list[0]
    bracket = {l1:[70, 300],l2:[70, 365],w1:[190,50],w2:[190, 175],l3:[190, 245],w4:[190, 335],w3:[320, 100],w5:[320,285],w6:[425,175],l6:[425, 375],w7:[559,270]}
    update_brackets(bracket, lista, directorio, peso)

    temp_list = [l1, l2]
    while w4 not in temp_list:
        print('\nMatch 4:', l1, 'vs', l2)
        w4 = input('Who won?: ')
    bracket = {l1:[70, 300],l2:[70, 365],w1:[190,50],w2:[190, 175],l3:[190, 245],w4:[190, 335],w3:[320, 100],w5:[320,285],w6:[425,175],l6:[425, 375],w7:[559,270]}
    update_brackets(bracket, lista, directorio, peso)

    temp_list = [l3, w4]
    while w5 not in temp_list:
        print('\nMatch 5:', l3, 'vs', w4)
        w5 = input('Who won?: ')
    temp_list.remove(w5)
    l5 = temp_list[0]
    bracket = {l1:[70, 300],l2:[70, 365],w1:[190,50],w2:[190, 175],l3:[190, 245],w4:[190, 335],w3:[320, 100],w5:[320,285],w6:[425,175],l6:[425, 375],w7:[559,270]}
    update_brackets(bracket, lista, directorio, peso)

    temp_list = [w3, w5]
    while w6 not in temp_list:
        print('\nMatch 6 FINAL:', w3, 'vs', w5)
        w6 = input('Who won?: ')
    temp_list.remove(w6)
    l6 = temp_list[0]
    bracket = {l1:[70, 300],l2:[70, 365],w1:[190,50],w2:[190, 175],l3:[190, 245],w4:[190, 335],w3:[320, 100],w5:[320,285],w6:[425,175],l6:[425, 375],w7:[559,270]}
    update_brackets(bracket, lista, directorio, peso)

    if w6 == w3:
        first_place = w6
        second_place = l6
        third_place = l5
        bracket = {l1:[70, 300],l2:[70, 365],w1:[190,50],w2:[190, 175],l3:[190, 245],w4:[190, 335],w3:[320, 100],w5:[320,285],w6:[425,175],l6:[425, 375],w7:[559,270]}
        update_brackets(bracket, lista, directorio, peso)
    else:
        temp_list = [w6, l6]
        while w7 not in temp_list:
            print('\nMatch 7 FINAL REMATCH:', w6, 'vs', l6)
            w7 = input('Who won?: ')
        temp_list.remove(w7)
        l7 = temp_list[0]

        first_place = w7
        second_place = l7
        third_place = l5
        bracket = {l1:[70, 300],l2:[70, 365],w1:[190,50],w2:[190, 175],l3:[190, 245],w4:[190, 335],w3:[320, 100],w5:[320,285],w6:[425,175],l6:[425, 375],w7:[559,270]}
        update_brackets(bracket, lista, directorio, peso)

    places = [first_place, second_place, third_place]
    return places

def cat5(lista, peso):
#    random.shuffle(lista)
    w1, l1, l2, w2, w3, w4, l3, l5, w6, w5, w7, w8, l8, w9 = ' '*1, ' '*2, ' '*3, ' '*4, ' '*5, ' '*5, ' '*7, ' '*8, ' '*9, ' '*10, ' '*11,  ' '*12, ' '*13, ' '*14
    bracket = {w1:[105,150],l1:[105, 250],l2:[105, 295],w2:[180, 55], w3:[180, 180], w4:[180, 265],l3:[180, 395],l5:[235, 235],w6:[235, 325],w5:[310, 105],w7:[310, 275],w8:[430, 170], l8:[430, 365], w9:[500, 265]}

    temp_list = [lista[0], lista[1]]
    while w1 not in temp_list:
        print('Match 1:', lista[0], 'vs', lista[1])
        w1 = input('Who won?: ')
    temp_list.remove(w1)
    l1 = temp_list[0]
    bracket = {w1:[105,150],l1:[105, 250],l2:[105, 295],w2:[180, 55], w3:[180, 180], w4:[180, 265],l3:[180, 395],l5:[235, 235],w6:[235, 325],w5:[310, 105],w7:[310, 275],w8:[430, 170], l8:[430, 365], w9:[500, 265]}
    update_brackets(bracket, lista, directorio, peso)

    temp_list = [lista[2], lista[3]]
    while w2 not in temp_list:
        print('\nMatch 2:', lista[2], 'vs', lista[3])
        w2 = input('Who won?: ')
    temp_list.remove(w2)
    l2 = temp_list[0]
    bracket = {w1:[105,150],l1:[105, 250],l2:[105, 295],w2:[180, 55], w3:[180, 180], w4:[180, 265],l3:[180, 395],l5:[235, 235],w6:[235, 325],w5:[310, 105],w7:[310, 275],w8:[430, 170], l8:[430, 365], w9:[500, 265]}
    update_brackets(bracket, lista, directorio, peso)

    temp_list = [w1, lista[4]]
    while w3 not in temp_list:
        print('\nMatch 3:', w1, 'vs', lista[4])
        w3 = input('Who won?: ')
    temp_list.remove(w3)
    l3 = temp_list[0]
    bracket = {w1:[105,150],l1:[105, 250],l2:[105, 295],w2:[180, 55], w3:[180, 180], w4:[180, 265],l3:[180, 395],l5:[235, 235],w6:[235, 325],w5:[310, 105],w7:[310, 275],w8:[430, 170], l8:[430, 365], w9:[500, 265]}
    update_brackets(bracket, lista, directorio, peso)

    temp_list = [l1, l2]
    while w4 not in temp_list:
        print('\nMatch 4:', l1, 'vs', l2)
        w4 = input('Who won?: ')
    bracket = {w1:[105,150],l1:[105, 250],l2:[105, 295],w2:[180, 55], w3:[180, 180], w4:[180, 265],l3:[180, 395],l5:[235, 235],w6:[235, 325],w5:[310, 105],w7:[310, 275],w8:[430, 170], l8:[430, 365], w9:[500, 265]}
    update_brackets(bracket, lista, directorio, peso)

    temp_list = [w2, w3]
    while w5 not in temp_list:
        print('\nMatch 5:', w2, 'vs', w3)
        w5 = input('Who won?: ')
    temp_list.remove(w5)
    l5 = temp_list[0]
    bracket = {w1:[105,150],l1:[105, 250],l2:[105, 295],w2:[180, 55], w3:[180, 180], w4:[180, 265],l3:[180, 395],l5:[235, 235],w6:[235, 325],w5:[310, 105],w7:[310, 275],w8:[430, 170], l8:[430, 365], w9:[500, 265]}
    update_brackets(bracket, lista, directorio, peso)

    temp_list = [w4, l3]
    while w6 not in temp_list:
        print('\nMatch 6:', w4, 'vs', l3)
        w6 = input('Who won?: ')
    bracket = {w1:[105,150],l1:[105, 250],l2:[105, 295],w2:[180, 55], w3:[180, 180], w4:[180, 265],l3:[180, 395],l5:[235, 235],w6:[235, 325],w5:[310, 105],w7:[310, 275],w8:[430, 170], l8:[430, 365], w9:[500, 265]}
    update_brackets(bracket, lista, directorio, peso)

    temp_list = [l5, w6]
    while w7 not in temp_list:
        print('\nMatch 7:', l5, 'vs', w6)
        w7 = input('Who won?: ')
    temp_list.remove(w7)
    l7 = temp_list[0]
    bracket = {w1:[105,150],l1:[105, 250],l2:[105, 295],w2:[180, 55], w3:[180, 180], w4:[180, 265],l3:[180, 395],l5:[235, 235],w6:[235, 325],w5:[310, 105],w7:[310, 275],w8:[430, 170], l8:[430, 365], w9:[500, 265]}
    update_brackets(bracket, lista, directorio, peso)

    temp_list = [w5, w7]
    while w8 not in temp_list:
        print('\nMatch 8 FINAL:', w5, 'vs', w7)
        w8 = input('Who won?: ')
    temp_list.remove(w8)
    l8 = temp_list[0]
    bracket = {w1:[105,150],l1:[105, 250],l2:[105, 295],w2:[180, 55], w3:[180, 180], w4:[180, 265],l3:[180, 395],l5:[235, 235],w6:[235, 325],w5:[310, 105],w7:[310, 275],w8:[430, 170], l8:[430, 365], w9:[500, 265]}
    update_brackets(bracket, lista, directorio, peso)

    if w8 == w5:
        first_place = w8
        second_place = l8
        third_place = l7
        bracket = {w1:[105,150],l1:[105, 250],l2:[105, 295],w2:[180, 55], w3:[180, 180], w4:[180, 265],l3:[180, 395],l5:[235, 235],w6:[235, 325],w5:[310, 105],w7:[310, 275],w8:[430, 170], l8:[430, 365], w9:[500, 265]}
        update_brackets(bracket, lista, directorio, peso)

    else:
        temp_list = [w8, l8]
        while w9 not in temp_list:
            print('\nMatch 9 FINAL REMATCH:', w8, 'vs', l8)
            w9 = input('Who won?: ')
        temp_list.remove(w9)
        l9 = temp_list[0]

        first_place = w9
        second_place = l9
        third_place = l7
        bracket = {w1:[105,150],l1:[105, 250],l2:[105, 295],w2:[180, 55], w3:[180, 180], w4:[180, 265],l3:[180, 395],l5:[235, 235],w6:[235, 325],w5:[310, 105],w7:[310, 275],w8:[430, 170], l8:[430, 365], w9:[500, 265]}
        update_brackets(bracket, lista, directorio, peso)

    places = [first_place, second_place, third_place]
    return places

def cat6(lista):
#    random.shuffle(lista)
    w1 = l1 = w2 = l2 = w3 = l3 = w4 = w6 = w7 = w5 = w8 = w9 = w10 = l10 = w11 = l11 =''

    temp_list = [lista[0], lista[1]]
    while w1 not in temp_list:
        print('Match 1:', lista[0], 'vs', lista[1])
        w1 = input('Who won?: ')
    temp_list.remove(w1)
    l1 = temp_list[0]

    temp_list = [lista[2], lista[3]]
    while w2 not in temp_list:
        print('\nMatch 2:', lista[2], 'vs', lista[3])
        w2 = input('Who won?: ')
    temp_list.remove(w2)
    l2 = temp_list[0]

    temp_list = [w1, lista[4]]
    while w3 not in temp_list:
        print('\nMatch 3:', w1, 'vs', lista[4])
        w3 = input('Who won?: ')
    temp_list.remove(w3)
    l3 = temp_list[0]

    temp_list = [w2, lista[5]]
    while w4 not in temp_list:
        print('\nMatch 4:', w2, 'vs', lista[5])
        w4 = input('Who won?: ')
    temp_list.remove(w4)
    l4 = temp_list[0]

    temp_list = [l2, l3]
    while w5 not in temp_list:
        print('\nMatch 5:', l2, 'vs', l3)
        w5 = input('Who won?: ')

    temp_list = [l1, l4]
    while w6 not in temp_list:
        print('\nMatch 6:', l1, 'vs', l4)
        w6 = input('Who won?: ')

    temp_list = [w3, w4]
    while w7 not in temp_list:
        print('\nMatch 7:', w3, 'vs', w4)
        w7 = input('Who won?: ')
    temp_list.remove(w7)
    l7 = temp_list[0]

    temp_list = [w5, w6]
    while w8 not in temp_list:
        print('\nMatch 8:', w5, 'vs', w6)
        w8 = input('Who won?: ')

    temp_list = [l7, w8]
    while w9 not in temp_list:
        print('\nMatch 9:', l7, 'vs', w8)
        w9 = input('Who won?: ')
    temp_list.remove(w9)
    l9 = temp_list[0]

    temp_list = [w7, w9]
    while w10 not in temp_list:
        print('\nMatch 10 FINAL:', w7, 'vs', w9)
        w10 = input('Who won?: ')
    temp_list.remove(w10)
    l10 = temp_list[0]

    if w10 == w7:
        first_place = w10
        second_place = l10
        third_place = l9
    else:
        temp_list = [w10, l10]
        while w11 not in temp_list:
            print('\nMatch 9 FINAL REMATCH:', w10, 'vs', l10)
            w11 = input('Who won?: ')
        temp_list.remove(w11)
        l11 = temp_list[0]

        first_place = w11
        second_place = l11
        third_place = l9

    places = [first_place, second_place, third_place]
    return places

#----------------------------------------------------------------------
''' With all the functions ready supporting n participants'''

file = open(directorio +'\RESULTS.txt','a')
file.write('THE RESULTS ARE:\n\n')
for i in dictionary_of_athletes:
    largo = len(dictionary_of_athletes[i])
    print('---------------------------------------------------------\n')
    print('CATEGORY '+ str(i)+':')
    cells = eval('cat'+str(largo)+'(dictionary_of_athletes[i], i)')
    file.write('CATEGORY '+ str(i)+':\n\n')
    for j in range(len(cells)):
        file.write('Place '+str(j+1)+': '+str(cells[j])+'\n')
    file.write('\n-----------------------------------------------\n')
file.close()

'''
IF YOU NEED TO RETRIEVE THE ORIGINAL BRACKET FILES
-------------------------------------------------------------------------------

import requests
import os

for i in range(3, 65):
    url = r'http://www.printyourbrackets.com/thumbs/'+str(i)+r'-Team-Double-Elimination.gif'
    response = requests.get(url)
    path = os.getcwd()+'\ARMWRESTLING_DOUBLE_ELIMINATION\Files\Brackets\\'
    file = open(path+str(i)+'.png','wb')
    file.write(response.content)
    file.close()

-------------------------------------------------------------------------------
'''


#%% Imports
import serial
import time

#%% Initializations

Serial_Port = "COM8"
Baud_Rate = 115200

SER = serial.Serial( Serial_Port, Baud_Rate)

#%% Main Run
if __name__ == "__main__":
    SER.flushInput()

    # Clear LCD
    SER.write(b'\xff\xff\x02***\xc8\xc8')

    time.sleep(1)
    # LCD Goto (Row, Col)
    row, col = (2, 4)
    tmp_packet = '\xff\xff\x03{}{}*\xc8\xc8'.format(chr(row), chr(col))
    SER.write(bytes(tmp_packet, encoding='ANSI'))

    time.sleep(1)
    # Send a string to show on LCD
    for char_i in 'Parham Joon!':
        tmp_packet = '\xff\xff\x04{}\x06*\xc8\xc8'.format(char_i)
        SER.write(bytes(tmp_packet, encoding='ANSI'))
        time.sleep(2000/Baud_Rate)

    time.sleep(1)
    # LCD Shift Entire Display to the Left
    SER.write(b'\xff\xff\x05\x18**\xc8\xc8')

    SER.close()


import utils
import os

# https://sourceforge.net/p/mingw-w64/wiki2/Cross%20Win32%20and%20Win64%20compiler/
def build_toolchain(target_triple, package_version, num_jobs):
	print('Building toolchain for', target_triple)

	gcc_version = '6.1.0'
	binutils_version = '2.26'
	mingw_w64_version = '4.0.6'

	print('Downloading sources')

	utils.should_have('https://downloads.sourceforge.net/project/mingw-w64/mingw-w64/mingw-w64-release/mingw-w64-v{0}.tar.bz2'.format(mingw_w64_version), '0c407394b0d8635553f4fbca674cdfe446aac223e90b4010603d863e4bdd015c')
	utils.should_have('ftp://ftp.gnu.org/gnu/gcc/gcc-{0}/gcc-{0}.tar.bz2'.format(gcc_version), '09c4c85cabebb971b1de732a0219609f93fc0af5f86f6e437fd8d7f832f1a351')
	utils.should_have('ftp://ftp.gnu.org/gnu/binutils/binutils-{0}.tar.bz2'.format(binutils_version), 'c2ace41809542f5237afc7e3b8f32bb92bc7bc53c6232a84463c423b0714ecd9')

	utils.extract_tar('mingw-w64-v{0}.tar.bz2'.format(mingw_w64_version))
	utils.extract_tar('gcc-{0}.tar.bz2'.format(gcc_version))
	utils.extract_tar('binutils-{0}.tar.bz2'.format(binutils_version))

	log_dir = 'target/log/{0}'.format(target_triple)
	build_dir = 'target/build/{0}'.format(target_triple)
	sysroot = os.path.abspath('target/sysroots/{0}'.format(target_triple))

	utils.should_be_empty(log_dir)
	utils.should_be_empty(build_dir + '/binutils')
	utils.should_be_empty(build_dir + '/gcc')
	utils.should_be_empty(build_dir + '/headers')
	utils.should_be_empty(build_dir + '/crt')
	utils.should_be_empty(sysroot)

	binutils_gcc_options = [
		'--disable-lto', # We don't need LTO because it won't work with rustc which is based on LLVM.
		'--disable-nls', # English, motherfucker, do you speak it?
		'--disable-multilib',
		'--with-pkgversion={0}'.format(package_version),
		'--target={0}'.format(target_triple),
		'--with-sysroot={0}'.format(sysroot),
		'--prefix={0}'.format(sysroot),
	]

	env = {
		'PATH': os.environ['PATH'],
		'CFLAGS': '-O2',
		'CXXFLAGS': '-O2',
		'CFLAGS_FOR_TARGET': '-O2',
		'CXXFLAGS_FOR_TARGET': '-O2',
	}

	# binutils
	utils.execute(env, log_dir + '/01.binutils-configure.log', build_dir + '/binutils', [
		'../../../downloads/binutils-{0}/configure'.format(binutils_version), '--enable-deterministic-archives',
	] + binutils_gcc_options)

	utils.execute(env, log_dir + '/02.binutils-make.log', build_dir + '/binutils', [
		'make',	'--jobs={0}'.format(num_jobs)
	])

	utils.execute(env, log_dir + '/03.binutils-make-install.log', build_dir + '/binutils', [
		'make', '--jobs={0}'.format(num_jobs), 'install'
	])

	# mingw-headers
	utils.execute(env, log_dir + '/04.headers-configure.log', build_dir + '/headers', [
		'../../../downloads/mingw-w64-v{0}/mingw-w64-headers/configure'.format(mingw_w64_version),
		'--host={0}'.format(target_triple),
		'--prefix={0}/{1}'.format(sysroot, target_triple),
	])

	utils.execute(env, log_dir + '/05.headers-make-install.log', build_dir + '/headers', [
		'make', '--jobs={0}'.format(num_jobs), 'install'
	])

	os.symlink(sysroot + '/{0}'.format(target_triple), sysroot + '/mingw')

	# gcc (1 of 2)
	utils.execute(env, log_dir + '/06.gcc-configure.log', build_dir + '/gcc', [
		'../../../downloads/gcc-{0}/configure'.format(gcc_version), '--enable-languages=c',
	] + binutils_gcc_options)

	utils.execute(env, log_dir + '/07.gcc-make-all-gcc.log', build_dir + '/gcc', [
		'make', '--jobs={0}'.format(num_jobs), 'all-gcc'
	])

	utils.execute(env, log_dir + '/08.gcc-make-install-gcc.log', build_dir + '/gcc', [
		'make', '--jobs={0}'.format(num_jobs), 'install-gcc'
	])

	env['PATH'] = '{0}/bin:'.format(sysroot) + env['PATH']

	# crt
	utils.execute(env, log_dir + '/09.crt-configure.log', build_dir + '/crt', [
		'../../../downloads/mingw-w64-v{0}/configure'.format(mingw_w64_version),
		'--host={0}'.format(target_triple),
		'--prefix={0}/{1}'.format(sysroot, target_triple),
		'--with-sysroot={0}'.format(target_triple),
	])

	utils.execute(env, log_dir + '/10.crt-make.log', build_dir + '/crt', [
		'make', '--jobs={0}'.format(num_jobs),
	])

	utils.execute(env, log_dir + '/11.crt-make-install.log', build_dir + '/crt', [
		'make', '--jobs={0}'.format(num_jobs), 'install'
	])

	# gcc (2 of 2)
	utils.execute(env, log_dir + '/12.gcc-make.log', build_dir + '/gcc', [
		'make', '--jobs={0}'.format(num_jobs)
	])

	utils.execute(env, log_dir + '/13.gcc-make-install.log', build_dir + '/gcc', [
		'make', '--jobs={0}'.format(num_jobs), 'install'
	])

	print('Toolchain for', target_triple, 'has been successfully built')


#!/usr/bin/env python3
import sys

from tqdm import tqdm
import argparse
import numpy as np

def parse_arguments():
    def to_zero_base(x):
        x = int(x)
        if x < 1:
            raise argparse.ArgumentTypeError("Field starts from 1.")
        return x - 1

    parser = argparse.ArgumentParser(description='')

    parser.add_argument(
        '-i',
        '--input',
        type=argparse.FileType('r'),
        default=sys.stdin,
        help='')

    parser.add_argument(
        '-d', '--delimiter', default='\t', help='default to \\t')

    parser.add_argument(
        '--mode', choices=['key', 'value', 'both'], required=True)
    parser.add_argument('--key-field', type=to_zero_base, default=0)
    parser.add_argument('--key-num', type=int, default=1)
    parser.add_argument('--value-num', type=int, default=1)

    parser.add_argument(
        '-o',
        '--output',
        type=argparse.FileType('w'),
        default=sys.stdout,
        help='')

    return parser.parse_args()

def main():
    args = parse_arguments()
    data = {}
    for line in args.input:
        if not line.strip():
            continue
        splits = line.strip().split(args.delimiter)
        data.setdefault(splits[args.key_field], []).append(line)

    if args.mode in ['key', 'both']:
        keys = list(data.keys())
        keys = np.random.choice(
            keys, min(len(keys), args.key_num), replace=False)
        data = {k: data[k] for k in keys}

    if args.mode in ['value', 'both']:
        data = {
            k: np.random.choice(v, min(len(v), args.value_num), replace=False)
            for k, v in tqdm(data.items(), desc='Sampling values')
        }

    for values in data.values():
        args.output.writelines(list(values))

if __name__ == '__main__':
    main()


"""
Created on Tue Apr 14 11:36:39 2020
This is the cart_UV code in pytorch
@author: abdul haseeb
"""
import os,time
import numpy as np
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import gen_net as gn
import gen_net3 as gn2
import gen_netV as gnV

import supportingFun as sf
import supporting_storm_fun as ss
from torch.autograd import Variable
import h5py as h5
import readData as rd
import scipy.io as sio
import sys
from torch.utils.data import TensorDataset
import random

sys.path.insert(0, "/Users/ahhmed/pytorch_unet")

from torchkbnufft.torchkbnufft import KbNufft
from torch.utils.data import Dataset, DataLoader
from torch.optim.lr_scheduler import ReduceLROnPlateau

from unetmodel import UnetClass
from dn_modelV import SmallModel

from scipy import ndimage
import gen_net1 as gn1
gpu=torch.device('cuda:0')
cpu=torch.device('cpu')
directory='/Users/ahhmed/Codes/TensorFlow_SToRM/Data1/FID142454v/d900/'
directory2='/Users/ahhmed/Codes/TensorFlow_SToRM/Data1/FID142454v/'

dtype = torch.float
directory1='/Users/ahhmed/pytorch_unet/SPTmodels/19Jun_101938pm_50ep_20Jun/'
#directory1='/Users/ahhmed/pytorch_unet/SPTmodels/21Jun_110930pm_100ep_21Jun/'
#directory1='/Users/ahhmed/pytorch_unet/SPTmodels/23Jun_111055pm_150ep_23Jun/'

#chkPoint='24'
chkPoint='52'

NF=900#100#900
nx=512
#N=400
N1=300
N2=50
nch=4
thres=0.05
nbasis=30
lam=0.01
st=0
batch_sz=100
TF=900
#%%
G=UnetClass().to(gpu)
GV=SmallModel().to(gpu)
G.load_state_dict(torch.load('wts-2U96.pt'))
GV.load_state_dict(torch.load('wts-2V96.pt'))
#optimizer=torch.optim.SGD([{'params':G.parameters(),'lr':5e-3,'momentum':0.9}])
optimizer=torch.optim.AdamW([{'params':G.parameters(),'lr':1e-4},{'params':GV.parameters(),'lr':1e-4}])
scheduler = ReduceLROnPlateau(optimizer, mode='min', factor=0.7, patience=6, verbose=True, min_lr=5e-5)

trnFiles=os.listdir('/Shared/lss_jcb/abdul/prashant_cardiac_data/Data/')
sz=len(trnFiles)
#data2=np.zeros((sz,1,n_select,N,N)).astype(np.complex64)
rndm=random.sample(range(sz),sz)
for fl in range(sz):
    dictn=sio.loadmat('/Shared/lss_jcb/abdul/prashant_cardiac_data/Data/'+trnFiles[rndm[fl]])
    #dictn=sio.loadmat('/Shared/lss_jcb/abdul/prashant_cardiac_data/meas_MID00185_FID142454_SAX_gNAV_gre11.mat')
    #with h5.File(directory2+'bCom_img1.h5','r') as f:
    #  # load the L matrix
    #  nav_r=np.asarray(f['/bCom_img/re'])
    #  nav_i=np.asarray(f['/bCom_img/im'])
    #  nav=nav_r+nav_i*1j
    #  nav=nav[0+st:NF+st]
    #  nav=nav.astype(np.complex64)
    #  nav=abs(nav)
      # load the L matrix
    L=np.asarray(dictn['L'])
      #L=L[0+st:NF+st,0+st:NF+st]
    L=L.astype(np.complex64)
    U,sb,V=np.linalg.svd(L)
    sb[NF-1]=0
    V=V[TF-nbasis:TF,0+st:NF+st]
    sb=np.diag(sb[NF-nbasis:NF,])*lam
    V=V.astype(np.float32)
    #Vn=np.zeros((sz,nbasis,NF),dtype='float32')
    #Vn[fl]=V

      # coil sensitivity maps
    csm=np.asarray(dictn['csm'])
    csm=csm.astype(np.complex64)
    csm=np.transpose(csm,[2,1,0])

    csm=csm[0:nch,:,:]
      ##csm1=np.fft.fftshift(np.fft.fftshift(csm,1),2)
    csm=np.fft.fftshift(np.fft.fftshift(csm,1),2)
    #csmTrn=np.zeros((sz,nch,nx,nx),dtype='complex64')
    #csmTrn[fl]=csm
    #del csm

      # reading the RHS: AHb
    S=np.asarray(dictn['S'])
    data=np.asarray(dictn['b'])
    tmp=np.zeros((nx*nx,1),dtype='complex64')
    kdata=np.zeros((NF,nch,nx*nx),dtype='complex64')

    for i in range(NF):
        for j in range(nch):
            indx=np.squeeze(S[i,0])
            tmp[indx-1]=data[i,j]
            kdata[i,j]=np.squeeze(tmp)
            tmp=np.zeros((nx*nx,1),dtype='complex64')
    kdata=np.transpose(kdata,(1,0,2))
    kdata=kdata[0:nch,0+st:NF+st,:]
    kdata=kdata.astype(np.complex64)
    kdata=np.reshape(kdata,(nch,int(NF/batch_sz)*batch_sz,nx,nx))
    #kdata=np.transpose(kdata,(0,1,3,2))
    kdata=np.fft.fftshift(np.fft.fftshift(kdata,2),3)
    #kdata=np.reshape(kdata,(nch,int(NF/batch_sz)*batch_sz,nx*nx))
    #kdata=np.reshape(kdata,(nch,int(NF/batch_sz)*batch_sz,nx,nx))
    #kdata=np.transpose(kdata,(0,1,3,2))
    kdata=np.reshape(kdata,(nch,int(NF/batch_sz)*batch_sz,nx*nx))

    kdata=kdata/np.max(np.abs(kdata))

    msk=np.zeros((NF,nx*nx),dtype='float32')
    for k in range(NF):
        indx=np.squeeze(S[k,0])
        msk[k,indx-1]=1

    maskTst=np.reshape(msk,(NF,nx,nx))
    #maskTst=np.transpose(maskTst,(0,2,1))
    maskTst=maskTst[0+st:NF+st]
    maskTst=np.fft.fftshift(np.fft.fftshift(maskTst,1),2)
    maskTst=np.reshape(maskTst,(NF,nx*nx))

    #res=np.reshape(res,(int(NF/batch_sz),batch_sz,nbasis,nx*nx))
    #%% make them pytorch tensors
    kdataT=torch.tensor(sf.c2r(kdata))
    csmT=torch.tensor(sf.c2r(csm))
    maskT=torch.tensor(maskTst)
    VT=torch.tensor(V)
    sT=torch.tensor(sb)
    #W=torch.tensor(W)
    #Nv=torch.tensor(nav)
    #%% take them to gpu

    csmT=csmT.to(gpu)
    maskT=maskT.to(dtype=torch.float32)
    VT=VT.to(gpu)
    #sT=sT.to(gpu)
    #Nv=Nv.to(gpu)
    #%%

    z=ss.ATBV(kdataT,csmT,VT)
    z=z.permute(3,0,1,2)
    z=torch.reshape(z,(1,2*nbasis,nx,nx)).to(gpu)

    #z = torch.randn((1,60,512,512),device=gpu, dtype=dtype)
    z = Variable(z,requires_grad=False)
    z1=torch.reshape(VT,(1,1,nbasis,NF))
    z1 = Variable(z1,requires_grad=False)
    #G.load_state_dict(torch.load('tempUfull.pt'))

        #%% class for genenrator based storm method   zz
    class AUV(nn.Module):
        def __init__(self,csmT,nbasis,bat_sz):
            super(AUV,self).__init__()
            self.csmT=csmT
            #self.mask=maskT
            self.nbasis=nbasis
            self.NF=bat_sz
            self.nch=csmT.shape[0]
            self.nx=csmT.shape[1]
            self.NX=self.nx*self.nx*2
            #self.tmp2=torch.FloatTensor(self.nch,self.nbasis,self.nx,self.nx,2).fill_(0)
            #self.tmp5=torch.FloatTensor(self.NF,self.nch*self.nx*self.nx*2).fill_(0)

        def forward(self,x,mask,v1):
            #tmp2=torch.FloatTensor(self.nch,self.nbasis,self.nx,self.nx,2).fill_(0)
            nbas=x.shape[0]
            m_res=ss.maskV(mask.cuda(),v1)
            m_res=m_res.unsqueeze(3).repeat(1,1,1,2)
            m_res=torch.reshape(m_res,(self.NF,nbasis,nx*nx*2))
            tmp5=torch.cuda.FloatTensor(self.nch,self.NF,self.nx*self.nx*2).fill_(0)
            x=torch.reshape(x,(nbas,self.nx,self.nx,2))
            for i in range(nch):
                tmp2=sf.pt_fft2c(sf.pt_cpx_multipy(x,self.csmT[i].repeat(nbas,1,1,1)))
                tmp2=torch.reshape(tmp2,(nbas,self.NX))
                tmp2=tmp2.repeat(self.NF,1,1)*m_res
                tmp5[i]=tmp2.sum(axis=1)

            del tmp2,x
            return tmp5.cpu()

    #%%

    torch.cuda.empty_cache()
    #res=maskV(maskTst,v1.detach().cpu().numpy())
    #maskT=torch.tensor(maskTst).to(dtype=torch.float32)

    loss=0
    #indx=torch.randint(0,NF,(NF,))
    #indx=torch.tensor(indx)
    #B=B[:,indx]
    #maskT=maskT[indx,]
    batch_sz=20
    bb_sz=int(NF/batch_sz)
    B=torch.reshape(kdataT,(nch,int(NF/batch_sz),batch_sz,nx*nx*2))
    maskT=torch.reshape(maskT,(int(NF/batch_sz),batch_sz,nx*nx))

    F=AUV(csmT,nbasis,batch_sz)
    #maskT=maskT.to(gpu)

    sf.tic()
    for ep1 in range(30):
        #inx=torch.randint(0,bb_sz,(bb_sz,))
        for bat in range(bb_sz):
            v1=GV(z1).squeeze(0).squeeze(0)
            v1=v1.permute(1,0)
            v1=torch.reshape(v1,(int(NF/batch_sz),batch_sz,nbasis))
            #mask_b=maskT[bat]

            u1=G(z)
            u1=torch.reshape(u1,(2,nbasis,nx,nx))
            u1=u1.permute(1,2,3,0)
            u1=torch.reshape(u1,(nbasis,nx*nx*2))
            b_est=F(u1,maskT[bat],v1[bat])

            #loss = criterion(out, target) + l1_regularization
            loss=(b_est-B[:,bat]).pow(2).sum()#+0.1*l1_reg#+(sT@W*u1).pow(2).sum()
            #if ep1%5==0:
            print(ep1,loss.item())

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            scheduler.step(loss.detach())
    #    if ep1%20==0:
    #        torch.save(u1,'res_'+str(ep1)+'.pt')
    sf.toc()
    if fl>=40 or fl==20:
        wtsFname1='wts-4U'+str(fl+1)+'.pt'
        torch.save(G.state_dict(),wtsFname1)
        wtsFname2='wts-4V'+str(fl+1)+'.pt'
        torch.save(GV.state_dict(),wtsFname2)
#%%
#rec = np.squeeze(u1.detach().cpu().numpy())
#rec=np.reshape(rec,(nbasis,nx,nx,2))
#rec = rec[:,:,:,0] + 1j*rec[:,:,:,1]
#for i in range(30):
#    plt.imshow((np.squeeze(np.abs(rec[i,:,:]))),cmap='gray')
#    plt.show()
#    plt.pause(0.05)
##xx=np.fft.fftshift(np.fft.fftshift(atbV,1),2)
#rec=np.reshape(rec,(nbasis,nx*nx))
#rec1=rec.T@V[:,0:NF]
#rec1=np.reshape(rec1,(nx,nx,NF))
##rec1=np.fft.fftshift(np.fft.fftshift(rec1,0),1)
##plt.figure(1,[15,15])
#for i in range(100):
#    plt.imshow((np.squeeze(np.abs(rec1[:,:,i]))),cmap='gray')
#    plt.show()
#    plt.pause(0.05)


##############################################################################
#
# Copyright (c) 2010 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""DottedName field tests
"""
from unittest import main, makeSuite

from six import b
from zope.schema import DottedName
from zope.schema.tests.test_field import FieldTestBase
from zope.schema.interfaces import InvalidDottedName, RequiredMissing

class DottedNameTest(FieldTestBase):
    """Test the DottedName Field."""

    _Field_Factory = DottedName

    def testValidate(self):
        field = self._Field_Factory(required=False)

        field.validate(None)
        field.validate('foo.bar')
        field.validate('foo.bar0')
        field.validate('foo0.bar')

        # We used to incorrectly allow ^: https://bugs.launchpad.net/zope.schema/+bug/191236
        self.assertRaises(InvalidDottedName, field.validate, 'foo.bar^foobar')
        self.assertRaises(InvalidDottedName, field.validate, 'foo^foobar.bar')
        # dotted names cannot start with digits
        self.assertRaises(InvalidDottedName, field.validate, 'foo.0bar')
        self.assertRaises(InvalidDottedName, field.validate, '0foo.bar')

    def testValidateRequired(self):
        field = self._Field_Factory(required=True)

        field.validate('foo.bar')

        self.assertRaises(RequiredMissing, field.validate, None)

def test_suite():
    suite = makeSuite(DottedNameTest)
    return suite


"""
ContractShipping Canada Post API
https://www.canadapost.ca/cpo/mc/business/productsservices/developers/services/shippingmanifest/default.jsf
"""
import logging
from lxml import etree
import requests
from canada_post.service import ServiceBase, CallLinkService
from canada_post.util import InfoObject

class Shipment(InfoObject):
    """
    Shipment class, is the return value of the CreateShipment service.
    It contains
      * tracking pin
      * return tracking pin
      * [shipment ]id
      * [shipment ]status
      * links is a dict of string --> dict where the keys are the rel
         attribute of each link (see the CreateShipment docs in the canadapost
         site. See the CreateShipment docstring for a link) and the keys have an
         'href' which is the link, the same 'rel' value and some other
         attributes, depending on each link
    """
    def __init__(self, xml=None, **kwargs):
        if xml is not None:
            self._from_xml(xml)
        super(Shipment, self).__init__(**kwargs)

    def _from_xml(self, xml):
        # I do this this way because I can't expect all return codes to have all
        #  values, I just fill in every Simple element in self
        for child in xml.getchildren():
            if child.tag == "shipment-id":
                self.id = child.text
            elif child.tag == "shipment-status":
                self.status = child.text
            elif child.tag == "links":
                # I can't make these into InfoObjects because there's a `self`
                #  rel object
                self.links = dict((link['rel'], link)
                    for link in map(lambda l: dict(l.attrib),
                                    child.findall("link")))
            else:
                # expected "tracking-pin" "return-tracking-pin"
                attrname = child.tag.replace("-", "_")
                setattr(self, attrname, child.text)

class CreateShipment(ServiceBase):
    """
    CreateShipment Canada Post API (for ContractShipping)
    https://www.canadapost.ca/cpo/mc/business/productsservices/developers/services/shippingmanifest/createshipment.jsf
    """
    URL ="https://{server}/rs/{customer}/{mobo}/shipment"
    log = logging.getLogger('canada_post.service.contract_shipping'
                            '.CreateShipment')
    def __init__(self, auth, url=None):
        if url:
            self.URL = url
        super(CreateShipment, self).__init__(auth)

    def set_link(self, url):
        """
        Sets the link in case you want to set it after initialization
        """
        self.URL = url

    def get_url(self):
        return self.URL.format(server=self.get_server(),
                               customer=self.auth.customer_number,
                               mobo=self.auth.customer_number)

    def _create_address_detail(self, parent, address, add_child):
        """
        Creates the address detail part of the request. You need to do any
        checks outside of this call
        """
        addr_detail = add_child("address-details", parent)
        add_child("address-line-1", addr_detail).text = address.address1
        add_child("address-line-2", addr_detail).text = address.address2
        add_child("city", addr_detail).text = address.city
        if address.province:
            add_child("prov-state", addr_detail).text = address.province
        add_child("country-code", addr_detail).text = address.country_code
        if address.postal_code:
            add_child("postal-zip-code",
                      addr_detail).text = unicode(address.postal_code)
        else:
            assert address.country_code not in ("US", "CA"), (
                "Addresses within {} require "
                "a postal code").format(address.country_code)

        return addr_detail

    def __call__(self, parcel, origin, destination, service, group):
        """
        Create a shipping order for the given parcels

        parcel: must be a canada_post.util.parcel.Parcel
        origin: must be a canada_post.util.address.Origin instance
        destination: must be a canada_post.util.address.Destination instance
        service: must be a canada_post.service.Service instance with at least
            the code parameter set up
        group: must be a string or unicode defining the parcel group that this
            parcel should be added to
        """
        debug = "( DEBUG )" if self.auth.debug else ""
        self.log.info(("Create shipping for parcel %s, from %s to %s{debug}"
                       .format(debug=debug)), parcel, origin, destination)

        # shipment
        shipment = etree.Element(
            "shipment", xmlns="http://www.canadapost.ca/ws/shipment")
        def add_child(child_name, parent=shipment):
            return etree.SubElement(parent, child_name)
        add_child("group-id").text = group
        add_child("requested-shipping-point").text = unicode(origin.postal_code)
        delivery_spec = add_child("delivery-spec")
        add_child("service-code", delivery_spec).text = service.code

        # sender
        sender = add_child("sender", delivery_spec)
        if origin.name:
            add_child("name", sender).text = origin.name
        assert origin.company, ("The sender needs a company name for "
                                "Contract Shipping service")
        add_child("company", sender).text = origin.company
        assert origin.phone, ("The sender needs a phone for "
                              "Contact Shipping Service")
        add_child("contact-phone", sender).text = origin.phone

        #destination details, first assertions
        assert any((origin.address1, origin.address2)), (
            "The sender needs an address to Create Shipping")
        assert origin.city, "Need the sender's city to Create Shipping"
        assert origin.province, "Need the sender's province to Create Shipping"
        sender_details = self._create_address_detail(sender, origin, add_child)
        # done sender

        # destination
        dest = add_child("destination", delivery_spec)
        if destination.name:
            add_child("name", dest).text = destination.name
        else:
            # TODO: if the Deliver to Post Office option is used, the name
            #  element must be present for the destination.
            pass
        if destination.company:
            add_child("company", dest).text = destination.company
        if destination.extra:
            add_child("additional-address-info").text = destination.extra
        if destination.phone:
            add_child("client-voice-number", dest).text = destination.phone
        else:
            # Required for destination when service is one of
            #  Expedited Parcel-USA, Xpresspost-USA, Xpresspost-International,
            #  Priority Worldwide Parcel or Pak
            #  (USA.EP, USA.XP, INT.XP, USA.PW.PARCEL, USA.PW.PAK,
            #   INT.PW.PARCEL, INT.PW.PAK)
            if service.code in ("USA.EP", "USA.XP", "INT.XP", "USA.PW.PARCEL",
                                "USA.PW.PAK", "INT.PW.PARCEL", "INT.PK.PAK"):
                assert False, ("Service {code} requires destination to have a "
                               "phone number").format(code=service.code)

        # destination details, first assertions
        assert any((destination.address1, destination.address2)), (
            "Must have address to Create Shipping")
        if not destination.province:
            assert destination.country_code not in ("CA", "US"), (
                "Country code is required for international shippings")
        # and then creation
        dest_details = self._create_address_detail(dest, destination, add_child)
        # done destination

        # TODO: options
        #options = add_child("options", delivery_spec)

        # parcel
        parcel_chars = add_child("parcel-characteristics", delivery_spec)
        add_child("weight", parcel_chars).text = unicode(parcel.weight)
        if all((parcel.length > 0, parcel.width > 0, parcel.height > 0)):
            dims = add_child("dimensions", parcel_chars)
            add_child("length", dims).text = unicode(parcel.length)
            add_child("width", dims).text = unicode(parcel.width)
            add_child("height", dims).text = unicode(parcel.height)
        add_child("unpackaged", parcel_chars).text = ("true"
                                                      if parcel.unpackaged
                                                      else "false")

        # TODO: notification
        #notification = add_child("notification", delivery_spec)

        # TODO: print-preferences
        #print_preferences = add_child("print-preferences", delivery_spec)

        # preferences
        preferences = add_child("preferences", delivery_spec)
        add_child("show-packing-instructions", preferences).text = "false"
        # TODO: these two are actually optional (may be "false") for CA
        # shippings
        add_child("show-postage-rate", preferences).text = "false"
        add_child("show-insured-value", preferences).text = "false"

        # settlement-info
        settlement = add_child("settlement-info", delivery_spec)
        # TODO: set paid-by-customer if a different customer is paying for this
        #add_child("paid-by-customer", settlement).text = FOOBAR
        assert self.auth.contract_number, ("Must have a contract number for "
                                           "contract shipping")
        add_child("contract-id", settlement).text = self.auth.contract_number
        # TODO: can be CreditCard as well
        add_child("intended-method-of-payment", settlement).text = "Account"

        headers = {
            'Accept': "application/vnd.cpc.shipment-v2+xml",
            'Content-type': "application/vnd.cpc.shipment-v2+xml",
            'Accept-language': "en-CA",
        }
        url = self.get_url()
        self.log.info("Using url %s", url)
        request = etree.tostring(shipment, pretty_print=self.auth.debug)
        self.log.debug("Request xml: %s", request)
        response = requests.post(url=url, data=request, headers=headers,
                                 auth=self.userpass())
        self.log.info("Request returned with status %s", response.status_code)
        self.log.debug("Request returned content: %s", response.content)

        if not response.ok:
            response.raise_for_status()

        # this is a hack to remove the namespace from the response, since this
        #breaks xpath lookup in lxml
        restree = etree.XML(response.content.replace(' xmlns="',
                                                     ' xmlnamespace="'))
        return Shipment(xml=restree)

class VoidShipment(CallLinkService):
    """
    Cancel a Contract Shipping created Shipment created by CreateShipment
    """
    log = logging.getLogger("canada_post.service.contract_shipping"
                            ".VoidShipment")
    link_rel = 'self'


import FWCore.ParameterSet.Config as cms

process = cms.Process('TEXTDUMP')
process.load("FWCore.MessageService.MessageLogger_cfi")
process.MessageLogger.cerr.FwkReport.reportEvery = 1
process.maxEvents = cms.untracked.PSet( input = cms.untracked.int32(100))
#process.SimpleMemoryCheck = cms.Service("SimpleMemoryCheck",ignoreTotal = cms.untracked.int32(1))
process.load("L1TriggerConfig.L1ScalesProducers.L1MuTriggerScalesConfig_cff")
process.load("L1TriggerConfig.L1ScalesProducers.L1MuTriggerPtScaleConfig_cff")
process.load("Configuration.StandardSequences.Geometry_cff")

process.options = cms.untracked.PSet(
    SkipEvent = cms.untracked.vstring('ProductNotFound')
)

process.L1TMuonText = cms.EDProducer(
    'L1TMuonTextDumper',
    doGen = cms.untracked.bool(True),
    genSrc = cms.untracked.InputTag("genParticles"),
    primitiveSrcs = cms.VInputTag(
    cms.InputTag('L1TMuonTriggerPrimitives','CSC'),
    cms.InputTag('L1TMuonTriggerPrimitives','DT'),
    cms.InputTag('L1TMuonTriggerPrimitives','RPC')
    ),
    converterSrcs = cms.VInputTag(
    cms.InputTag('L1CSCTFTrackConverter'),
    cms.InputTag('L1DTTFTrackConverter'),
    cms.InputTag('L1RPCbTFTrackConverter'),
    cms.InputTag('L1RPCfTFTrackConverter'),
    cms.InputTag('L1TMuonSimpleDeltaEtaHitMatcher')
    ),
    lutParam = cms.PSet(
    isBeamStartConf = cms.untracked.bool(True),
    ReadPtLUT = cms.bool(False),
    PtMethod = cms.untracked.uint32(32)
    )
)

process.content = cms.EDAnalyzer("EventContentAnalyzer")

process.sptf = cms.EDProducer(
    'sptf',
    primitiveSrcs = cms.VInputTag(
    cms.InputTag('L1TMuonTriggerPrimitives','CSC'),
    cms.InputTag('L1TMuonTriggerPrimitives','DT'),
    cms.InputTag('L1TMuonTriggerPrimitives','RPC')
    ),
    converterSrcs = cms.VInputTag(
    cms.InputTag('L1CSCTFTrackConverter'),
    cms.InputTag('L1DTTFTrackConverter'),
    cms.InputTag('L1RPCbTFTrackConverter'),
    cms.InputTag('L1RPCfTFTrackConverter'),
    cms.InputTag('L1TMuonSimpleDeltaEtaHitMatcher')
    ),
    lutParam = cms.PSet(
    isBeamStartConf = cms.untracked.bool(True),
    ReadPtLUT = cms.bool(False),
    PtMethod = cms.untracked.uint32(32)
    )
)

##process.L1TMuonVerilogBasedMatcher = cms.EDProducer(
##    'L1TMuonVerilogBasedMatcher',
##
##    primitiveSrcs = cms.VInputTag(
##    cms.InputTag('L1TMuonTriggerPrimitives','CSC')
#    cms.InputTag('L1TMuonTriggerPrimitives','CSC'),
#    cms.InputTag('L1TMuonTriggerPrimitives','DT'),
#    cms.InputTag('L1TMuonTriggerPrimitives','RPC')
##    ),
##    converterSrcs = cms.VInputTag(
##    cms.InputTag('L1CSCTFTrackConverter'),
##    cms.InputTag('L1DTTFTrackConverter'),
##    cms.InputTag('L1RPCbTFTrackConverter'),
##    cms.InputTag('L1RPCfTFTrackConverter'),
##    cms.InputTag('L1TMuonSimpleDeltaEtaHitMatcher')
##    ),
##    lutParam = cms.PSet(
##    isBeamStartConf = cms.untracked.bool(True),
##    ReadPtLUT = cms.bool(False),
##    PtMethod = cms.untracked.uint32(32)

##    )
##)

process.DiagMaker = cms.EDAnalyzer(
    'DiagMaker',
    testVar = cms.untracked.bool(False)
)

infile = 'file:L1TMuon2.root'
fileOutName = "test.root"

process.source = cms.Source(
    'PoolSource',
    fileNames = cms.untracked.vstring(infile)

    #eventsToProcess = cms.untracked.VEventRange('1:590','1:602','1:640','1:906','1:1029','1:1318','1:1622','1:1765','1:1817','1:2205','1:2206','1:2494','1:2541')
    #eventsToProcess = cms.untracked.VEventRange('1:1925-1:1927','1:2769-1:2771','1:3669-1:3671'),
    #eventsToSkip= cms.untracked.VEventRange('1:896')
    )

process.TFileService = cms.Service("TFileService",
    fileName = cms.string(
	fileOutName
))

#process.L1TMuonSequence = cms.Sequence(process.L1TMuonVerilogBasedMatcher * process.DiagMaker)
#process.L1TMuonSequence = cms.Sequence(process.L1TMuonVerilogBasedMatcher)
process.L1TMuonSequence = cms.Sequence(process.L1TMuonText)
#process.L1TMuonSequence = cms.Sequence(process.L1TMuonText * process.sptf * process.DiagMaker)

process.L1TMuonPath = cms.Path(process.L1TMuonSequence)

process.schedule = cms.Schedule(process.L1TMuonPath)


from common import *
import autograd.numpy as np
import matplotlib.pyplot as plt
import autograd.numpy.random as rng
import cov
from autograd.numpy.random import multivariate_normal as rmvn
from infalg import GradientAscent, AdaGrad
from autograd import grad
import fullgp
from DataStream import DataStream
from DataGrid import DataGrid
import time
import timeit

# Draw Rademacher variate.
def rademacher(M):
  s = rng.rand(M)
  idx = s >= 0.5
  out = -np.ones(M)
  out[idx] = 1.0
  return out

# Compute initial variational parameters. M is the number of pseudo-data.
#
# Inputs:
# M - the number of pseudo-data.
# phi_h - existing dictionary of parameters. Created if not provided.
#
# Outputs:
# phi_h - dictionary of parameters, transformed to be real-valued.
#
def InitVariationalParameters(M, phi_h={}):
  phi_h['m_q'] = np.zeros(M)
  phi_h['w_h'] = BoundedToReal(0.99 * np.ones(M), lb=0.0, ub=1.0)
  phi_h['eps_h'] = BoundedToReal(1e-4, lb=0.0)
  return phi_h

# Compute the parameters.
#
# Inputs:
# phi - dictionary of parameters. Should be unbounded.
#
# Outputs:
# m_q - mean of approximate posterior distribution.
# w - diagonal of de-correlation matrix.
# eps - auxilliary noise parameter.
#
def ComputeVariationalParameters(phi_h):
  return phi_h['m_q'],\
    RealToBounded(phi_h['w_h'], lb=0.0, ub=1.0),\
    RealToBounded(phi_h['eps_h'], lb=0.0)

# Compute the elbo (lower bound on the log marginal probability). See
# dissertation for explanation of the various opaquely-named quantities.
#
# Inputs:
# phi_h - transformed variational parameters.
# datastream - DataStream object to provide access to data.
# Z_grid - inducing point locations. (M)
# fcov - covariance function handle. (handle)
# s2n_h - noise parameter s2n := log(1 + exp(s2nh)). (scalar)
#
# Outputs:
# elbo - noisy estimate of the elbo.
#
def elbo(phi_h, Z_grid, datastream, fcov, s2n_h, prnt=False):

  # Get nex inputs and outputs from the data-stream object.
  Z, M, N = Z_grid.data, Z_grid.M, datastream.N
  X, y, L = datastream.GetNextMiniBatch()

  # Pre-compute invariant qtts.
  m_q, w, eps = ComputeVariationalParameters(phi_h)
  Gamma = ComputeSpectrum(fcov, Z) + eps
  s2n = RealToBounded(s2n_h, lb=0.0)
  K0, KXZ = fcov(np.zeros(1)).flatten(), fcov(X, Z)

  # Add likelihood expectation term using the last draws.
  mu = np.dot(KXZ, fft(ifft(m_q) / Gamma))
  elbo = lpdfDiagNormal(y, mu, np.array([s2n]), L) * N / L

  # Subtract trace term - Monte Carlo approximation used.
  Q = rng.randn(L, 1)
  GammaT = np.reshape(Gamma, [M, 1])
  R = ifft(np.reshape(w, [M, 1]) * fft(ifft(np.dot(KXZ.T, Q)) / GammaT))
  elbo -= 0.5 * N * np.sum(np.sum(sqrabs(R), 1) * Gamma) / (s2n * L)

  # Subtract deterministic part of KL-divegence with prior.
  W = np.diag(w)
  elbo -= 0.5 * (np.sum(sqrabs(ifft(m_q)) / Gamma) - M - 2 * np.sum(np.log(w)))

  # Subtract stochastic part of KL-divergence with prior.
  est, T = 0.0, 1
  for t in range(T):
    z = rng.randn(M)
    est += np.dot(z, np.real(fft(ifft(w * fft(Gamma * ifft(w * z))) / Gamma)))
  #z = rng.randn(M)
  #elbo -= 0.5 * np.dot(z, np.real(fft(ifft(w * fft(Gamma * ifft(w * z))) / Gamma)))
  #exact_cov = np.dot(ConstructCircCov(1.0 / Gamma), np.dot(W, np.dot(\
  #  ConstructCircCov(Gamma), W)))
  #exact = np.trace(exact_cov)
  #elbo -= 0.5 * exact
  elbo -= 0.5 * est / T
  if prnt:
    print('est', np.mean(est))
    print('var', np.var(est))
    print('exact', exact)
    plt.plot(est)
    plt.figure()
    plt.imshow(exact_cov)
    plt.colorbar()
    plt.show()
  #elbo -= 0.5 * exact

  return elbo

# Estimate the posterior mean and var at the input locations X given the
# inducing point locations Z, the variational parameters phi and the number of
# Monte Carlo samples over which to average T.
#
# Inputs:
# phi_h - transformed variational parameters.
# Z_grid - DataGrid object containing pseudo-data inputs.
# X - inputs. (N)
# fcov - covariance function handle. (handle)
# T - number of Monte Carlo samples to average over. (scalar)
#
# Outputs:
# mu - posterior mean estimates at X.
# s2 - posterior variance estimates at X.
#
def posterior(phi_h, Z_grid, X, fcov, T):

  # Pre-compute invariant qtts.
  N, M, Z = X.shape[0], Z_grid.M, Z_grid.data
  m_q, w, eps = ComputeVariationalParameters(phi_h)
  Gamma = ComputeSpectrum(fcov, Z) + eps
  KXZ = fcov(X, Z)

  Ktilde, Kself = ifft(KXZ[N / 2]), fcov(X[0:1], diag=True)

  # Directly compute the posterior mean.
  mu = np.real(np.dot(KXZ, fft(ifft(m_q) / Gamma)))

  # Compute Monte Carlo estimate of mu and s2.
  s2, K0 = np.zeros(N), fcov(np.array([0.0])).flatten()

  s2_cond = (Kself - np.sum(sqrabs(Ktilde) / Gamma))
  for t in range(T):

    # Draw samples for u.
    u_tilde = ifft(m_q + w * rndCircNormal(0.0, Gamma))
    mu_cond = np.real(np.dot(KXZ, fft(u_tilde / Gamma)))

    # Accumulate posterior mean values.
    s2 += (mu_cond - mu)**2

  # Average over Monte Carlo samples and return.
  return mu, s2 / T + s2_cond

# Knowing the kernel parameters, infer distribution over latent variables.
#
# Inputs:
# fcov - covariance function, implicit parameters.
# data_stream - DataStream object that containing the data.
# Z_grid - DataGrid object containing the pseudo-data input locations.
# phi_h - initial variational parameters. See InitVariationalParameters for
#         requirements.
# s2n_h - observation noise. (1), (>0)
# iters - number of iterations.
# infalg - inference algorithm function handle.
# collect_delta - number of iterations to wait between collecting elbo.
#
# Outputs:
# elbos - logged elbos.
#
def infer(fcov, datastream, Z_grid, phi_h, iters, inf_alg, collect_delta):

  # Define objective function and inference routine.
  L = lambda phi_h : elbo(phi_h, Z_grid, datastream, fcov, phi_h['s2n_h'])
  inf_obj = inf_alg(grad(L), phi_h)

  # Iterate for the specified amount of time.
  J, j = iters / collect_delta, 0
  elbos = np.empty(J)
  for itr in range(iters):
    inf_obj.update(phi_h)
    if itr % collect_delta == 0:
      elbos[j] = elbo(phi_h, Z_grid, datastream, fcov, phi_h['s2n_h'])
      print(elbos[j])
      j += 1

  return elbos

def main():
  pass

if __name__ == '__main__':

  # Define pseudo-data.
  lb, ub, N, s2n = 0.0, 10.0, 250, 1e-3
  extend, M, iters, collect_delta = 5.0, N / 2, 2500, 25
  Z_grid = DataGrid(lb - extend, ub + extend, M)

  # Define the covariance matrix.
  print('Define covariance function.')
  pars = cov.init_eq(s2=1.0, l2=1.0)
  delta = (ub - lb + 2 * extend)
  fcov = cov.circ_factory(cov.eq, delta, pars)

  # Generate some data and define streaming object.
  print('Generate toy data.')
  rng.seed(15485863)
  X1 = rng.uniform(low=lb, high=ub / 3.0, size=N / 2)
  X2 = rng.uniform(low=ub * 2.0 / 3.0, high=ub, size=N / 2)
  X = rng.permutation(np.hstack([X1, X2]))
  #X = rng.uniform(low=lb, high=ub, size=N)
  #X = np.linspace(lb, ub, N)
  y = rmvn(np.zeros(N), fcov(X, X) + s2n * np.eye(N))
  datastream = DataStream(25, X, y)

  # Define initial variational parameters and inference routine.
  print('Define variational parameters and inference.')
  phi_h = InitVariationalParameters(M)
  phi_h['s2n_h'] = BoundedToReal(s2n, lb=0.0)
  inf_alg = lambda df, phi_h : AdaGrad(df, 1.0, phi_h)

  # Perform inference.
  print('Perform inference.')
  elbos = infer(fcov, datastream, Z_grid, phi_h, iters, inf_alg, collect_delta)

  # Display results.
  print('Plot results.')

  # Print elbos.
  plt.plot(elbos[2:])
  plt.title('Elbos')

  # Print posterior with uncertainty etc.
  Z = Z_grid.data
  m_q, w, eps = ComputeVariationalParameters(phi_h)
  W = np.diag(w)
  Gamma = ComputeSpectrum(fcov, Z) + eps
  Sigma_q = w ** 2 * np.mean(Gamma)
  std2_q = 2*np.sqrt(Sigma_q)
  Xhat = np.linspace(lb - extend / 1, ub + extend / 1, 1000)
  mu, s2 = posterior(phi_h, Z_grid, Xhat, fcov, 500)
  std_2 = 2 * np.sqrt(s2)
  fullgp_posterior, lml = fullgp.infer(X, y, fcov, s2n)
  mu_u, s2_u, Sigma_u = fullgp_posterior(Z)
  mu_u, std2_u = mu_u(), 2 * np.sqrt(s2_u())
  plt.figure()
  plt.plot(Xhat, mu, 'b', Xhat, mu + std_2, 'b--', Xhat, mu - std_2, 'b--',\
    Z, np.zeros(Z.shape), 'kx', X, y, 'rx',\
    Z, m_q, 'g', Z, m_q + std2_q, 'g--', Z, m_q - std2_q, 'g--',\
    Z, mu_u, 'r', Z, mu_u + std2_u, 'r--', Z, mu_u - std2_u, 'r--')
  plt.title('mu')

  # Plot the approximate and exact posterior covariance matrices.
  Sigma_q = np.real(np.dot(W, np.dot(ConstructCircCov(Gamma), W)))
  plt.figure()
  plt.imshow(Sigma_q)
  plt.title('q Sigma')
  plt.colorbar()
  plt.figure()
  plt.imshow(Sigma_u())
  plt.title('u Sigma')
  plt.colorbar()
  plt.figure()
  plt.plot(Sigma_q[M/2], 'b', Sigma_u()[M/2], 'r')
  plt.title('Sigma M/2')
  plt.figure()
  plt.plot(Sigma_q[M/2] - Sigma_u()[M/2])
  plt.title('Sigma delta')

  plt.show()


# Download stopwords from NLTK
import nltk

nltk.download('stopwords')

from nltk.corpus import stopwords
import numpy as np
import string
import re

stop = stopwords.words('english')

# Remove stopwords
def rmStopwords(text):
    try:
        result = text.lower().split()
        result = ' '.join(word for word in result if word not in stop)
        return result
    except: return ''

# Remove normal and non-ASCII punctuation
def cleanText(text):
    try:
        result = text.lower()
        result = re.sub(r'[^\x00-\x7F]+', ' ', result)
        result = result.translate(str.maketrans('', '', string.punctuation))
        return result
    except: return ''

# Remove empty lines and nans
def rmBadLines(data, column):
    data = data[data[column] != '']
    data = data[data[column] != ' ']
    data = data[data[column] != np.nan]
    data.drop_duplicates(inplace=True)
    data.reset_index(drop=True, inplace=True)
    return data

import pandas as pd
import pyodbc

#from procFuncs import *

#----- Define path to save data
data_dir = '/Users/kdegrave/Documents/Projects/Search/data/'

#----- Get specialty, condition, procedure, facility, and provider data
con1 = pyodbc.connect(driver='{SQL Server}', server='HGPRODMDMDB01', database='HealthMaster')

q_spec = "SELECT refSpecialtyDescription, refSpecialistDescription, refSpecialistsDescription from HealthMaster.ref.Specialty"
q_cond = "SELECT refConditionDescription from HealthMaster.ref.Condition"
q_faci = "SELECT refFacilityName from HealthMaster.ref.Facility"
q_proc = "SELECT refProcedureMedicalDescription, refProcedureMedicalDescriptions from HealthMaster.ref.ProcedureMedical"

df_spec = pd.read_sql(q_spec, con1)
df_faci = pd.read_sql(q_faci, con1)
df_proc = pd.read_sql(q_proc, con1)
df_cond = pd.read_sql(q_cond, con1)

con2 = pyodbc.connect(driver='{SQL Server}', server='HGPRODBIPROC01', database='ODS1')

q_prov = "SELECT FirstName, MiddleName, LastName from ODS1.Base.Provider"

df_prov = pd.read_sql(q_prov, con2)

#----- Apply functions to clean text
df_spec = df_spec.melt().drop('variable', axis=1)
df_spec.columns = ['refSpecialty']

df_proc = df_proc.melt().drop('variable', axis=1)
df_proc.columns = ['refProcedure']

df_spec['refSpecialty'] = df_spec['refSpecialty'].apply(cleanText).apply(rmStopwords)
df_proc['refProcedure'] = df_proc['refProcedure'].apply(cleanText).apply(rmStopwords)

df_faci['refFacilityName'] = df_faci['refFacilityName'].apply(cleanText).apply(rmStopwords)
df_faci.columns = ['refFacility']

df_cond['refConditionDescription'] = df_cond['refConditionDescription'].apply(cleanText).apply(rmStopwords)
df_cond.columns = ['refCondition']

df_cond = rmBadLines(df_cond, 'refCondition')
df_spec = rmBadLines(df_spec, 'refSpecialty')
df_proc = rmBadLines(df_proc, 'refProcedure')
df_faci = rmBadLines(df_faci, 'refFacility')

#----- Combine provider firstname, middlename, and lastname
df_prov.fillna(' ', inplace=True)

df_prov['refProvider'] = df_prov['FirstName'] + ' ' + df_prov['MiddleName'] + ' ' + df_prov['LastName']
df_prov = df_prov[['refProvider']]
df_prov = rmBadLines(df_prov, 'refProvider')

#----- Remove duplicate entries from facilities, then conditions, then procedures
lcond = df_cond['refCondition'].tolist()
lspec = df_spec['refSpecialty'].tolist()
lproc = df_proc['refProcedure'].tolist()
lfaci = df_faci['refFacility'].tolist()
lprov = df_prov['refProvider'].tolist()

m = lprov + lproc + lcond + lspec
df_faci = pd.DataFrame()
df_faci['refFacility'] = np.setdiff1d(lfaci, m)

m = lprov + lfaci + lproc + lspec
df_cond = pd.DataFrame()
df_cond['refCondition'] = np.setdiff1d(lcond, m)

m = lprov + lfaci + lcond + lspec
df_proc = pd.DataFrame()
df_proc['refProcedure'] = np.setdiff1d(lproc, m)

df_proc = df_proc.drop_duplicates().reset_index(drop=True)
df_spec = df_spec.drop_duplicates().reset_index(drop=True)
df_cond = df_cond.drop_duplicates().reset_index(drop=True)
df_faci = df_faci.drop_duplicates().reset_index(drop=True)
df_prov = df_prov.drop_duplicates().reset_index(drop=True)

#----- Save data
df_prov.to_csv(data_dir + 'provider_gazette_cleaned.csv', index=False)
df_spec.to_csv(data_dir + 'specialty_gazette_cleaned.csv', index=False)
df_cond.to_csv(data_dir + 'condition_gazette_cleaned.csv', index=False)
df_proc.to_csv(data_dir + 'procedure_gazette_cleaned.csv', index=False)
df_faci.to_csv(data_dir + 'facility_gazette_cleaned.csv', index=False)


import os
import nltk
from Stemmer import *
from Lemmatizer import *
from Normalizer import *
import math
import operator
import numpy as np
import collections

class Appearance:

    def __init__(self, docId, frequency):
        self.docId = docId
        self.frequency = frequency

    def __repr__(self):
        return str(self.__dict__)

from heapq import heappop, heappush

def heap_sort(array):
    heap = []
    for element in array:
        heappush(heap, element)

    ordered = []

    while heap:
        ordered.append(heappop(heap))

    return ordered

def read_data(path):
    contents = []
    for filename in os.listdir(path):
        data = open(path + '/' + filename, 'r', encoding='utf8').read()
        filename = re.sub('\D', "", filename)
        contents.append({
            'docId': (int(filename)),
            'text': data
        })
    return contents

def tokenize_and_remove_punctuations(s):
    single_chars = ['»', '.', '«', '،', '؟', '"', '#', ')', '(', '*', ',', '-', '/', ':', '[', ']', '،', '?', '…', '۰',
                    '۱', '۲', '۳', '۴', '۵', '۶', '۷', '۸', '۹', '-', '+', '=', '@', '$', '$', '%', '^', '&', '-', '_',
                    '{', '}', '\'', '/', ';', '  ', '>', '<', '•', '٪', '؛']
    text = s.strip(u'\xa0\u200c \t\n\r\f\v')
    text = text.replace('\n', ' ')
    text = Normalizer().normalize(text)
    # print(text)
    for sc in single_chars:
        if text.__contains__(sc):
            text = text.replace(sc, ' ')
    terms = text.split(' ')
    for term in terms:
        if term.__eq__(''):
            terms.remove(term)
        if term.isnumeric():
            terms.remove(term)
    return terms

def get_stopwords():
    stop_words = [word for word in open('stopwords.txt', 'r', encoding='utf8').read().split('\n')]
    return stop_words

def stem_words(tokens):
    stemmer = Stemmer()
    stemmed_words = [stemmer.stem(token) for token in tokens]
    return stemmed_words

def lem_words(tokens):
    lemmetizer = Lemmatizer()
    stemmed_words = [lemmetizer.lemmatize(token) for token in tokens]
    return stemmed_words

def remove_stop_words(tokens):
    stop_words = get_stopwords()
    filtered_words = [token for token in tokens if token not in stop_words and len(token) > 2]
    return filtered_words

def preprocess_data(contents):
    dataDict = {}
    for content in contents:
        tokens = tokenize_and_remove_punctuations(content['text'])
        filtered_tokens = remove_stop_words(tokens)
        stemmed_tokens = stem_words(filtered_tokens)
        lem_tokens = lem_words(filtered_tokens)
        filtered_tokens1 = remove_stop_words(lem_tokens)
        # dataDict[content['docId']] = lem_tokens
        dataDict[content['docId']] = filtered_tokens1
        # dataDict[content['docId']] = filtered_tokens
        # dataDict[content['docId']] = tokens
        # dataDict[content['docId']] = stemmed_tokens
    return dataDict

def preprocess_queries(input_query):
    queriesDict = {}
    queries = input_query.split(' ')
    i = 1
    for query in queries:
        tokens = tokenize_and_remove_punctuations(query)
        filtered_tokens = remove_stop_words(tokens)
        stemmed_tokens = stem_words(filtered_tokens)
        lem_tokens = lem_words(filtered_tokens)
        filtered_tokens1 = remove_stop_words(lem_tokens)
        queriesDict[i] = filtered_tokens1
        i += 1
    return queriesDict

def get_vocabulary(data):
    tokens = []
    for token_list in data.values():
        tokens = tokens + token_list
    fdist = nltk.FreqDist(tokens)
    return list(fdist.keys())

def generate_inverted_index(data):
    all_words = get_vocabulary(data)
    index = {}
    for word in all_words:
        for doc, tokens in data.items():
            if word in tokens:
                if word in index.keys():
                    index[word].append(doc)
                else:
                    index[word] = [doc]
    return index

# def create_inverted_index(data):
#     words = {word: [] for word in np.unique(np.concatenate(list(data.values())))}
#     dicts_keys = data.keys()
#     for key in dicts_keys:
#         u_vocab, counts = np.unique(data[key], return_counts=True)
#         for i in range(len(u_vocab)):
#             words[u_vocab[i]].append({'docID': key, 'frequency': counts[i]})
#     od = collections.OrderedDict(sorted(words.items()))
#     return od

def calculate_tf(tokens):
    tf_score = {}
    for token in tokens:
        tf_score[token] = tokens.count(token)
    return tf_score

def calculate_idf(data):
    idf_score = {}
    N = len(data)
    all_words = get_vocabulary(data)
    for word in all_words:
        word_count = 0
        for token_list in data.values():
            if word in token_list:
                word_count += 1
        idf_score[word] = math.log10(N / word_count)
    return idf_score

def calculate_tfidf(data, idf_score):
    scores = {}
    for key, value in data.items():
        scores[key] = calculate_tf(value)
    for doc, tf_scores in scores.items():
        for token, score in tf_scores.items():
            tf = score
            idf = idf_score[token]
            tf_scores[token] = (1 + math.log10(tf)) * idf
    return scores

def calculate_tfidf_queries(queries, idf_score):
    scores = {}
    for key, value in queries.items():
        scores[key] = calculate_tf(value)
    for key, tf_scores in scores.items():
        for token, score in tf_scores.items():
            idf = 0
            tf = score
            if token in idf_score.keys():
                idf = idf_score[token]
            tf_scores[token] = (1 + math.log10(tf)) * idf
    return scores
lemmatizer = Lemmatizer()
normalizer = Normalizer()
print(normalizer.normalize('می روم'))
print(lemmatizer.lemmatize(normalizer.normalize("می روم")))
# stemmer = Stemmer()
# print(stemmer.stem(normalizer.normalize("رفته بودم")))
# data = read_data("docs")
# preprocessed_data = preprocess_data(data)
#
# inverted_index = generate_inverted_index(preprocessed_data)
# print(inverted_index)
# # inverted_index = create_inverted_index(preprocessed_data)
# query = "لیگ قهرمانان آسیا"
# queries = preprocess_queries(query)
# print(inverted_index)
# idf_scores = calculate_idf(preprocessed_data)
# data_scores = calculate_tfidf(preprocessed_data, idf_scores)
# query_scores = calculate_tfidf_queries(queries, idf_scores)
# # print(query_scores)
# # print(data_scores)
#
#
# query_docs = {}
# for key, value in queries.items():
#     doc_sim = {}
#     for term in value:
#         if term in inverted_index.keys():
#             docs = inverted_index[term]
#             # print(docs)
#             for doc in docs:
#                 doc_score = data_scores[doc][term]
#                 doc_length = math.sqrt(sum(x ** 2 for x in data_scores[doc].values()))
#                 query_score = query_scores[key][term]
#                 query_length = math.sqrt(sum(x ** 2 for x in query_scores[key].values()))
#                 cosine_sim = (doc_score * query_score) / (doc_length * query_length)
#                 if doc in doc_sim.keys():
#                     doc_sim[doc] += cosine_sim
#                 else:
#                     doc_sim[doc] = cosine_sim
#     ranked = sorted(doc_sim.items(), key=operator.itemgetter(1), reverse=True)
    # print(ranked)
    # ranked = heap_sort(doc_sim.items())
    # print(ranked)
    # query_docs[key] = ranked

# print(query_docs)


from flaskr.fetch import index
import os
from flask_fontawesome import FontAwesome
from flask import Flask
from flask import (
    Blueprint, flash, g, redirect, render_template, request, session, url_for
)
from flask_pymongo import PyMongo
from flask_bootstrap import Bootstrap
from flaskr.auth import login_required
# search import
import json
from flaskr.search.index import Index
from flaskr.search.timing import timing
import requests
import os.path
from flaskr.search.documents import Abstract
import time
from lxml import etree
import gzip
from flaskr.db import get_db
from math import ceil
# -------------------------------------

def load_documents():
    start = time.time()
    db = get_db()
    jobs = db.jobs_info.find()
    jobs = list(jobs)
    print(len(jobs))
    doc_id = 0
    for job in jobs:
        # print(doc_id)
        if(job['cleanTitle']):
            title = job['cleanTitle']
        else:
            title = " "
        # url = job['url']
        url = job['url']
        address = job['address']
        company = job['company']
        maxSalary = job['maxSalary']
        yield Abstract(ID=doc_id, title=title, url=url, address=address, company=company, maxSalary=maxSalary)
        doc_id += 1

    end = time.time()
    print(f'Parsing XML took {end - start} seconds')

def index_documents(documents, index):
    for i, document in enumerate(documents):
        index.index_document(document)
        # if i % 50 == 0:
        #     print(f'Indexed {i} documents', end='\r')
    return index

def create_app(test_config=None):
    # create and configure the app
    app = Flask(__name__, instance_relative_config=True)
    app.config.from_mapping(
        SECRET_KEY='dev'
    )
    app.config["MONGO_URI"] = "mongodb://localhost:27017/data_intergration"
    Bootstrap(app)
    mongo = PyMongo(app)
    app.db = mongo.db

    if test_config is None:
        # load the instance config, if it exists, when not testing
        app.config.from_pyfile('config.py', silent=True)
    else:
        # load the test config if passed in
        app.config.from_mapping(test_config)

    # ensure the instance folder exists
    try:
        os.makedirs(app.instance_path)
    except OSError:
        pass

    # Register blueprint
    from . import salaryPredict
    app.register_blueprint(salaryPredict.bp)
    from . import charts
    app.register_blueprint(charts.bp)
    from . import auth
    app.register_blueprint(auth.bp)
    from . import fetch
    app.register_blueprint(fetch.bp)
    app.add_url_rule('/', endpoint='index')
    app.add_url_rule('/home', endpoint='home')
    from . import search1
    app.register_blueprint(search1.bp)
    # a simple page that says hello

    with app.app_context():
        index = index_documents(load_documents(), Index())
        print(f'Index contains {len(index.documents)} documents')

    # @app.route('/home', methods=('GET', 'POST'))
    # def home():
    #     if request.method == 'GET':
    #         text = request.args.get('search_text')
    #         jobs = index.search(text, search_type='AND')
    #         return render_template("jobs/home.html", jobs=jobs, page=1)

    @app.route("/home", methods=('GET', 'POST'))
    @login_required
    def home():
        global db
        page_num = 1
        db = get_db()
        text_s = ""
        jobs = db.jobs_info.find()
        if request.args.get('page_num'):
            page_num = int(request.args.get('page_num'))
        if request.args.get('search_text'):
            text = request.args.get('search_text')
            text_s = request.args.get('search_text')
            if(text != None):
                jobs = index.search(text, search_type='AND')

        jobs = list(jobs)
        jobs = jobs[(page_num-1)*20:page_num*20]
        jobs_count = db.jobs_info.count()
        page_count = ceil(jobs_count/20)
        page = f"{page_num}/{page_count}"
        return render_template("jobs/home.html", jobs=jobs, page=page, jobs_count=jobs_count, text_s=text_s, page_count=page_count)
        # @app.route('/')
        # def home_page():
        #     jobs_info = mongo.db.jobs_info
        #     cur = jobs_info.find({
        #         "address": {"$regex": "Hoàng Quốc Việt+"}
        #     })
        #     job_list = []
        #     for job in cur:
        #         job_list.append(job)
        #     return "\n".join(str(job_list))

    return app


from django.urls import path
from . import views

app_name = 'docs_templates'

urlpatterns = [
	path('', views.List_of_documentsView.as_view(), name = 'list_of_documents'),
	path('<str:document_url>/', views.documents_context, name = 'document_template'),

]


"""
Train the word2vec embeddings
"""
import gensim, logging
from gensim.test.utils import common_texts, get_tmpfile
from gensim.models import Word2Vec
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

class MySentences(object):
    def __init__(self, fname, N):
        self.fname = fname
        self.N = N

    def __iter__(self):
        with open(self.fname, 'rb') as f:
            for i,line in enumerate(f):
                if i%N==0:
                    yield gensim.utils.simple_preprocess(line)

if __name__=='__main__':

    _inpt = ""
    _size = 80
    _window = 10
    N = 10

    input_file = "dataset/wiki_en"+_inpt+".txt"
    sentences = MySentences(input_file, N)

    model = Word2Vec(
        sentences,
        size=_size,
        window=_window,
        min_count=5,
        workers=10)
    """ Discussion on parameter 'iter'
        有些人會調iter=? (也就是EPOCH) default是5、大一點會比較準，可是:
        1. Word embeddings準，和用起來好不好常常 沒 有 關係
        2. 調幾次就要跑幾遍(dramatically increase the running time)
        => 有多餘時間再試試別的iter
    """
    model.save("word2vec{0}_s{1}_w{2}.model".format(_inpt,_size,_window))


from __future__ import print_function
import time
import torch.nn as nn

import argparse
import torch.backends.cudnn as cudnn
import torch.nn.functional as F
import torch.optim as optim
import torch.utils.data.distributed
from torchvision import models, datasets, transforms
import horovod.torch as hvd
import timeit
import numpy as np

import horovod.torch as hvd
from grace_dl.torch.communicator.allgather import Allgather
from grace_dl.torch.communicator.allreduce import Allreduce
from grace_dl.torch.communicator.broadcast import Broadcast
from grace_dl.torch.compressor.topk import TopKCompressor
from grace_dl.torch.memory.residual import ResidualMemory

from grace_dl.torch.compressor.none import NoneCompressor
from grace_dl.torch.memory.none import NoneMemory

from grace_dl.torch.compressor.powersgd  import PowerSGDCompressor
from grace_dl.torch.memory.powersgd import PowerSGDMemory

from grace_dl.torch.compressor.efsignsgd import EFSignSGDCompressor
from grace_dl.torch.memory.efsignsgd import EFSignSGDMemory

from grace_dl.torch.compressor.qsgd import QSGDCompressor
from grace_dl.torch.compressor.onebit import OneBitCompressor
from grace_dl.torch.compressor.natural import NaturalCompressor
from grace_dl.torch.compressor.fp16 import FP16Compressor
from grace_dl.torch.compressor.randomk import RandomKCompressor
from grace_dl.torch.compressor.signsgd import SignSGDCompressor
from grace_dl.torch.compressor.signum import SignumCompressor
from grace_dl.torch.compressor.terngrad import TernGradCompressor
from grace_dl.torch.compressor.threshold import ThresholdCompressor

# Benchmark settings
parser = argparse.ArgumentParser(description='PyTorch Synthetic Benchmark',
                                 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--fp16-allreduce', action='store_true', default=False,
                    help='use fp16 compression during allreduce')
parser.add_argument('--model', type=str, default='resnet50',
                    help='model to benchmark')
parser.add_argument('--batch-size', type=int, default=32,
                    help='input batch size')
parser.add_argument('--num-warmup-batches', type=int, default=10,
                    help='number of warm-up batches that don\'t count towards benchmark')
parser.add_argument('--num-batches-per-iter', type=int, default=10,
                    help='number of batches per benchmark iteration')
parser.add_argument('--num-iters', type=int, default=10,
                    help='number of benchmark iterations')
parser.add_argument('--no-cuda', action='store_true', default=False,
                    help='disables CUDA training')
parser.add_argument('--use-adasum', action='store_true', default=False,
                    help='use adasum algorithm to do reduction')

parser.add_argument('--epochs', type=int, default=10, metavar='N',
                    help='number of epochs to train (default: 10)')
parser.add_argument('--lr', type=float, default=0.01, metavar='LR',
                    help='learning rate (default: 0.01)')
parser.add_argument('--momentum', type=float, default=0.5, metavar='M',
                    help='SGD momentum (default: 0.5)')
parser.add_argument('--seed', type=int, default=42, metavar='S',
                    help='random seed (default: 42)')
parser.add_argument('--log-interval', type=int, default=10, metavar='N',
                    help='how many batches to wait before logging training status')

args = parser.parse_args()
args.cuda = not args.no_cuda and torch.cuda.is_available()

hvd.init()
#torch.manual_seed(args.seed)

if args.cuda:
    # Horovod: pin GPU to local rank.
    torch.cuda.set_device(hvd.local_rank())
    # torch.cuda.manual_seed(args.seed)

cudnn.benchmark = True

# Set up standard model.
model = getattr(models, args.model)()

# By default, Adasum doesn't need scaling up learning rate.
lr_scaler = hvd.size() if not args.use_adasum else 1

if args.cuda:
    # Move model to GPU.
    model.cuda()
    # If using GPU Adasum allreduce, scale learning rate by local_size.
    if args.use_adasum and hvd.nccl_built():
        lr_scaler = hvd.local_size()

# optimizer = optim.SGD(model.parameters(), lr=0.01 * lr_scaler)

# Horovod: (optional) compression algorithm.
# compression = hvd.Compression.fp16 if args.fp16_allreduce else hvd.Compression.none

# Horovod: wrap optimizer with DistributedOptimizer.
# optimizer = hvd.DistributedOptimizer(optimizer,
#                                      named_parameters=model.named_parameters(),
#                                      compression=compression,
#                                      op=hvd.Adasum if args.use_adasum else hvd.Average)

# new add - Horovod: scale learning rate by the number of GPUs.
optimizer = optim.SGD(model.parameters(), lr=args.lr * hvd.size(),
                      momentum=args.momentum)

# Horovod: broadcast parameters & optimizer state.
hvd.broadcast_parameters(model.state_dict(), root_rank=0)
hvd.broadcast_optimizer_state(optimizer, root_rank=0)

# GRACE: compression algorithm.
#grc = Allgather(TopKCompressor(0.01), ResidualMemory(), hvd.size())
#grc = Allgather(RandomKCompressor(0.01), ResidualMemory(), hvd.size())
#grc = Allgather(ThresholdCompressor(0.01), ResidualMemory(), hvd.size())
#grc = Allgather(FP16Compressor(), ResidualMemory(), hvd.size())
#grc = Allgather(TernGradCompressor(), ResidualMemory(), hvd.size())
grc = Allgather(SignumCompressor(0.9), ResidualMemory(), hvd.size())
#grc = Allgather(SignSGDCompressor(), ResidualMemory(), hvd.size())
#compressor = OneBitCompressor()
#memory = EFSignSGDMemory(lr=0.1)
#memory = ResidualMemory()
#grc = Broadcast(compressor, memory, hvd.size())
#grc = Broadcast(EFSignSGDCompressor(lr=0.1), EFSignSGDMemory(lr=0.1), hvd.size())
#compressor = NaturalCompressor()
#memory = ResidualMemory()
#grc = Allreduce(compressor, memory)

#grc = Allgather(NoneCompressor(), NoneMemory(), hvd.size())

#grc = Allgather(PowerSGDCompressor(), NoneMemory(), hvd.size())

#compressor = PowerSGDCompressor()
#memory = PowerSGDMemory(q_memory=compressor.q_memory, compress_rank=1)
#grc = Broadcast(compressor, memory, hvd.size())
#grc = Broadcast(PowerSGDCompressor(), memory, hvd.size())

#grc = Allgather(EFSignSGDCompressor(0.3), EFSignSGDMemory(lr), hvd.size())
#grc = Allgather(NaturalCompressor(), NoneMemory())
#grc = Allgather(QSGDCompressor(15), NoneMemory(), hvd.size())

# Horovod: wrap optimizer with DistributedOptimizer.
optimizer = hvd.DistributedOptimizer(optimizer, grc, named_parameters=model.named_parameters())

# Set up fixed fake data
data = torch.randn(args.batch_size, 3, 224, 224)
target = torch.LongTensor(args.batch_size).random_() % 1000
if args.cuda:
    data, target = data.cuda(), target.cuda()

def benchmark_step():
    torch.cuda.synchronize()
    forward_time = time.time()
    optimizer.zero_grad()
    output = model(data)
    loss = F.cross_entropy(output, target)
    loss.backward()
    optimizer.step()
    torch.cuda.synchronize()
    backward_time = time.time()
#    print("computation time: ", round(backward_time-forward_time, 3))
 #   print("communication time: ", round(time.time()-backward_time, 3))

def log(s, nl=True):
    if hvd.rank() != 0:
        return
    print(s, end='\n' if nl else '')

log('Model: %s' % args.model)
log('Batch size: %d' % args.batch_size)
device = 'GPU' if args.cuda else 'CPU'
log('Number of %ss: %d' % (device, hvd.size()))

# Warm-up
log('Running warmup...')
timeit.timeit(benchmark_step, number=args.num_warmup_batches)

# Benchmark
log('Running benchmark...')
img_secs = []
for x in range(args.num_iters):
    time1 = timeit.timeit(benchmark_step, number=args.num_batches_per_iter)
    img_sec = args.batch_size * args.num_batches_per_iter / time1
    log('Iter #%d: %.1f img/sec per %s' % (x, img_sec, device))
    img_secs.append(img_sec)

# Results
img_sec_mean = np.mean(img_secs)
img_sec_conf = 1.96 * np.std(img_secs)
log('Img/sec per %s: %.1f +-%.1f' % (device, img_sec_mean, img_sec_conf))
log('Total img/sec on %d %s(s): %.1f +-%.1f' %
    (hvd.size(), device, hvd.size() * img_sec_mean, hvd.size() * img_sec_conf))


from collections import defaultdict
from bisect import bisect_left, bisect_right

n, m = int(input()) + 1, 1000000007
q, p = [[] for i in range(n)], [[] for i in range(n)]
w = [0] + list(map(int, input().split()))
for i in range(int(input())):
    u, v = map(int, input().split())
    p[u].append(v)
    q[v].append(u)
r = set(i for i in range(1, n) if not p[i] or not q[i])
s, t = 0, 1
while r:
    i = r.pop()
    s += w[i]
    for j in p[i]:
        q[j].remove(i)
        if not q[j]: r.add(j)
    for j in q[i]:
        p[j].remove(i)
        if not p[j]: r.add(j)
r = set(i for i in range(1, n) if p[i] and q[i])
while r:
    i = r.pop()
    h = p[i]
    d, k = w[i], 1
    while h:
        i = h.pop()
        if not i in r: continue
        r.remove(i)
        h += p[i]
        if w[i] == d: k += 1
        elif w[i] < d: d, k = w[i], 1
    s += d
    t = (t * k) % m
print(s, t)