#!/usr/bin/python
# Solved by Bogdan Trif @   Completed on Mon, 14 Nov 2016, 23:32
#The  Euler Project  https://projecteuler.net
'''
                                            Roman numerals      -        Problem 89
For a number written in Roman numerals to be considered valid there are basic rules which must be followed.
Even though the rules allow some numbers to be expressed in more than one way there is always a "best" way of writing a particular number.

For example, it would appear that there are at least six ways of writing the number sixteen:

                                                                                IIIIIIIIIIIIIIII
                                                                                VIIIIIIIIIII
                                                                                VVIIIIII
                                                                                XIIIIII
                                                                                VVVI
                                                                                XVI

However, according to the rules only XIIIIII and XVI are valid, and the last example is considered to be the most efficient,
as it uses the least number of numerals.

The 11K text file, roman.txt (right click and 'Save Link/Target As...'), contains one thousand numbers written in valid,
but not necessarily minimal, Roman numerals; see About... Roman Numerals for the definitive rules for this problem.

Find the number of characters saved by writing each of these in their minimal form.

Note: You can assume that all the Roman numerals in the file contain no more than four consecutive identical units.
Traditional Roman numerals are made up of the following denominations:
                                                                        I = 1
                                                                        V = 5
                                                                        X = 10
                                                                        L = 50
                                                                        C = 100
                                                                        D = 500
                                                                        M = 1000
Only one I, X, and C can be used as the leading numeral in part of a subtractive pair.
I can only be placed before V and X.
X can only be placed before L and C.
C can only be placed before D and M.
'''

import time

print('\n--------------------------TESTS------------------------------')

filename = "pb089_roman.txt"
def load_file(filename):
    f = open(filename, 'r')
    text= f.read()
    f.close()
    rom_num = [i for i in text.split('\n')]
    return rom_num

init_rom = load_file(filename)
print(len(init_rom) , init_rom)

            # MMMMCMLXXXXV        ;  LXXXX = XC
            # MMMDCCLXXXVIIII         ;   VIIII = IX
            # DCCCCIIII                       ;   DCCCC = CM
            # DCCCCIIII                        ;     IIII = IV
            # MCCCCLXXIX                  ;   MCCCC = MCD
            # MMMDCCCXXXXV        ;       CXXXX = CXL

def replace_roman(rnum):
    ''':Description: Use the dictionary ROMAN to shorten the roman numerals
    :param rnum:  string,  Must be a roman numeral like : 'MMMMDCCCCXXXXVI'
    :return:    another string which is a shortened version of the numeral    '''
# rnum = 'MMMDCCLXXXVIIII'
    ROMAN = {1: ['MCCCC', 'MCD'], 2: ['DCCCC', 'CM'], 3: ['CCCC', 'CD'], 4: ['LXXXX', 'XC'],\
         5: ['CXXXX', 'CXL'], 6: ['XXXX', 'XL'], 7: ['VIIII', 'IX'], 8: ['IIII', 'IV']}
    for j in ROMAN :
        # print(ROMAN[j][0], ROMAN[j][1] )
        u = rnum.find(ROMAN[j][0])
        if u != -1  :
            s = rnum
            rnum = rnum.replace(ROMAN[j][0], ROMAN[j][1] )
            # print( str(iter) + '.  ' ,  ROMAN[j][0], ROMAN[j][1]    ,u,'   ' , s ,'    ', rnum )
    return rnum

print( '\nTest replace_rom function : ',replace_roman('MMMDCCLXXXVIIII') )
print( 'Test replace_rom function : ',replace_roman('XXXXVIIII') )
print( 'Test replace_rom function : ',replace_roman('MMMMDCCCCXXXXVI') )
print( 'Test replace_rom function : ',replace_roman('MMMDCCCCXXXVIIII') )
print( 'Test replace_rom function : ',replace_roman('MMMCCCXXXXV') )
print( 'Test replace_rom function : ',replace_roman('CLXXXXIX') )
print( 'Test replace_rom function : ',replace_roman('MCCCCXXIIII') )
print( 'Test replace_rom function : ',replace_roman('DLXXXX') )
print( 'Test replace_rom function : ',replace_roman('MMCCCCXXXXV') )
print( 'Test replace_rom function : ',replace_roman('MMMMCCCLXXXXVII') )
print( 'Test replace_rom function : ',replace_roman('MDCCCLXXXX') )
print( 'Test replace_rom function : ',replace_roman('MMCCCLXXXXIX') )
print( 'Test replace_rom function : ',replace_roman('MMMCCCXXVIIII') )

ROMAN = {1: ['MCCCC', 'MCD'], 2: ['DCCCC', 'CM'], 3: ['CCCC', 'CD'], 4: ['LXXXX', 'XC'], \
                    5: ['CXXXX', 'CXL'], 6: ['XXXX', 'XL'], 7: ['VIIII', 'IX'], 8: ['IIII', 'IV']}
# print(ROMAN,'\n')

t = 'MMCCCLXXXXIX'
print( '\nFind Test :  ' ,t.find( 'LXXXX') )
print( 'Find Test : ' ,t.find( ROMAN[1][1]) )
print( 'Find Test : ' ,t.find( ROMAN[1][0]) )
print('\nReplace Test : ',t.replace('LXXXX', 'XC'))
print('Replace Test : ',t.replace(ROMAN[1][0], ROMAN[1][1] ))

print('\n================  My FIRST SOLUTION,  VERY FAST ===============\n')
t1  = time.time()

def pb089():
    final_rom=[]
    iter=0
    for s in init_rom:
        t = replace_roman(s)
        iter+=1
        # print( str(iter) + '.  ' ,  s ,'      ',  t )
        final_rom.append(t)
    # print(final_rom)
    initial = ''.join(init_rom)
    final =  ''.join(final_rom)
    # print('\n', initial,'\n',final)
    res = len(initial) - len(final)
    return print('\nAnswer: ', res)

pb089()                     #   Answer:  743

t2  = time.time()
print('\nCompleted in :', round((t2-t1)*1000,6), 'ms\n\n')              # Completed in : 9.000778 ms

print('\n===============OTHER SOLUTIONS FROM THE EULER FORUM ==============')
print('\n--------------------------SOLUTION 1,  mbh038, England --------------------------')
t1  = time.time()

def PE_0089(filename):

    rnalts=[('IIII','IV'),('VIV','IX'),('XXXX','XL'),('LXL','XC'),('CCCC','CD')
            ,('DCD','CL'),('DDDD','CM')]

    with open(filename,'r') as file:
        data  = file.readlines()
    romans= [''.join([x for x in line.rstrip()]) for line in data]

    initialcount,finalcount=0,0

    for line in romans:
        initialcount+=len(line)
        for kv_pair in rnalts:
            line=line.split(kv_pair[0])
            if len(line)==2:
                line=line[0]+kv_pair[1]+line[1]
            else: line=line[0]
        finalcount+=len(line)
    print(initialcount-finalcount)

PE_0089(filename)

t2  = time.time()
print('\nCompleted in :', round((t2-t1)*1000,6), 'ms\n\n')

print('\n--------------------------SOLUTION 2,  aolea, Spain --------------------------')
t1  = time.time()

with open(filename) as f:
    roman89 =f.read().splitlines()
print(roman89)
count = 0
for j in roman89:
    if 'IIII' in j:
        count = count + 2
for j in roman89:
    if 'VIIII' in j:
        count = count + 1
for j in roman89:
    if 'XXXX' in j:
        count = count + 2
for j in roman89:
    if 'LXXXX' in j:
        count = count + 1
for j in roman89:
    if 'CCCC' in j:
        count = count + 2
for j in roman89:
    if 'DCCCC' in j:
        count = count + 1
print(count)

t2  = time.time()
print('\nCompleted in :', round((t2-t1)*1000,6), 'ms\n\n')

print('\n--------------------------SOLUTION 3, deadPix3l  --------------------------')
t1  = time.time()

#!/usr/bin/env python
total=0
with open(filename,'r') as f:
    for line in f:
        orig = line

        line = line.replace('IIIII','V')
        line = line.replace('IIII','IV')
        line = line.replace('VV','X')
        line = line.replace('VIV','IX')
        line = line.replace('XXXXX','L')
        line = line.replace('XXXX','XL')
        line = line.replace('LL','C')
        line = line.replace('LXL','XC')
        line = line.replace('CCCCC','D')
        line = line.replace('CCCC','CD')
        line = line.replace('DD','M')
        line = line.replace('DCD','CM')
        total+=len(orig)-len(line)

print (total)

t2  = time.time()
print('\nCompleted in :', round((t2-t1)*1000,6), 'ms\n\n')

print('\n--------------------------SOLUTION 4,  archeuclid   --------------------------')
t1  = time.time()

from re import search

def space_saved(rom):
    res = 0
    p = {4: 2, 5: 4, 6: 4, 7: 4, 8: 4, 9: 7}
    q = {'I': 'VIIII', 'X': 'LXXXX', 'C': 'DCCCC'}

    for l in ['I', 'X', 'C']:
        for n in range(9, 3, -1):
            if search('%s{%d}'%(l, n), rom) is not None:
                res += p[n]
                if n == 4:
                    if search(q[l], rom) is not None:
                        res += 1
                break

    return res

def p89():
    res = 0
    f = open(filename)
    for rom in f:
        res += space_saved(rom)
    return print(res)

p89()

t2  = time.time()
print('\nCompleted in :', round((t2-t1)*1000,6), 'ms\n\n')

print('\n--------------------------SOLUTION 5, bancaldo, Italy  --------------------------')
t1  = time.time()

def filter_romans(path):
    with open(path) as inputfile:
        romans = [num.strip() for num in inputfile]
    startchr = sum([len(r) for r in romans])

    for bad, correct in [("VIIII", "IX"), ("IIII", "IV"),
                         ("LXXXX", "XC"), ("XXXX", "XL"),
                         ("DCCCC", "CM"), ("CCCC", "CD"),]:
        for roman in romans:
            index = romans.index(roman)
            if bad in roman:
                romans[index] = roman.replace(bad, correct)
    return startchr - sum([len(r) for r in romans])

if __name__ == "__main__":
    start = time.time()
    result = filter_romans(filename)
    print ("euler 89: %s\nElapsed time %ss." % (result, time.time() - start))

t2  = time.time()
print('\nCompleted in :', round((t2-t1)*1000,6), 'ms\n\n')

print('\n--------------------------SOLUTION 6, charlie_mojocoa  --------------------------')
t1  = time.time()

def minimize_roman_numeral(roman):
  """Returns the minimized roman numeral from a valid roman numeral."""
  translations = [('DCCCC', 'CM'), ('CCCC', 'CD'), ('LXXXX', 'XC'),
                  ('XXXX', 'XL'), ('VIIII', 'IX'), ('IIII', 'IV')]

  for key, value in translations:
    roman = roman.replace(key, value)

  return roman

if __name__ == '__main__':
  diff = 0

  with open(filename, 'r') as f:
    for line in f:
      line = line.strip()
      min_roman = minimize_roman_numeral(line)
      diff += len(line) - len(minimize_roman_numeral(min_roman))

  print (diff)

t2  = time.time()
print('\nCompleted in :', round((t2-t1)*1000,6), 'ms\n\n')


import os
import fnmatch

ROOT_DIR = '/home/tse/CV/self_mtcnn/tar_tarbl'
SOURCE_TXT = '/home/tse/CV/self_mtcnn/train_box_landmark.txt'
FINAL_TXT = '/home/tse/CV/self_mtcnn/train_box_landmark_anno.txt'

'''
to add every ".jpg"'s dir into their ".json"s.(both bbox & landmark)
'''

def add_jpgpath_into_json(ROOT_DIR):
    for root, dirs, files in os.walk(ROOT_DIR):
        for file in files:
            file_name = os.path.join(root, file)
            if fnmatch.fnmatch(file_name, '*jpg'):
                json_file = file_name[:file_name.index('.')] + '.json'

                count = 0

                with open(json_file, 'r+') as j:
                    old = j.read()  # 读取
                    old = eval(old)  # 字符串转列表

                    #                     j.seek(0)# 找第一行
                    #                     j.truncate()# 开始删除

                    for idx, value in enumerate(old):
                        if len(old[idx]) == 4:
                            count += 1

                if count == 1:  # 只有一个bbox
                    with open(json_file, 'r+') as j1:
                        old1 = j1.read()  # 读取 字符串
                        j1.seek(0)
                        j1.truncate()

                        # new1 = old1.replace('], [', ' ').replace(', ', ' ').replace('[[', ' ').replace(']]', ' ')
                        j1.write(file_name)
                        j1.write(old1)

                if count == 2:  # 有两个bbox
                    with open(json_file, 'r+') as j2:
                        old2 = j2.read()
                        old2 = eval(old2)  # 字符串转列表
                        j2.seek(0)
                        j2.truncate()

                        count2 = 0
                        for idx2, value2 in enumerate(old2):
                            if len(old2[idx2]) == 4:
                                count2 += 1
                            if count2 == 2:
                                list1 = old2[0:idx2]
                                str1 = ' '.join('%s' % id for id in list1)
                                # str1 = str1.replace('] [', ' ').replace('[', ' ').replace(']', ' ').replace(', ', ' ')
                                j2.write(file_name)
                                j2.write(str1)
                                j2.write('\n')

                                list2 = old2[idx2:]
                                str2 = ' '.join('%s' % id for id in list2)
                                # str2 = str2.replace('] [', ' ').replace('[', ' ').replace(']', ' ').replace(', ', ' ')
                                j2.write(file_name)
                                j2.write(str2)

                                break

'''
merge all '.json's to a '.txt'
'''
def merge_json_2_txt(ROOT_DIR):
    for root, dirs, files in os.walk(ROOT_DIR):

        for file in files:
            file_name = os.path.join(root, file)
            print(file_name)

            if fnmatch.fnmatch(file_name, '*json'):
                '''
                make sure to open file with 'a' mode,otherwise the old content will be delete after add new content,
                finally,you only can have a last record.
                '''
                with open(file_name, 'r') as f:
                    record = f.read()
                    record = record.replace('], [',' ').replace('] [',' ').replace('[[',' ').replace(', ',' ').replace('[',' ').replace(']',' ')
                with open(SOURCE_TXT, 'a') as t:
                    t.write(record)
                    t.write('\n')
                '''
                or
                '''
                with open(file_name,'r') as f:
                    for line in f:
                        with open(FINAL_TXT,'a') as t:
                            t.write(line)
                            t.write('\n')

def clearBlankLine():
    file1 = open(SOURCE_TXT, 'r', encoding='utf-8')  # source '.txt'
    file2 = open(FINAL_TXT, 'w', encoding='utf-8')  # '.txt' without blank line
    try:
        for line in file1.readlines():
            if line == '\n':
                line = line.strip("\n")
            file2.write(line)
    finally:
        file1.close()
        file2.close()
if __name__ == '__main__':

    add_jpgpath_into_json(ROOT_DIR)
    merge_json_2_txt(ROOT_DIR)
    clearBlankLine()


#!/usr/bin/python3

"""[SUBREDDIT]
"""

def number_of_subscribers(subreddit):
    import requests

    url = 'https://www.reddit.com/r/{}/about.json'.format(subreddit)
    req = requests.get(url,
                       headers={'User-Agent': 'My User Agent 1.0'},
                       allow_redirects=False)

    if req.status_code != 200:
        return 0
    return req.json()['data']['subscribers']


"""ApiTest URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/3.1/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.conf.urls import url
from django.contrib import admin
from django.urls import path
from MyApp.views import *

urlpatterns = [
    url(r'^admin/', admin.site.urls),
    url(r'^welcome/$', welcome),#获取菜单
    url(r'^case_list/$', case_list),#用例表
    url(r'^home/$', home),#进入首页
    url(r'^child/(?P<eid>.+)/(?P<ord>.*)',child),#返回子页面
    url(r'^login/$', login),#进入登录页
    url(r'^login_action/$',login_action),#登录
    url(r'^register_action/$',register_action),#注册
    url(r'^accounts/login/$', login),#非登录页面跳转登录
    url(r'^logout/$', logout),#退出
    url(r'^pei/$', pei),#吐槽
    url(r'^help/$',api_help),#进入帮助文档
]


# multivariate multi-step encoder-decoder lstm
from math import sqrt
from numpy import split
from numpy import array
from pandas import read_csv
import matplotlib.pyplot as plt
from sklearn.metrics import mean_squared_error
from matplotlib import pyplot
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Flatten
from keras.layers import LSTM
from keras.layers import RepeatVector
from keras.layers import TimeDistributed
import pandas as pd

import numpy as np
np.random.seed(1234)

def split_dataset(data):
    # split into standard hours of day
    train,val,test = data[1:8713], data[8713:8737],data[8737:]
    print(train.shape)
    print(test.shape)
    print('==============================')
    # restructure into windows of weekly data
    train = array(split(train, len(train) / 24))
    val = array(split(val, len(val) / 24))
    test = array(split(test, len(test) / 24))
    print(train)
    print(test)
    print(val)
    print('==============================')
    print(train.shape)
    print(val.shape)
    print(test.shape)
    return train, val,test

# evaluate one or more weekly forecasts against expected values
def evaluate_forecasts(actual, predicted):
    scores = list()
    # calculate an RMSE score for each day
    for i in range(actual.shape[1]):
        # calculate mse
        mse = mean_squared_error(actual[:, i], predicted[:, i])
        # calculate rmse
        rmse = sqrt(mse)
        # store
        scores.append(rmse)
    # calculate overall RMSE
    s = 0
    for row in range(actual.shape[0]):
        for col in range(actual.shape[1]):
            s += (actual[row, col] - predicted[row, col]) ** 2
    score = sqrt(s / (actual.shape[0] * actual.shape[1]))
    return score, scores

# summarize scores
def summarize_scores(name, score, scores):
    s_scores = ', '.join(['%.1f' % s for s in scores])
    print('%s: [%.3f] %s' % (name, score, s_scores))

# convert history into inputs and outputs
def to_supervised(train, n_input, n_out=24):
    # flatten data
    data = train.reshape((train.shape[0] * train.shape[1], train.shape[2]))
    print('data is',data)
    X, y = list(), list()
    in_start = 0
    # step over the entire history one time step at a time
    for _ in range(len(data)):
        # define the end of the input sequence
        in_end = in_start + n_input
        out_end = in_end + n_out
        # ensure we have enough data for this instance
        if out_end <= len(data):
            X.append(data[in_start:in_end, :])
            y.append(data[in_end:out_end, 0])
        # move along one time step
        in_start += 1
    print('==============================')
    print(array(X))
    print('==============================')
    print(array(y))
    print('input,Output',array(X), array(y))
    return array(X), array(y)

# train the model
def build_model(train, n_input):
    # prepare data
    train_x, train_y = to_supervised(train, n_input)
    print("train x data",train_x)
    print("train y data", train_y)
    # define parameters
    verbose, epochs, batch_size = 2, 70, 16
    n_timesteps, n_features, n_outputs = train_x.shape[1], train_x.shape[2], train_y.shape[1]
    # reshape output into [samples, timesteps, features]
    train_y = train_y.reshape((train_y.shape[0], train_y.shape[1], 1))
    # define model
    model = Sequential()
    model.add(LSTM(200, activation='relu', input_shape=(n_timesteps, n_features)))
    model.add(RepeatVector(n_outputs))
    model.add(LSTM(200, activation='relu', return_sequences=True))
    model.add(TimeDistributed(Dense(100, activation='relu')))
    model.add(TimeDistributed(Dense(1)))
    model.compile(loss='mse', optimizer='adam')
    # fit network

    model.fit(train_x, train_y, epochs=epochs, batch_size=batch_size, verbose=verbose)
    return model

# make a forecast
def forecast(model, history, n_input,avg_power)  :
    # flatten data
    data = array(history)
    print('History data is /%n',data)
    data = data.reshape((data.shape[0] * data.shape[1], data.shape[2]))
    # retrieve last observations for input data
    input_x = data[-n_input:, ]
    print('input data',input_x)
    # reshape into [1, n_input, n]
    input_x = input_x.reshape((1, input_x.shape[0], input_x.shape[1]))
    # forecast the next week
    yhat = model.predict(input_x, verbose=2)
    # we only want the vector forecast
    yhat = yhat[0]
    yhat = [x*avg_power for x in yhat]
    fore1 = pd.DataFrame(yhat)
    fore1.to_csv('forecasted.csv')

    print('model prediction is',fore1)
    test_data = test[:, :, 0]
    test_data = [x*avg_power for x in test_data]
    test_data = pd.DataFrame(test_data)
    fore2= test_data.T
    print(fore2)
    fore2.to_csv('actual.csv')

    aa = [x for x in range(24)]
    fig = plt.figure()
    plt.plot(aa, fore2[:24], marker='.', label="actual")
    plt.plot(aa, fore1[:24], 'r', marker='.', label="prediction")
    plt.ylabel('Load', size=15)
    plt.xlabel('Time step', size=15)
    plt.legend(fontsize=15)
    plt.show()
    fig.savefig('prediction with (t).png')
    return yhat

# evaluate a single model
def evaluate_model(train, test, n_input,val,avg_power):
    # fit model
    model = build_model(train, n_input)
    # history is a list of weekly data
    history = [x for x in val]
    print("History data is",history)
    # walk-forward validation over each hour
    predictions = list()
    for i in range(len(test)):
        # predict the load for 24 hour
        yhat_sequence = forecast(model, history, n_input,avg_power)
        print(yhat_sequence)
        # store the predictions
        predictions.append(yhat_sequence)
        # get real observation and add to history for predicting the next 24 hour
        history.append(test[i, :])
    # evaluate predictions days for each week
    predictions = array(predictions)
    score, scores = evaluate_forecasts(test[:, :, 0], predictions)
    test_data= test[:, :, 0]
    print(test_data)
    return score, scores

# load the new file
dataset = pd.read_csv('2007(1).csv',sep=',', low_memory=False,na_values=['nan','?'],
                 parse_dates={'dt' : ['Date', 'TIME']}, infer_datetime_format=True, index_col='dt')
print(dataset)
avg=dataset.mean(axis = 0, skipna = True)
print(avg)
print(avg[0])
dataset= (dataset.div(dataset.mean(axis=0), axis=1))
print(dataset)

# split into train and test
train, val, test = split_dataset(dataset.values)
# evaluate model and get scores
n_input = 24
score, scores = evaluate_model(train, test, n_input,val,avg[0])

# summarize scores
summarize_scores('lstm', score, scores)
# plot scores


with open('10.txt', 'r') as f:
	for line in f.readlines():
		split = line.split(" ")
		x1 = int(split[0])
		y1 = int(split[1])
		x2 = int(split[2])
		y2 = int(split[3])
		a = (y2 - y1) / (x2 - x1)
		b = (y1 - (x1*a))
		print('('+str(round(a))+' '+str(round(b))+')')

'''
I solve for a by finding the change in y over the change in x.
In order to solve for b, I plug in one of the provided points fitting in the formula, subtracting the x coordinate times a from the y coordinate to find what b would have to be for the formula to intersect that point.
'''


def babygin(player):
    cnt = 0
    lis=[0]*12
    for i in player:
        cnt += 1
        lis[i] +=1
        if lis[i] == 3:
            cnt -= 1
            break
        if i >= 2 and( 0 not in lis[i-2:i+1] or 0 not in lis[i-1:i+2] or 0 not in lis[i:i+3]):
            cnt -= 1
            break
        elif i == 1 and( 0 not in lis[i-1:i+2] or 0 not in lis[i:i+3]):
            cnt -= 1
            break
        elif i == 0 and 0 not in lis[i:i+3]:
            cnt -= 1
            break
    return cnt
T = int(input())
for t in range(1,T+1):
    card = list(map(int,input().split()))
    player_1, player_2 = [], []
    for i in range(len(card)):
        player_1.append(card[i]) if i % 2 == 0 else player_2.append(card[i])
    ans1 = babygin(player_1)
    ans2 = babygin(player_2)
    if ans1 > ans2:
        print("#{} {}".format(t,2))
    elif ans1 == ans2 == 6:
        print("#{} {}".format(t,0))
    else:
        print("#{} {}".format(t,1))


import importlib.util
import os

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

spec = importlib.util.spec_from_file_location("pc", BASE_DIR + "/cluster-engine/process.py")
pc = importlib.util.module_from_spec(spec)
spec.loader.exec_module(pc)

spec2 = importlib.util.spec_from_file_location("catalog", BASE_DIR + "/intelligent-catalog-service/catalog.py")
catalog = importlib.util.module_from_spec(spec2)
spec2.loader.exec_module(catalog)

settings = {
    'check': {'interval': 0, 'checkcode': 0},
    'input_type': '',
    'input_settings': '',
    'index_features': 'NAME',
    'ui_features': '',
    'output_type': '',
    'output_settings': '',
    "pre_processed_file_path": BASE_DIR + "/cluster-engine/test/",
    "pre_processed_file_name": "sample_test.json",
    "pre_processed_file_type": 'json',
    "processed_file_path": BASE_DIR + "/cluster-engine/test/",
    "processed_file_name": "sample_test_clusters.json",
    "processed_file_type": 'json',
    "processed_yet_file_path": BASE_DIR + "/cluster-engine/data/processed/",
    "web_data_file_path": BASE_DIR + "/intelligent-catalog-web/app/data/",
    "web_data_file_name": "dotz_model.json",
    "web_data_file_type": "json",
    "items_updated_file_path": BASE_DIR + "/intelligent-catalog-web/app/data/",
    "items_updated_file_name": "items_updated.json",
    "items_updated_file_type": "json",
}

catalog = catalog.Catalog(settings)

result = pc.process_catalog(catalog)

print(result)


#!/usr/bin/python
import os

from setuptools import setup, find_packages

SRC_DIR = os.path.dirname(__file__)
CHANGES_FILE = os.path.join(SRC_DIR, "CHANGES")

with open(CHANGES_FILE) as fil:
    version = fil.readline().split()[0]

setup(
    url="https://github.com/gurunars/dict-validator",
    name="dict-validator",
    install_requires=[],
    description="A library for structural data validation.",
    version=version,
    packages=find_packages(exclude=["test"]),
    author="Anton Berezin",
    author_email="gurunars@gmail.com",
    include_package_data=True
)


# -*- coding:utf-8 -*-

from evb5 import *
from ch341a_dll import *
from gbert import *
import os
import sys
from ctypes import *
import time
# sys.path.append(os.path.join(os.path.split(os.path.dirname(os.path.abspath(__file__)))[0], 'dll'))
sys.path.append(os.path.join(os.path.split(os.path.dirname(os.path.abspath(__file__)))[0], 'config'))
from common import *
# os.putenv('path', os.environ['path'] + ';' + DLL_PATH)

class INIT_BERT(object):
    def __init__(self, name, sn, rate, prbs):
        self.name = name
        self.sn = sn.encode("utf-8")

        if name == 'SBERT' or name == 'SEVB5':
            self.rate = 0 if rate<1.5 else 1
        elif name == 'GBERT':
            self.rate = 2 if rate<1.5 else 9
        # self.prbs = 0 if prbs == 7 else 1
        self.prbs = prbs
        self.bert = None

    def bert_init(self):
        if self.name == 'SBERT' or self.name == 'SEVB5':
            self.bert = SEVB5(0x800, DLL_PATH+'\\EVB5.dll')
            if None == self.bert.EVB5_Init(self.sn):
                return None
            print(self.bert.USBHandle)
            ch341 = CH341A_DLL(self.bert.USBHandle, DLL_PATH + '\\CH341A.dll')
            # 如果有sbert，则初始化sbert
            if ch341.i2c_slave_search_dll(0x80) == 0:
                if self.bert.evb5_sbert_init(self.rate, self.prbs):
                    return None
                if self.bert.evb5_set_shdn_vtr(1):
                    return None
            else:
                return None

            if self.bert.evb5_Set_BEN_Mode(0):
                return None
            if self.bert.evb5_Set_BEN_Level(0):
                return None

        elif self.name == 'GBERT':
            self.bert = GBERT(DLL_PATH+'\\GBERT.dll')
            if not self.bert.gbert_init_sfptxdisable(self.sn, 0, self.prbs, self.rate):
                return None
        else:
            return None
        return True

    def bert_start(self, testtime):
        p_statuscode = pointer(c_int())
        p_string = create_string_buffer(100)
        p_errnumber = pointer(c_double())
        p_errratio = pointer(c_double())
        p_elapsedtime = pointer(c_double())
        if self.name == 'SBERT' or self.name == 'SEVB5':
            if self.bert.evb5_sbert_start(self.rate, testtime):
                return None
            if self.bert.evb5_sbert_read(self.rate, self.prbs, p_statuscode, p_string, p_errnumber, p_errratio, p_elapsedtime):
                return None
            if p_statuscode.contents.value == -2:
                return None

        elif self.name == 'GBERT':
            if 0 == self.bert.gbert_start(testtime):
                return None

        else:
            return None
        return True

    def bert_read(self, testtime):
        time.sleep(testtime)
        p_statuscode = pointer(c_int())
        p_string = create_string_buffer(100)
        p_errnumber = pointer(c_double())
        p_errratio = pointer(c_double())
        p_elapsedtime = pointer(c_double())
        if self.name == 'SBERT' or self.name == 'SEVB5':
            for i in range(3):
                if self.bert.evb5_sbert_read(self.rate, self.prbs, p_statuscode, p_string, p_errnumber, p_errratio,
                                             p_elapsedtime):
                    return None
                if p_statuscode.contents.value in [-2, -1, 0, 1]:
                    break
            if p_statuscode.contents.value < -2:
                return None

        elif self.name == 'GBERT':
            if 0 == self.bert.gbert_check_ratio(p_errratio, p_statuscode):
                return None

        else:
            return None
        return (p_statuscode.contents.value, p_errratio.contents.value)

    def bert_stop(self):
        if self.name == 'SBERT' or self.name == 'SEVB5':
            if self.bert.evb5_sbert_end():
                return None

        elif self.name == 'GBERT':
            if 0 == self.bert.gbert_stop():
                return None

        else:
            return None
        return True

if __name__ == '__main__':
    bert = INIT_BERT('GBERT', 'GBRT022020120506', 1.25, 7)
    print(bert.bert_init())


"""
create a class for family members
1.add a method for family members are the in family class
2. Remove one family member with ID
"""

class Family:

    def __init__(self):
        self.family = []

    def add_member_family(self, name, age, mid):
        d = {"name": name, 'age': age, "id": mid}
        self.family.append(d)
        return True

    def no_of_families(self):
        return len(self.family)

    def remove_one_family_by_id(self, mid):
        self.family.pop(1)
        return mid

x = Family()

x.add_member_family("charan", 23, 12345)
x.add_member_family("ravi", 25, 123456)
x.add_member_family("kumar", 27, 123457)
x.add_member_family("nagesh", 30, 1234556)
x.add_member_family("suneetha", 23, 12345568)
print("The familes are:", x.family)

y = x.no_of_families()
print("The total no.of families are:", y)

z = x.remove_one_family_by_id(12345)
print("The deleted is is:",z)


# -*- coding: utf-8 -*-
# Copyright ©2017 Shah Rustam.

from app.main import *

class EmployeeEdit(MethodView):
    def get(self, id):
        employee = Employee.get_by_id(int(id))
        if employee:
            return render_template('employeeEdit.html', title='Edit employee', page='employees', employee=employee)
        abort(404)

    def post(self, id):
        employee = Employee.get_by_id(int(id))
        if employee:
            firstName = request.form['firstName']
            if firstName == "":
                return ErrorResponse.makeEmpty('firstName')
            secondName = request.form['secondName']
            if secondName == "":
                return ErrorResponse.makeEmpty('secondName')
            email = request.form['email']
            if email == "":
                return ErrorResponse.makeEmpty('email')
            if not validate_email(email):
                return ErrorResponse.makeInvalidFormat('email')
            phone = request.form['phone']
            if phone == "":
                return ErrorResponse.makeEmpty('phone')
            if not phoneCheck(phone):
                return ErrorResponse.makeInvalidFormat('phone')
            birthDate = request.form['birthDate']
            if birthDate == "":
                return ErrorResponse.makeEmpty('birthDate')
            if not dateCheck(birthDate):
                return ErrorResponse.makeInvalidFormat('birthDate')
            birthDate = datetime.strptime(birthDate, "%d/%m/%Y")
            employee.firstName = firstName
            employee.secondName = secondName
            employee.email = email
            employee.phone = phone
            employee.birthDate = birthDate
            employee.put()
            department = Department.get_by_id(employee.department)
            return jsonify({'status': 1, 'data': {'department':department.alias}})
        abort(404)

def phoneCheck(phone):
    check = re.match('^\+380[0-9]{9}$', phone)
    if check:
        if check.group(0) == phone:
            return True
    return False


from .test_data import *
import time
from logging import debug
from .initiate_driver import *
from .function_lib import *

def test_book_flight(test_setup_teardown):

    driver.find_element_by_id("gosuggest_inputSrc").send_keys(get_data_from_excel(EXCEL_PATH, 1, 2))
    time.sleep(1)
    source_elems = driver.find_elements_by_xpath('//ul[contains(@id, "react-autosuggest")]//li')
    try:
        for elem in source_elems:
            sr_data = str(elem.text).split("\n")[0]
            if sr_data == get_data_from_excel(EXCEL_PATH, 2, 2):
                elem.click()
                break
            else:
                continue
        time.sleep(5)
        driver.find_element_by_id("gosuggest_inputDest").send_keys(get_data_from_excel(EXCEL_PATH, 4, 2))
        dest_elems = driver.find_elements_by_xpath('//ul[contains(@id, "react-autosuggest")]//li')
        for elem in dest_elems:
            ds_data = str(elem.text).split("\n")[0]
            if ds_data == get_data_from_excel(EXCEL_PATH, 5, 2):
                elem.click()
                break
            else:
                continue
    except:
        print("There is some exception")


import logging
import sys
import logging.handlers

from logext.formatter import ColoredFormatter

def logging_hello_world():

    logger = logging.Logger("test")
    handler = logging.StreamHandler(stream=sys.stdout)
    formatter = ColoredFormatter()
    handler.setFormatter(fmt=formatter)
    logger.setLevel(logging.DEBUG)
    logger.addHandler(handler)
    logger.debug("hello world")

    return True

def test_logging_hello_world():
    assert logging_hello_world() is True


# Definition for singly-linked list.
# class ListNode:
#     def __init__(self, x):
#         self.val = x
#         self.next = None

class Solution:
    def addTwoNumbers(self, l1, l2):
        a = self.makeToNum(l1)
        b = self.makeToNum(l2)

        return self.makeList(a+b)

    def makeToNum(self, li):
        st = str(li.val)
        while(li.next != None):
            st = str(li.next.val) + st
            li = li.next
        return int(st)

    def makeList(self, num):
        st = str(num)
        node1 = ListNode(st[0])
        for i in range(1, len(st)):
            l = ListNode(st[i])
            l.next = node1
            node1 = l

        return node1


__author__ = 'yinyanhe'
"""
Given an unsorted array, find the maximum difference between the successive elements in its sorted form.

Try to solve it in linear time/space.

Return 0 if the array contains less than 2 elements.

You may assume all elements in the array are non-negative integers and fit in the 32-bit signed integer range.
"""
import math
def  maximumGap(num):
    """
    :param num: List[int]
    :return:int
    """
    if len(num)<2: return 0
    minV=maxV=num[0]
    for a in num:
        minV=min(minV, a)
        maxV=max(maxV, a)
    gap=int(math.ceil(float(maxV-minV)/(len(num)-1)))
    bucketNum=int(math.ceil(float(maxV-minV)/gap))
    bucketMax=[0]*bucketNum
    bucketMin=[2**31-1]*bucketNum
    for i in num:
        if i==minV or i==maxV:
            continue
        idx=(i-minV)/gap
        bucketMax[idx]=max(bucketMax[idx], i)
        bucketMin[idx]=min(bucketMin[idx], i)
    maxGap=0
    previous=minV
    for i in range(bucketNum):
        if bucketMin[i]==2**31-1 and bucketMax[i]==0:
            continue
        maxGap=max(maxGap, bucketMin[i]-previous)
        previous=bucketMax[i]
    maxGap=max(maxGap, maxV-previous)
    return maxGap


import oauth2
import urllib.parse
import json

# Criando Objeto do tipo Twitter
class Twitter:

    def __init__(self, consumer_key, consumer_secret, token_key, token_secret):
        self.conexao(consumer_key, consumer_secret, token_key, token_secret)

    def conexao(self, consumer_key, consumer_secret, token_key, token_secret):

        # Atributos do objeto
        self.consumer = oauth2.Consumer(consumer_key, consumer_secret)
        self.token = oauth2.Token(token_key, token_secret)
        self.cliente = oauth2.Client(self.consumer, self.token)

    def tweet(self, novo_tweet):
        twittar_codificada = urllib.parse.quote(novo_tweet, safe='')
        requisicao_twittar = self.cliente.request(
            "https://api.twitter.com/1.1/statuses/update.json?status=" + twittar_codificada, method='POST')
        twittar_decodificar = requisicao_twittar[1].decode()
        twittar_objeto = json.loads(twittar_decodificar)


"""
For characterization of one single network calculate:

    1. Degree distribution
        -> plot, fit Powerlaw
    2. Centralities
    3. Spectral Clustering
    4. HQCut-CLustering
    5. GO-Enrichment of Clusters

=> before that:
    For each Pair of Proteomes, generate mutual Proteins and fit power law degree distribution
    for soft thresholding to Core => Comparability.

Interpret, compare

Use COG-mappings to characterize clusters as pathogenicity specific, apathogenicity specific
Map and characterize VFs

Theory
======

1.  High number of links inside a community corresponds to functional relationship of this community
        => plot this density
2.  Ideal result (how does this relate to the possible derivation of links between species?) would be
    3 types of clusters:
        1. Clusters that contain mostly specific proteins
            -> Question: what's the role of the other proteins?
        2. Clusters that are equally mixed
        3. Clusters that are mostly shared:
            -> Same Question as in 1

    => Calculate a Score for the mixedness of specific and non-specific Proteins
    => Relate that to GO-annotations
"""
from matplotlib import pyplot as plt

import pandas as pd
import numpy as np
import os
import warnings

import networkx as nx
import time

from pandas.io.pytables import HDFStore
from oct2py import octave
from oct2py.utils import Oct2PyError

from scipy.sparse import csr_matrix

from src import lib

from collections import ChainMap, defaultdict
from functools import partial

from goatools.go_enrichment import GOEnrichmentStudy
from goatools.obo_parser import GODag
from goatools.associations import read_associations

octave.addpath('~/tools/octave/Qcut')
STORE = 'data/proteomes_.h5'

def get_string_id(locus_tag, organism):
    if type(locus_tag) == float:
        return float('nan')
    return '.'.join([organism, locus_tag])

def read_action_codes():
    with open('data/action_codes.tsv', 'r') as f:
        action_codes = {}
        action_codes_reverse = {}
        for line in f:
            interaction, code = line.strip().split('\t')
            code = int(code)
            action_codes[interaction] = code
            action_codes_reverse[code] = interaction

    return ChainMap(action_codes, action_codes_reverse)

def get_action_codes(interactions):
    interaction_types = list(set(interactions))
    action_codes = dict(zip(interaction_types, range(1, len(interactions), 3)))
    action_codes_reverse = dict(zip(range(1, len(interactions), 3), interaction_types))
    return ChainMap(action_codes, action_codes_reverse)

def store_action_codes(action_codes):
    with open('data/action_codes.tsv', 'w') as f:
        for _map in action_codes.maps:
            if type(list(_map.keys())[0]) == str:
                for key, code in _map.items():
                    f.write('\t'.join([key, str(code)]))
                    f.write('\n')

def add_action_code(action_codes, action):
    _max = max(filter(lambda x: type(x) == int, action_codes.values()))
    code = _max + 3
    action_codes[code] = action
    action_codes[action] = code

def read_actions(filename, threshold):
    data = pd.read_csv(filename, sep='\t', compression='gzip')
    try:
        action_codes = read_action_codes()
    except FileNotFoundError:
        action_codes = get_action_codes(data.loc[:, 'mode'])
        store_action_codes(action_codes)

    ids = data.loc[:, 'item_id_a'].unique().tolist()
    size = len(ids)

    raw_interactions = pd.DataFrame(np.zeros((size, size)), dtype=float, index=ids)
    interaction_types = pd.DataFrame(np.zeros((size, size)), dtype=int, index=ids)

    for line in data.iterrows():
        a, b, _type, mode, acting, score = line[1]
        i_a = ids.index(a)
        i_b = ids.index(b)

        if score < threshold:
            continue

        raw_interactions.values[i_a, i_b] = 1
        try:
            code = action_codes[_type]
        except KeyError:
            add_action_code(action_codes, _type)
            store_action_codes(action_codes)
            code = action_codes[_type]

        if mode == 'activation':
            code += 1
        elif mode == 'inhibition':
            code -= 1

        interaction_types.values[i_a, i_b] = code

    return raw_interactions, interaction_types

def get_proto_clustering(proteome, clusters):
    """
    re-format results as returned by HQ-Cut into format that can be used as Input for
    Cluster-Objects
    :param clusters:
    :return:
    """
    p_clustering = pd.Series(np.zeros(proteome.interactions.shape[0]),
                             index=proteome.interactions.index, dtype=int)

    for i, cluster in enumerate(clusters):
        ind = (cluster - 1).astype(int)
        p_clustering.iloc[ind] = i

    return p_clustering

def HQCut(net):
    """
    :param net:
    :return:
    """
    net = csr_matrix(net.values)
    t1 = time.time()
    clusters = octave.QcutPlus(net)
    t2 = time.time()
    print('QCut clustering done. Took {} minutes'.format((t2 - t1) / 60.0))

    t1 = time.time()
    h_clusters = octave.HQcut(net, clusters)
    t2 = time.time()

    print('HQCut clustering done. Took {} minutes'.format((t2 - t1) / 60.0))

    q = octave.Q(h_clusters, net)
    return clusters, h_clusters, q

class Proteome(object):
    """
    Interaction types are encoded by Integers. Build one global Index of Interaction types
    """

    def __init__(self, _id, label, threshold):
        self._id = _id
        self.label = label
        self.interactions = None
        self.interaction_types = None
        self.proteins = None
        self.cogs = None
        self.patric_annotation = None
        self.virulence_proteins = None
        self.load_data(threshold)
        self.load_patric_annotation()
        self.set_maping_index()
        self.set_virulence_proteins()
        self.cog_index = self.load_cog_index()

    def normalize(self, factor):
        self.interactions = self.interactions.divide(factor)

    def load_data(self, threshold, key=None):

        print('Loading data for Proteome {}; keys = {}'.format(self._id, key))
        loaders = []
        with HDFStore(STORE) as store:
            print('load from hdf5')
            if key is not None:
                try:
                    setattr(self, key, store['{}/{}'.format(self._id, key)])
                except KeyError:
                    if key in ['interactions', 'interaction_types', 'proteins']:
                        loaders.append('read_actions')
                    elif key == 'cogs':
                        loaders.append('load_cogs')
                    else:
                        raise KeyError

            else:
                try:
                    self.interactions = store['{}/interactions'.format(self._id)]
                    self.interaction_types = store['{}/interaction_types'.format(self._id)]
                    self.proteins = store['{}/proteins'.format(self._id)]
                except KeyError:
                    loaders.append('read_actions')
                try:
                    self.cogs = store['{}/cogs'.format(self._id)]
                except KeyError:
                    loaders.append('load_cogs')

            if 'read_actions' in loaders:
                print('load from tsv')
                interactions, types = read_actions('data/{}.protein.actions.v10.txt.gz'.format(self._id),
                                                   threshold=threshold)
                store.put('{}/interactions'.format(self._id), interactions)
                store.put('{}/interaction_types'.format(self._id), types)
                proteins = interactions.index.to_series()
                store.put('{}/proteins'.format(self._id), proteins )
                self.interactions = interactions
                self.interaction_types = types
                self.proteins = proteins
            if 'load_cogs' in loaders:
                print('load from tsv')
                self.load_cogs()
                store.put('{}/cogs'.format(self._id), self.cogs)

    def load_cogs(self):
        if self.proteins is None:
            self.load_data('proteins')

        with open('data/COG.mappings.v10.txt', 'r') as f:
            next(f)
            cogs = []
            for line in f:
                line = line.strip().split()
                p_id = line[0]
                if p_id not in self.proteins:
                    continue
                cog = line[3]
                annot = line[4]
                values = {'COG': cog, 'protein': p_id, 'annotation': annot}
                cogs.append(values)

        self.cogs = pd.DataFrame(cogs)

    def go_assoc_filename(self):
        return 'data/go_annotations/{}_go_associations.tsv'.format(self._id)

    def load_patric_annotation(self):
        dir_ = 'data/patric'
        filename = [os.path.join(dir_, x) for x in os.listdir(dir_) if self._id in x and 'spgene' in x][0]
        try:
            self.patric_annotation = pd.read_csv(filename, sep='\t')
        except:
            warnings.warn('No PATRIC annotations found')
            self.patric_annotation = pd.DataFrame()

    def set_maping_index(self):
        get_string_id_ = partial(get_string_id, organism=self._id)

        patric_1 = self.patric_annotation.loc[self.patric_annotation.loc[:, 'refseq_locus_tag'].dropna().index, :]
        patric_1.index = pd.Index(patric_1.loc[:, ('refseq_locus_tag')].apply(get_string_id_))

        self.patric_annotation = patric_1

    def set_virulence_proteins(self):
        roles = self.patric_annotation[self.patric_annotation.index.isin(self.proteins)]\
                    .loc[:, 'classification'].dropna()

        self.virulence_proteins = roles.loc[roles.str.match('.*[vV]irulence.*')].index

    def proto_clustering(self, protein_subset):
        return pd.Series(self.interactions.index.isin(protein_subset).astype(int),
                         index=self.interactions.index)

    def unique_cogs(self):
        c = self.cog_index.value_counts()
        return pd.Series(c[np.where(c == 1)[0]].index)

    def unique_cogs_percentage(self):
        c = self.cog_index.value_counts()
        return c[np.where(c > 1)[0]].sum() / len(self.proteins)

    def load_cog_index(self):
        prot_to_cog = self.cogs.loc[:, ('COG', 'protein')].set_index('protein')['COG'].to_dict()
        s = pd.Series(self.interactions.index).map(prot_to_cog)
        s.index = self.interactions.index
        return s

    def add_betweenness(self, centralities, key=None):
        with HDFStore(STORE) as store:
            if key is None:
                store.put('{}/centralities/betweenness/raw'.format(self._id), centralities)
            else:
                store.put('{}/centralities/betweenness/{}'.format(self._id, key), centralities)

    def get_betweenness(self, key=None):
        try:
            with HDFStore(STORE) as store:
                if key is None:
                    betweenness = store['{}/centralities/betweenness/raw'.format(self._id)]
                else:
                    betweenness = store['{}/centralities/betweenness/{}'.format(self._id, key)]
        except KeyError:
            betweenness = betweenness_centrality(self.interactions)
            self.add_betweenness(betweenness, key)

        return betweenness.loc[betweenness.index[betweenness.index.isin(self.interactions.index)]]

    def add_closeness(self, centralities, key=None):
        with HDFStore(STORE) as store:
            if key is None:
                store.put('{}/centralities/closeness/raw'.format(self._id), centralities)
            else:
                store.put('{}/centralities/closeness/{}'.format(self._id, key), centralities)

    def get_closeness(self, key=None):
        try:
            with HDFStore(STORE) as store:
                if key is None:
                    closeness = store['{}/centralities/closeness/raw'.format(self._id)]
                else:
                    closeness = store['{}/centralities/closeness/{}'.format(self._id, key)]
        except KeyError:
            closeness = closeness_centrality(self.interactions)
            self.add_closeness(closeness, key)

        return closeness.loc[closeness.index[closeness.index.isin(self.interactions.index)]]

    def add_clustering(self, clusters, key=None):
        with HDFStore(STORE) as store:
            if key is None:
                store.put('{}/clustering/raw'.format(self._id), clusters)
            else:
                store.put('{}/clustering/{}'.format(self._id, key), clusters)

    def get_clustering(self, key=None):
        with HDFStore(STORE) as store:
            if key is None:
                return store['{}/clustering/raw'.format(self._id)]
            else:
                return store['{}/clustering/{}'.format(self._id, key)]

    def HQCut(self):
        try:
            hq_cut = Clustering(self, self.get_clustering('HQCut'), 0)
            q_cut = Clustering(self, self.get_clustering('QCut'), 0)
        except KeyError:
            q_cut, hq_cut, q = HQCut(self.interactions)
            hq_cut = get_proto_clustering(self, hq_cut)
            q_cut = get_proto_clustering(self, q_cut)
            self.add_clustering(hq_cut, 'HQCut')
            self.add_clustering(q_cut, 'QCut')
            hq_cut = Clustering(self, hq_cut, 0)
            q_cut = Clustering(self, q_cut, 0)

        return hq_cut, q_cut

    def subnet_clustering(self):
        p_clustering = pd.Series(np.zeros(self.interactions.shape[0]),
                                 index=self.interactions.index, dtype=int)
        for i, net in enumerate(self.connected_subnets()):
            p_clustering.loc[net.index] = i

        return Clustering(self, p_clustering, 0)

    def connected_subnets(self):
        graph = nx.from_numpy_matrix(self.interactions.values)
        for comp in nx.connected_component_subgraphs(graph):
            sub_net = pd.DataFrame(np.array(nx.to_numpy_matrix(comp)))
            sub_net.index = self.interactions.index[comp.nodes()]
            yield sub_net

    def summary(self):
        proteins = pd.DataFrame(self.interactions.index, columns=['protein'], index=self.interactions.index)
        proteins = pd.merge(proteins, self.cogs, on='protein')

        proteins.loc[:, 'virulence'] = proteins.loc[:, 'protein'].isin(self.virulence_proteins)

        degrees = pd.DataFrame(self.interactions.sum(axis=1), columns=['connectivity'])
        proteins = pd.merge(proteins, degrees, left_on='protein', right_index=True)

        betw = pd.DataFrame(self.get_betweenness())
        proteins = pd.merge(proteins, betw, left_on='protein', right_index=True)

        closeness = pd.DataFrame(self.get_closeness())
        proteins = pd.merge(proteins, closeness, left_on='protein', right_index=True)

        return proteins

    def cut_away_disconnected_nodes(self):
        ind = np.where((self.interactions.values.sum(axis=1) > 0))[0]
        self.interactions = self.interactions.iloc[ind, ind]
        self.cog_index = self.cog_index.iloc[ind]

    def set_biggest_component(self):
        self.interactions = max(connected_subnets(self.interactions), key=lambda x: x.shape[0])
        self.cog_index = self.cog_index.loc[self.interactions.index]

def clustering_coefficients(net):
    g = nx.from_numpy_matrix(net.values)
    s = pd.Series(nx.clustering(g))
    s.index = net.index
    return s

def connected_subnets(net):
    graph = nx.from_numpy_matrix(net.values)
    for comp in nx.connected_component_subgraphs(graph):
        sub_net = pd.DataFrame(np.array(nx.to_numpy_matrix(comp)))
        sub_net.index = net.index[comp.nodes()]
        yield sub_net

class Clustering(object):
    def __init__(self, proteome, proto_clustering, min_size):
        self.proteome = proteome
        self.clusters = []
        self.cog_index = defaultdict(lambda: [])

        q = Q(proteome.interactions.values, proto_clustering.values)

        for i, cluster in enumerate(generate_clusters(proteome,
                                                      proto_clustering,
                                                      min_size)):
            cluster['q'] = q[i]
            self.register(cluster)

        self.virulence_coefficients()

    def register(self, cluster):
        i = len(self.clusters)
        self.clusters.append(cluster)
        for cog in cluster['cogs']:
            self.cog_index[cog].append(i)

    def virulence_coefficients(self):
        for cluster in self.clusters:
            vir = (pd.Series(cluster['proteins']).isin(
                self.proteome.virulence_proteins
            )).sum()
            cluster['virulence_coefficient'] = vir / float(len(cluster['proteins']))

    def summary(self):
        summary = []
        for cluster in self.clusters:
            size = len(cluster['proteins'])
            n_cogs = len(cluster['cogs'])

            summary.append((size, cluster['virulence_coefficient'], n_cogs/size, cluster['q']))

        return pd.DataFrame(summary, columns=['size', 'virulence_coefficient', 'cog-coverage', 'q'])

    def compare(self, cluster):
        own_clusters = set()
        cogs = cluster['cogs']
        for cog in cogs:
            own_clusters = own_clusters.union(self.cog_index[cog])

        scores = []
        coverages = []
        sizes = []

        for i in own_clusters:
            own_cluster = self.clusters[i]
            l_p1 = len(own_cluster['proteins'])
            l_p2 = len(cluster['proteins'])
            l_c = len(own_cluster['cogs'])
            coverages.append(l_c / l_p1)
            score = own_cluster['cogs'].isin(cluster['cogs']).sum() / l_p1 + cluster['cogs'].isin(own_cluster['cogs']).sum() / l_p2
            score /= 2
            score *= (1/((l_p1 - l_p2)**2 + 1))
            scores.append(score)
            sizes.append(l_p1)

        return pd.DataFrame(list(zip(scores, sizes, coverages, own_clusters)),
                            columns=['score', 'size', 'COG_coverage', 'cluster_id'])

    def align(self, clustering):
        pass

    def enrichment(self, propagate_counts, alpha, min_size=0, max_size=float('inf')):
        population = set(self.proteome.proteins)
        assoc = read_associations(self.proteome.go_assoc_filename())
        obo_dag = GODag(obo_file='data/go_annotations/go-basic.obo')

        for cluster in self.clusters:
            study = set(cluster['proteins'])

            l = len(study)

            if l > max_size or l < min_size:
                continue

            g = GOEnrichmentStudy(population, assoc, obo_dag,
                                  propagate_counts=propagate_counts,
                                  alpha=alpha,
                                  pvalcalc='fisher',
                                  methods=['bonferroni'])

            results = g.run_study(study)

            cluster['GO_terms'] = []

            for r in results:
                if r.p_bonferroni < 0.05:
                    cluster['GO_terms'].append(r)

    def cluster_summary(self, cluster_id):
        try:
            go_terms = []

            for x in self.clusters[cluster_id]['GO_terms']:
                go_terms.append({
                    'GO': x.GO,
                    'name': x.name,
                    'ratio in study': '{}/{}'.format(*x.ratio_in_study),
                    'ratio in pop': '{}/{}'.format(*x.ratio_in_pop),
                    'p_uncorrected': x.p_uncorrected,
                    'p_bonferroni': x.p_bonferroni
                })

            go_terms = pd.DataFrame(go_terms)
        except KeyError:
            go_terms = None

        p_sum = self.proteome.summary()
        ind = np.where(p_sum['protein'].isin(self.clusters[cluster_id]['proteins']))[0]

        return p_sum.iloc[ind], go_terms

    def plot(self, cluster_id, filename=None):
        A = self.subnet(cluster_id)
        graph = nx.from_numpy_matrix(A.values)
        pos = nx.spring_layout(graph)

        vir_nodes = np.where(A.index.isin(self.proteome.virulence_proteins))[0]
        other_nodes = np.where(~A.index.isin(self.proteome.virulence_proteins))[0]

        sizes = self.proteome.get_betweenness()

        sizes /= sizes.max()
        sizes = sizes[A.index]
        sizes *= 500
        sizes += 10

        vir_sizes = list(sizes[A.index[vir_nodes]])
        other_sizes = list(sizes[A.index[other_nodes]])

        nx.draw_networkx_nodes(graph, pos=pos, nodelist=list(vir_nodes), node_size=vir_sizes,
                               node_color='#e03e3e')
        nx.draw_networkx_nodes(graph, pos=pos, nodelist=list(other_nodes), node_size=other_sizes,
                               node_color='#77a6ba')

        #width = [graph[x[0]][x[1]]['weight'] * 4 for x in graph.edges()]

        nx.draw_networkx_edges(graph, pos=pos, width=2, alpha=0.5)

        nx.draw_networkx_edges(graph, pos=pos)
        plt.axis('off')
        if filename is None:
            plt.show()
        else:
            plt.savefig(filename)

    def subnet(self, cluster_id):
        ind = np.where(self.proteome.interactions.index.isin(
            self.clusters[cluster_id]['proteins']))[0]
        return self.proteome.interactions.iloc[ind, ind]

def enrichment(proteome, study, propagate_counts, alpha, p):
    population = set(proteome.proteins)
    assoc = read_associations(proteome.go_assoc_filename())
    obo_dag = GODag(obo_file='data/go_annotations/go-basic.obo')

    study = set(study)

    g = GOEnrichmentStudy(population, assoc, obo_dag,
                          propagate_counts=propagate_counts,
                          alpha=alpha,
                          pvalcalc='fisher',
                          methods=['bonferroni'])

    results = g.run_study(study)

    go_terms = []

    for r in results:
        if r.p_bonferroni < p:
            go_terms.append({
                'GO': r.GO,
                'name': r.name,
                'ratio in study': '{}/{}'.format(*r.ratio_in_study),
                'ratio in pop': '{}/{}'.format(*r.ratio_in_pop),
                'p_uncorrected': r.p_uncorrected,
                'p_bonferroni': r.p_bonferroni
            })

    return pd.DataFrame(go_terms)

def Q(net, clusters):
    """
    :param np.Array net:
    :param np.Array clusters:
    :return:
    """
    C = clusters.max()
    q = pd.Series([0]*C, index=range(1, C+1), dtype=float)

    degrees = net.sum(axis=1)

    M = degrees.sum()

    for c in range(1, C+1):
        #print('calculating Q for cluster {}'.format(c))
        ind = np.where(clusters == c)[0]
        community = net[ind, ind]
        e = community.sum() / 2
        d = degrees[ind].sum()
        q[c] = np.divide(e, M) - np.power(np.divide(d, 2 * M), 2)

    q['total'] = q.sum()

    return q

def generate_clusters(proteome, proto_clustering, min_size):
    d = defaultdict(lambda: [])

    for protein, cluster in proto_clustering.to_dict().items():
        d[cluster].append(protein)

    for cluster, proteins in d.items():
        if len(proteins) > min_size:
            ind = proteome.interactions.index.isin(proteins)
            ind = proteome.interactions.index[ind]
            yield {
                'proteins': proteins,
                'cogs': proteome.cog_index[ind].dropna()
            }

def betweenness_centrality(net):
    """
    :param pd.DataFrame net:
    :return:
    """
    graph = nx.from_numpy_matrix(net.values)
    c = pd.Series(nx.betweenness_centrality(graph), name='Betweenness centrality')
    c.index = net.index
    return c

def closeness_centrality(net):
    """
    :param pd.Dataframe net:
    :return:
    """
    graph = nx.from_numpy_matrix(net.values)
    c = pd.Series(nx.closeness_centrality(graph), name='Closeness centrality')
    c.index = net.index
    return c

def derive_network(P1, P2, method=None, *args, **kw_args):
    """
    :param similarities: similarity matrix
    :param method:
    :param args:
    :param kw_args:
    :return:
    """
    pass

def get_hubs(summary, factor=2):
    values = summary['Betweenness centrality']
    threshold = values.mean() + values.std() * factor
    high_betweenness = summary.iloc[np.where(values > threshold)[0], :]['protein']

    values = summary.loc[:, 'Closeness centrality']
    threshold = values.mean() + values.std() * factor
    high_closeness = summary.iloc[np.where(values > threshold)[0], :]['protein']

    values = summary.loc[:, 'connectivity']
    threshold = values.mean() + values.std() * factor
    high_connectivity = summary.iloc[np.where(values > threshold)[0], :]['protein']

    subset = high_betweenness[high_betweenness.isin(high_connectivity)]

    hubs = summary.loc[subset[subset.isin(high_closeness)].index].sort_values(by='virulence')

    return hubs

class NetSplit(object):
    """
    split interactions of P1 according to mutual COGs with P2
    :param P1:
    :param P2:
    :return:
    """

    def __init__(self, P1, P2):
        self.P1 = P1
        self.P2 = P2
        self.interactions = P1.interactions

        v1 = P1.cog_index.isin(P2.unique_cogs())
        v2 = P1.cog_index.isin(P1.unique_cogs())
        core = v1 & v2
        periphery = ~P1.cog_index.isin(P2.cog_index)
        intermediate = ~periphery & ~core

        core = np.where(self.interactions.index.isin(core[core].index))[0]
        periphery = np.where(self.interactions.index.isin(periphery[periphery].index))[0]
        intermediate = np.where(self.interactions.index.isin(intermediate[intermediate].index))[0]

        self.split_index = (core, intermediate, periphery)

    @property
    def core(self):
        return self.interactions.iloc[self.split_index[0], self.split_index[0]]

    @property
    def periphery(self):
        return self.interactions.iloc[self.split_index[2], self.split_index[2]]

    @property
    def intermediate(self):
        return self.interactions.iloc[self.split_index[1], self.split_index[1]]

    def summary(self):
        s = self.P1.summary()
        regions = pd.DataFrame(self.clustering(), columns=['region'])
        return pd.merge(s, regions, left_on='protein', right_index=True)

    def diff_degrees(self, net_split):
        """
        compare degrees in core network
        raises error, when NetSplits don't belong together
        :param net_split:
        :return:
        """
        d1 = pd.Series(self.core.values.sum(axis=1), index=ns1.core.index, name='{} degrees'.format(self.P1._id))
        d2 = pd.Series(net_split.core.values.sum(axis=1), index=ns1.core.index, name='{} degrees'.format(net_split.P1._id))
        diff = (d1 - d2).abs()
        diff.rename('difference', inplace=True)
        return pd.concat([d1, d2, diff], axis=1)

    def diff_closeness(self, net_split, key=None):
        """
        compare degrees in core network
        raises error, when NetSplits don't belong together
        :param net_split:
        :return:
        """
        try:
            closeness1 = self.P1.get_closeness(key=key)
        except KeyError:
            closeness1 = closeness_centrality(self.interactions)
            self.P1.add_closeness(closeness1, key=key)

        closeness1.rename('Closeness centrality {}'.format(self.P1._id), inplace=True)

        try:
            closeness2 = net_split.P1.get_closeness(key=key)
        except KeyError:
            closeness2 = closeness_centrality(net_split.interactions)
            net_split.P1.add_closeness(closeness2, key=key)

        closeness2.rename('Closeness centrality {}'.format(self.P2._id), inplace=True)

        closeness1 = closeness1[self.core.index]
        closeness2 = closeness2[net_split.core.index]

        closeness2.index = closeness1.index
        diff = (closeness1 - closeness2).abs()
        diff.rename('Closeness difference', inplace=True)

        return pd.concat([closeness1, closeness2, diff], axis=1)

    def diff_betweenness(self, net_split, key=None):
        """
        compare degrees in core network
        raises error, when NetSplits don't belong together
        :param net_split:
        :return:
        """
        try:
            betweenness1 = self.P1.get_betweenness(key=key)
        except KeyError:
            betweenness1 = betweenness_centrality(self.interactions)
            self.P1.add_betweenness(betweenness1, key=key)

        betweenness1.rename('Betweenness centrality {}'.format(self.P1._id), inplace=True)

        try:
            betweenness2 = net_split.P1.get_betweenness(key=key)
        except KeyError:
            betweenness2 = betweenness_centrality(net_split.interactions)
            net_split.P1.add_betweenness(betweenness2, key=key)

        betweenness2.rename('Betweenness centrality {}'.format(self.P2._id), inplace=True)

        betweenness1 = betweenness1[self.core.index]
        betweenness2 = betweenness2[net_split.core.index]

        betweenness2.index = betweenness1.index

        diff = (betweenness1 - betweenness2).abs()
        diff.rename('Betweenness difference', inplace=True)
        return pd.concat([betweenness1, betweenness2, diff], axis=1)

    def clustering(self):
        """
        return partition of network as Clustering
        :return:
        """
        assignments = pd.Series(['core']*self.interactions.shape[0],
                                index=self.interactions.index, dtype=str)

        assignments.iloc[self.split_index[1]] = 'ambiguous'
        assignments.iloc[self.split_index[2]] = 'periphery'

        return assignments

def degree_distribution_plot(P1, P2, filename=None):
    """
    Degree distributions
    """

    degrees = P1.interactions.sum()

    fig, axes = plt.subplots(nrows=2, ncols=2)

    degrees.plot(kind='hist', loglog=False, bins=20, ax=axes[0, 0])
    degrees.plot(kind='kde', loglog=True, xlim=(1, 60), ylim=(10e-10, 1), ax=axes[0, 1])

    degrees = P2.interactions.sum()

    degrees.plot(kind='hist', loglog=False, bins=20, ax=axes[1, 0])
    degrees.plot(kind='kde', loglog=True, xlim=(1, 60), ylim=(10e-10, 1), ax=axes[1, 1])

    plt.tight_layout(pad=0.4, w_pad=0.5, h_pad=1.0)

    if filename is not None:
        plt.savefig(filename)
    else:
        plt.show()

def dump(hubs, sub_dir_name, C, patric_annotation):
    try:
        os.makedirs('results/{}/no_hubs'.format(sub_dir_name))
    except:
        pass
    try:
        os.makedirs('gfx/{}/no_hubs'.format(sub_dir_name))
    except:
        pass

    # print(C1.summary().sort_values('virulence_coefficient', ascending=False))
    for i in range(len(C.clusters)):
        summary, go_terms = C.cluster_summary(i)
        s = pd.merge(summary, patric_annotation, left_on='protein', right_index=True, how='outer')

        summary = s[s['protein'].isin(summary['protein'])]

        if go_terms is not None:
            if summary['protein'].isin(hubs['protein']).any():
                summary.to_csv('results/{}/{}_cluster_{}_sum.tsv'.format(sub_dir_name, sub_dir_name, i), sep='\t')
                go_terms.to_csv('results/{}/{}_cluster_{}_go.tsv'.format(sub_dir_name, sub_dir_name, i), sep='\t')

                C.plot(i, 'gfx/{}/{}.png'.format(sub_dir_name, i))
                plt.close()

            else:
                summary.to_csv('results/{}/no_hubs/{}_cluster_{}_sum.tsv'.format(sub_dir_name, sub_dir_name, i), sep='\t')
                go_terms.to_csv('results/{}/no_hubs/{}_cluster_{}_go.tsv'.format(sub_dir_name, sub_dir_name, i), sep='\t')

                C.plot(i, 'gfx/{}/no_hubs/{}.png'.format(sub_dir_name, i))
                plt.close()

def align(C1, C2, s_min):
    results = []

    for i in range(len(C1.clusters)):

        result = C2.compare(C1.clusters[i])
        if result.shape[0] > 0:
            for r in result.iterrows():
                s = r[1]['score']
                if s > s_min:
                    results.append((i, int(r[1]['cluster_id']), s))

    return pd.DataFrame(results, columns=['C1_ID', 'C2_ID', 'score'])

if __name__ == '__main__':

    P1 = Proteome('198214', True)
    P2 = Proteome('316385', False)

    P1.normalize(1000)
    P2.normalize(1000)

    ns1 = NetSplit(P1, P2)
    ns2 = NetSplit(P2, P1)

    C1_HQ, C1_Q = P1.HQCut()
    C2_HQ, C2_Q = P2.HQCut()


import pandas as pd
from PyQt5 import QtCore

class PandasModel(QtCore.QAbstractTableModel):
    """
    Model that allow to represent pandas DataFrame as QtTable.

    Arguments
    ----------
    df : pandas.DataFrame
        DataFrame to be table initiated with.
    parent : bool
        Optional built-in method in QtCore to inherit properties of similar Qt object.
    """
    def __init__(self, df=pd.DataFrame(), parent=None):
        QtCore.QAbstractTableModel.__init__(self, parent=parent)
        self._df = df
        self._original_df = df.copy(deep=True)

    def headerData(self, section, orientation, role=QtCore.Qt.DisplayRole):
        """

        Parameters
        ----------
        section : str
            Section of DataFrame to take headers from.
        orientation : QtCore.Qt object
            One of Qt attributes that determine table orientation via headers placements
            (horizontal or vertical).
        role : QtCore.Qt object
            Parameter that responsible for choosing if data to be rendered visually as QString.

        Returns
        -------
        None
        """
        if role != QtCore.Qt.DisplayRole:
            return QtCore.QVariant()

        if orientation == QtCore.Qt.Horizontal:
            try:
                return self._df.columns.tolist()[section]
            except (IndexError, ):
                return QtCore.QVariant()
        elif orientation == QtCore.Qt.Vertical:
            try:
                return self._df.index.tolist()[section]
            except (IndexError, ):
                return QtCore.QVariant()

    def data(self, index, role=QtCore.Qt.DisplayRole):
        """

        Parameters
        ----------
        index : pandas.DataFrame.index
            Index of DataFrame for table to be rendered.
        role : QtCore.Qt object
            Parameter that responsible for choosing if data to be rendered visually as QString.

        Returns
        -------
        QtCore.QVariant
        """
        if role != QtCore.Qt.DisplayRole:
            return QtCore.QVariant()

        if not index.isValid():
            return QtCore.QVariant()
        row = index.row()
        column = index.column()

        return QtCore.QVariant(str(self._df.ix[row, column]))

    def setData(self, index, value, role):
        """

        Parameters
        ----------
        index : pandas.DataFrame.index
            Index of DataFrame for table to be rendered.
        value : str
            Values to be added to table.
        role : QtCore.Qt object
            Parameter that responsible for choosing if data to be rendered visually as QString.

        Returns
        -------

        """
        row = self._df.index[index.row()]
        col = self._df.columns[index.column()]
        if hasattr(value, 'toPyObject'):
            value = value.toPyObject()
        else:
            dtype = self._df[col].dtype
            if dtype != object:
                value = None if value == '' else dtype.type(value)
        self._df.set_value(row, col, value)
        return True

    def rowCount(self, parent=QtCore.QModelIndex()):
        return len(self._df.index)

    def columnCount(self, parent=QtCore.QModelIndex()):
        return len(self._df.columns)

    def sort(self, column, order):
        """
        Implementation of excel-like column sorting via built-in Qt methods.

        Parameters
        ----------
        column : pandas.DataFrame
            Using pandas columns attribute to get list of columns.
        order : QtCore.Qt.AscendingOrder
            Qt built-in attribute to set oder of table's columns sort.
        Returns
        -------
        None
        """
        colname = self._df.columns.tolist()[column]
        self.layoutAboutToBeChanged.emit()
        self._df.sort_values(colname, ascending=order == QtCore.Qt.AscendingOrder, inplace=True)
        self._df.reset_index(inplace=True, drop=True)
        self.layoutChanged.emit()

    def reset(self):
        # Implementation of method that allow to go back to original df,
        # in sense of resetting used filters.
        self.layoutAboutToBeChanged.emit()
        # just reassign self._df to original
        self._df = self._original_df.copy(deep=True)
        self.layoutChanged.emit()

    def export(self, original=True):
        # Outputting to excel file via DataFrame in-built method.
        if original:
            self._original_df.to_excel('output.xlsx')
        else:
            self._df.to_excel('output_filtered.xlsx')

    def btn_clk(self):
        # Method that defines behaviour of import button.
        path = self.lineEdit.text()
        df = pd.read_csv(path)
        model = PandasModel(df)
        self.tableView.setModel(model)


#!/usr/bin/env python
# -*- coding: UTF-8 -*-

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import tensorflow as tf
import sys
from vocab import Vocab
from lib.models.sentiment.baseatt import BaseAttentions

# ***************************************************************
class MCGAttention(BaseAttentions):
    """"""

    # =============================================================
    def __call__(self, dataset, moving_params=None):
        """"""

        vocabs = dataset.vocabs
        inputs = dataset.inputs
        targets = dataset.targets
        reuse = (moving_params is not None)
        self.tokens_to_keep3D = tf.expand_dims(tf.to_float(tf.greater(inputs[:, :, 0], vocabs[0].ROOT)), 2)
        self.sequence_lengths = tf.reshape(tf.reduce_sum(self.tokens_to_keep3D, [1, 2]), [-1, 1])
        self.n_tokens = tf.reduce_sum(self.sequence_lengths)
        self.moving_params = moving_params

        word_inputs = vocabs[0].embedding_lookup(inputs[:, :, 0], inputs[:, :, 1], moving_params=self.moving_params)
        istarget = inputs[:, :, 2]  # batch_size, bucket_size
        bftarget = inputs[:, :, 3]  # same above
        aftarget = inputs[:, :, 4]  # same above
        nontarget = 1 - istarget
        top_recur = self.embed_concat(word_inputs)  # batch_size, bucket_size, word_dim(200)
        # Bottom/Original RNN layers
        for i in xrange(self.n_recur):
            with tf.variable_scope('RNN%d' % i, reuse=reuse):
                top_recur, _ = self.RNN(top_recur)
                # ====================================================================================================
        with tf.variable_scope('Attention_context', reuse=reuse):
            """"""
            if moving_params is None:
                top_recur = tf.nn.dropout(top_recur, 0.6, seed=76)  # batch_size, bucket_size, lstm_dim(300)
            # =======================================================
            htscore = self.getTarHd(top_recur, istarget)  # get the target_word average score

            attenrseq = attenlseq = attenwseq = htscore

            for j in range(self.n_remem):
                with tf.variable_scope('MEMWAtt%d'%j, reuse=reuse):
                    input_size = attenwseq.get_shape().as_list()[-1]
                    batch_size = tf.shape(attenwseq)[0]
                    input_shape = tf.pack([batch_size, 1, input_size])
                    out_shape = tf.pack([batch_size, input_size])
                    attenwseq = tf.reshape(attenwseq, input_shape)
                    if moving_params is None:
                        attenwseq = tf.nn.dropout(attenwseq, 0.6, seed=77)
                    attenwseq,_ = self.MRNN(attenwseq)
                    attenwseq = tf.reshape(attenwseq, out_shape)
                    attenwseq = self.compAtt(top_recur, attenwseq, nontarget, scope='allatt', reuse=reuse)
                    if moving_params is None:
                        attenwseq = tf.nn.dropout(attenwseq, 0.5, seed=78)

            for j in range(self.n_remem):
                with tf.variable_scope('MEMLAtt%d'%j, reuse=reuse):
                    input_size = attenlseq.get_shape().as_list()[-1]
                    batch_size = tf.shape(attenlseq)[0]
                    input_shape = tf.pack([batch_size, 1, input_size])
                    out_shape = tf.pack([batch_size, input_size])
                    attenlseq = tf.reshape(attenlseq, input_shape)
                    if moving_params is None:
                        attenlseq = tf.nn.dropout(attenlseq, 0.6, seed=77)
                    attenlseq,_ = self.MRNN(attenlseq)
                    attenlseq = tf.reshape(attenlseq, out_shape)
                    attenlseq = self.compAtt(top_recur, attenlseq, nontarget, scope='leftatt', reuse=reuse)
                    if moving_params is None:
                        attenlseq = tf.nn.dropout(attenlseq, 0.5, seed=78)

            for j in range(self.n_remem):
                with tf.variable_scope('MEMRAtt%d'%j, reuse=reuse):
                    input_size = attenrseq.get_shape().as_list()[-1]
                    batch_size = tf.shape(attenrseq)[0]
                    input_shape = tf.pack([batch_size, 1, input_size])
                    out_shape = tf.pack([batch_size, input_size])
                    attenrseq = tf.reshape(attenrseq, input_shape)
                    if moving_params is None:
                        attenrseq = tf.nn.dropout(attenrseq, 0.6, seed=77)
                    attenrseq,_ = self.MRNN(attenrseq)
                    attenrseq = tf.reshape(attenrseq, out_shape)
                    attenrseq = self.compAtt(top_recur, attenrseq, nontarget, scope='rightatt', reuse=reuse)
                    if moving_params is None:
                        attenrseq = tf.nn.dropout(attenrseq, 0.5, seed=78)

            for j in range(self.n_remem):
                with tf.variable_scope('MEMRHT%d'%j, reuse=reuse):
                    input_size = htscore.get_shape().as_list()[-1]
                    batch_size = tf.shape(htscore)[0]
                    input_shape = tf.pack([batch_size, 1, input_size])
                    out_shape = tf.pack([batch_size, input_size])
                    htscore = tf.reshape(htscore, input_shape)
                    if moving_params is None:
                        htscore = tf.nn.dropout(htscore, 0.6, seed=77)
                    htscore, _ = self.MRNN(htscore)
                    htscore = tf.reshape(htscore, out_shape)
                    if moving_params is None:
                        htscore = tf.nn.dropout(htscore, 0.5, seed=78)

            sntVec = self.Seq2Pb(attenwseq, attenlseq, attenrseq, relu=True, hitarget=htscore, reuse=reuse)
            attout = self.attoutput(sntVec, targets)
            return attout
            # ======================================================================================================================
            #
            # #=============================================================

    def prob_argmax(self, parse_probs, rel_probs, tokens_to_keep):
        """"""

        parse_preds = self.parse_argmax(parse_probs, tokens_to_keep)
        rel_probs = rel_probs[np.arange(len(parse_preds)), parse_preds]
        rel_preds = self.rel_argmax(rel_probs, tokens_to_keep)
        return parse_preds, rel_preds


import os

import sqlite3

def clear_webauthn_entries():
    conn = sqlite3.connect('/home/damian/Documents/Educational/MEng/webauthn-firewall/webauthn-firewall.db')
    conn.execute('delete from webauthn_entries ;')
    conn.commit()
    conn.close()

def main():
    for n_users in range(1, 30 + 1):
        clear_webauthn_entries()
        os.system('python3.6 test_gogs.py {}'.format(n_users))

if __name__ == '__main__':
    main()


import random

nums = list(range(1, 36))
jackpot = random.sample(nums, k=5)
print("Power ball numbers are >")
for x in jackpot:
    random.choice(jackpot)
    print(x)
power_num = random.randint(1, 10)
print(f"Power number = {power_num}")
nums2 = list(range(1, 33))
mega_6 = random.sample(nums2, k=6)
print(f"Mega 6 numbers are: {mega_6}")


# Programming Exercise 3-2
#
# Program to find which of two rectangles has the greater area.
# This program will get two sets of lengths and widths from a user,
# use them to calculate and compare two area values,
# and display a message comparing those areas

# Local variables
# you need length, width and area for A and for B
# initialize these as floats

# Get length A from the user and convert it to a float

# Get width A from the user and convert it to a float

# Get length B from the user and convert it to a float

# Get width B from the user and convert it to a float

# Calculate area A

# Calculate area B

# Print each area, formatting the values to 2 decimal places

# if area A is greater, print "A is greater" message.

# else if area B is greater, print "B is greater" message.

# else print "A and B are equal" message.

length_A = float(input("Enter rectangle A's length: "))
width_A = float(input("Enter rectangle A's width: "))
length_B = float(input("Enter rectangle B's length: "))
width_B = float(input("Enter rectangle B's width: "))

area_a = length_A * width_A
area_b = length_B * width_B

if area_a > area_b :
    out = "The Area of A is greater than the Area of B"
elif area_b > area_a :
    out = "The Ares of B is greater than the Area of A"
else :
    out = "The Areas of both A and B are equal"

area_a = "{0:.2f}".format(area_a)
area_b = "{0:.2f}".format(area_b)

print("Area A : " + area_a)
print("Area B : " + area_b)
print(out)


import pytest
from palindrome.products import *

class TestPalindromes:

    @pytest.mark.parametrize(("value", "expected_low", "expected_high"), [
        [2, 10, 100],
        [3, 100, 1000],
        [4, 1000, 10000], ])
    def test_range_for_n_digit_nums(self, value, expected_low, expected_high):
        r = range_for_n_digit_nums(value)
        assert r.low == expected_low and r.high == expected_high

    def test_palindromes(self):
        assert generate_palindromes(num_digits=3)

    def test_palindrome_range(self):
        palindromes = generate_palindromes(num_digits=3)
        assert len(palindromes) == 900
        for p in palindromes:
            assert p > 100000 and p < 1000000

    @pytest.mark.parametrize(("value", "expected"), [
        [411, 114],
        [515, 515],
        [666, 666],
        [723, 327], ])
    def test_reverse_num(self, value, expected):
        assert reverse_num(value) == expected

    @pytest.mark.parametrize(("value", "expected"), [
        [2, 9009],
        [3, 906609], ])
    def test_largest_palindrome(self, value, expected):
        assert largest_palindrome(value).palindrome == expected


import hashlib
import os

def file_auth(file_name, buffer_size=65536):
    md5 = hashlib.md5()
    with open(file_name, 'rb') as src:
        while True:
            data_src = src.read(buffer_size)
            if not data_src:
                break
            md5.update(data_src)
    return md5.hexdigest()

def dir_auth(dir_name, buffer_size=65536):
    hash_list = []
    for root, dirs, files in os.walk(dir_name, topdown=True):
        for name in files:
            file_name = os.path.join(root, name)
            hash_list.append(file_auth(file_name, buffer_size))
    return hash_list


import pandas as pd
import os,sys
#直接用pandas讀取txt似乎不好用？
#或者考慮先用python系統函數組入excel，再用pandas讀取來作業

sys.path.insert(0,r'C:\Users\sssssyyyy\Desktop\問卷組入')
from functions import *

df0 = pd.DataFrame({},index=[0,1])
df2 = pd.DataFrame({},index=[0,1,2,3,4])
for path,dirs,files in os.walk(os.getcwd()):
    for f in files:
        fnhead,fntail = os.path.splitext(f)
        if fntail == '.txt':
            print (fnhead)
            #df = pd.read_table(os.path.join(path,f),sep='[ |\t]',engine='python')

            df = pd.read_csv(os.path.join(path,f),engine='python',header=None)
            df2 = df.append(df2,sort=False)
print (df2)
deleteAnyway('111.xlsx')
df2.to_excel('111.xlsx',index=None,columns=None)


import prueba as pr
name_Workwook = 'FLOW Operation and support report ASAP 7.0.2 JAMU66, WASS133 2018-11-26.xlsx'
name_Sheet = 'JAMU WEEK'
'''
object1 = pr.PerformancePandas(name_Workwook, name_Sheet)
encontrados = object1.busqueda('WASS BAR_COMG')#regresa estadisticas de las coincidencias encontradas
indice = object1.obtener_indice()
data = object1.obtener_data(indice[0][0])
data = object1.text_To_Columns(data)
object1.write_pandas_xlsx_document(data, 'nuevodocmnetiss.xlsx')
print(encontrados)
print(type(indice))
#print(data)

'''

#Hacer una instancia de la clase
instanca_Data = pr.PerformancePandas(name_Workwook, name_Sheet)

#Buscar todas las tablas necesarias, las siguientes lineas regresan solo las estadisticas de las coincidencias encontradas
#luego de hacer cada busqueda se obtine el indice de las coincidencias
#el siguiente paso es obtener los datos
#transformar de texto a columnas
'''wos host'''
encontrados_wos = instanca_Data.busqueda('WOS HOST')
indice = instanca_Data.obtener_indice()
data_wos = instanca_Data.obtener_data(indice[0][0] + 2) #al indice le sumamos 2 para evitar poner el nombre de la tabla y la linea que separa al nombre de los datos
data_wos = instanca_Data.text_To_Columns(data_wos)

'''completed'''
encontrados_complete = instanca_Data.busqueda('COMPLETECOUNT HOST')
indice = instanca_Data.obtener_indice()
data_complete = instanca_Data.obtener_data(indice[0][0] + 2)
data_complete = instanca_Data.text_To_Columns(data_complete)

'''failed '''
encontrados_failed = instanca_Data.busqueda('FAILEDDETAILED')
indice = instanca_Data.obtener_indice()
data_failed = instanca_Data.obtener_data(indice[0][0] + 2)
data_failed = instanca_Data.text_To_Columns(data_failed)

'''timeout'''
encontrados_timeout = instanca_Data.busqueda('TIMEOUTDETAILED')
indice = instanca_Data.obtener_indice()
data_timeout = instanca_Data.obtener_data(indice[0][0] + 2)
data_timeout = instanca_Data.text_To_Columns(data_timeout)

#Convertir a diccionarios para no perder la informacion a la hora de ordenar y alinear
dictionary_wos = instanca_Data.convert_to_directory(data_wos[:,1], data_wos[:,0])
dictionary_complete = instanca_Data.convert_to_directory(data_complete[:,1], data_complete[:,0])
dictionary_failed = instanca_Data.convert_to_directory(data_failed[:,1], data_failed[:,0])
dictionary_timeout = instanca_Data.convert_to_directory(data_timeout[:,1], data_timeout[:,0])

#podemos sacar los archivos correspondientes a las tablas encontradas
instanca_Data.write_pandas_xlsx_document(instanca_Data.dictionary_to_pyMatrix(dictionary_timeout), 'timeout-JAMU.xlsx')
instanca_Data.write_pandas_xlsx_document(instanca_Data.dictionary_to_pyMatrix(dictionary_failed), 'failed-JAMU.xlsx')
'''+*******************ESTA pendiente la linea anterior********************'''

#Luego que estan los diccionarios hay que alinear los vectores

data_wos, data_complete = instanca_Data.alignment(data_wos[:,1], data_complete[:,1])
data_wos, data_failed = instanca_Data.alignment(data_wos, data_failed[:,1])
data_wos, data_timeout = instanca_Data.alignment(data_wos, data_timeout[:,1])

#crear cada una de las matrices individuales
completed_Matrix = instanca_Data.create_Matrix_Excel(data_wos, data_complete, dictionary_wos, dictionary_complete)
failed_Matrix = instanca_Data.create_Matrix_Excel(data_wos, data_failed, dictionary_wos, dictionary_failed)
timeout_Matrix = instanca_Data.create_Matrix_Excel(data_wos, data_timeout, dictionary_wos, dictionary_timeout)

#Crear la matriz completa
full_Matrix = []
for i in range(len(data_wos)):
    full_Matrix.append(
        completed_Matrix[i] + failed_Matrix[i][2:] + timeout_Matrix[i][2:])

#escribir el archivo de excel
instanca_Data.write_pandas_xlsx_document(full_Matrix, 'Performance-JAMU.xlsx')
'''
faileddetailed_count = instanca_Data.busqueda('FAILEDDETAILED')
timeoutdetailed_count = instanca_Data.busqueda('TIMEOUTDETAILED')

print('Estadisticas encontradas: \n')
print('\nWOS HOST: ', wos_host)
print('\nCOMPLETE: ', complete_count)
print('\nFAILED', faileddetailed_count)
print('\nTIMEOUT', timeoutdetailed_count)
'''


import os, sys
import numpy as np
import glob
import numpy
import Tkinter, tkFileDialog
from monDimReduce import SAXSDimReduce
from peakBBA import peakFitBBA
from save_wafer_heatMap import FWHMmap, contrastMap
from input_file_parsing import parse_config

import time

print('*************************************************************')
print('*********************************** Begin Batch Processing...')
print('*************************************************************')

# grab monitor folder
#root = Tkinter.Tk()
#root.withdraw()

dataPath = os.path.expanduser('~/monHiTp/testBkgdCSV/')
#dataPath = os.path.expanduser('~/data/bl1-5/Nov2017/Takeuchi/NiTiCuCo_SampleID2/')
#dataPath = os.path.expanduser('~/data/bl1-5/May2018/AlTiCo/data/k11a/')
#dataPath = tkFileDialog.askdirectory(title='Select folder to process')
if dataPath is '':
    print('No data folder selected, aborting...')
    sys.exit()

configPath = os.path.expanduser('~/monHiTp/config')
#configPath = tkFileDialog.askopenfilename(title='Select Config File')
if configPath is '':
    print('No config file supplied, aborting...')
    sys.exit()

print('Config File: ' + configPath)
print('Folder to process: ' + dataPath)
print('')

##########################################Extension chooser?...

fileList = glob.glob(os.path.join(dataPath + 'Processed/', '*1D.csv'))
if len(fileList) == 0:
    sys.exit('No files found')

# Sort out config file
config = parse_config(configPath)
if config:
    skipExisting = config['skipExisting']
    hiLimit = config['highlightLimit']
else:
    #Qrange, peakShape, peakNo, fit_order, hiLimit = None, None, None, None, None
    print('no config file')

files = fileList[config['startImg']:config['endImg']]
fileGen = (x for x in files)

loopTime = []
stage2Time = []
for filePath in fileGen:
    filename = os.path.basename(filePath)
    fileRoot, ext = os.path.splitext(filename)

    if skipExisting:
        procPath = dataPath+'Processed/peak_details/'+fileRoot+'_curveParams.csv'
        hasParamsFile = os.path.isfile(procPath)
        if hasParamsFile:
            print('Skipping file: {}'.format(procPath))
            continue

    start = time.time()

    print('{0}'.format(filePath))
    print(filename + ' detected, processing')

    # fake a path in prev folder
    # Assume name of format xxxxx_[index]_1D.csv
    fakeName = filename[:-7]+'.tif'
    fakePath = dataPath + fakeName
    print(fakePath)
    ########## Begin data reduction scripts ###########################
    stage1int = time.time()

    peakFitBBA(fakePath, config)
    stage2int = time.time()
    ########## Visualization #########################################
    # Pulling info from master CSV
    FWHMmap(fakePath)
    contrastMap(fakePath, hiLimit)

    print(filename + ' completed')

    end = time.time()
    loopTime += [(end-start)]
    stage2Time += [(stage2int - stage1int)]

    #break

# Evaluate performance
avgTime = np.mean(loopTime)
maxTime = np.max(loopTime)
avg2 = np.mean(stage2Time)
max2 = np.max(stage2Time)
print('====================================================')
print('====================================================')
print('Files finished processing')
print('-----Avg {:.4f}s / file, max {:.4f}.s / file'.format(avgTime, maxTime))
print('-----Stage2: Avg {:.4f}s / file, max {:.4f}.s / file'.format(avg2, max2))
print('-----Total Time Elapsed {:4f}s'.format(np.sum(loopTime)))
print('====================================================')
print('====================================================')


from typing import Dict
from dbgen.core.parsing import parse_line

def parse_psp_gpaw(log:str) -> str:
    """
    Returns simplified string representation of the pseudopotnetial used in a
    DFT calculation
    """
    rawpspth = parse_line(log,'setups',0)

    if rawpspth is None:
        return 'paw' # default
    else:
        pspth = rawpspth.split(': ')[-1].strip()
        if 'gpaw-setups-0.6.6300' in pspth:
            return 'oldpaw'
        elif 'setups/' == pspth[:7]:
            return 'paw'
        else:
            return pspth


#!/usr/bin/python
import angr,simuvex,claripy
import sys,os
import logging,traceback
import copy,re
import time
import networkx
try:
    import pickle as pickle
except ImportError:
    import pickle

from utils_sig import *

ARCH = 'aarch64'
BASE = 0xffffff8008080000;

def try_dfg():
    b = load_kernel_image(sys.argv[1],ARCH,BASE)
    start = BASE+0xc440c4
    end = BASE+0xc44178
    cfg = get_cfg_acc(b,start,end)
    func = cfg.functions[start]
    n = cfg.get_any_node(addr=start,anyaddr=True)
    '''
    print func
    for n in func.nodes:
        print n
    func.dbg_draw('test.png')
    '''
    #Note that we should choose 'no optimization' to ensure that vex code faithfully reflect relationship between original asm code.
    #E.g.
    # 1 mov x19,x0
    # 2 cbz x19,0xffff6000
    #We want to see that ins 2 depends on ins 1, but with vex optimization, x0 may be stored in t0 previously, and this same t0 will be used directly
    #in both ins 1 and 2, thus we lost the dependency between ins 1 and 2. (now they both depends on t0, but without relationship between themselves).
    dfg = b.analyses.DFG(nodes=[n.addr],opt_level=0)

    print('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
    print_vex_blocks(b,[n.addr])
    print('<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<')
    ndfg = dfg.dfgs[start]
    for node in ndfg.nodes():
        print(node)
    print('<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<')
    for root in get_root_nodes_from_dfg(ndfg):
        print(root)

def try_vfg():
    b = load_kernel_image(sys.argv[1],ARCH,BASE)
    start = BASE+0xc440c4
    end = BASE+0xc44178
    cfg = get_cfg_fast(b,start,end)

    st = b.factory.blank_state(addr=start)
    st.options.add(simuvex.o.CALLLESS)
    # To prevent the engine from discarding log history
    st.options.add(simuvex.o.TRACK_ACTION_HISTORY)
    st.inspect.b('symbolic_variable', when=simuvex.BP_AFTER, action=sym_capture)
    st.inspect.b('address_concretization', when=simuvex.BP_AFTER, action=addr_conc_capture)

    vfg = b.analyses.VFG(cfg=cfg, context_sensitivity_level=0, start=start, function_start=start, initial_state=None, interfunction_level=0)
    print(vfg)

def print_cfg_graph(start,end):
    b = load_kernel_image(sys.argv[1],ARCH,BASE)
    #x3
    #print b.arch.translate_register_name(offset = 0x28, size = 8)
    cfg = get_cfg_acc(b,start,end)
    func_cfg = get_func_cfg(cfg,start,normalize=False)
    _get_as = lambda n:(hex(n.addr) if n.addr is not None else str(n.addr),hex(n.size) if n.size is not None else str(n.size))
    n_set = set()
    p_set = set()
    s_set = set()
    for n in func_cfg.nodes():
        (addr,sz) = _get_as(n)
        print('---------%s,%s---------' % (addr,sz))
        n_set.add(addr + '-' + sz)
        preds_str = 'Preds: '
        for x in func_cfg.predecessors(n):
            (addr,sz) = _get_as(x)
            preds_str = preds_str + '[%s,%s] '%(addr,sz)
            p_set.add(addr + '-' + sz)
        succs_str = 'Succs: '
        for x in func_cfg.successors(n):
            (addr,sz) = _get_as(x)
            succs_str = succs_str + '[%s,%s] '%(addr,sz)
            s_set.add(addr + '-' + sz)
        print(preds_str)
        print(succs_str)
    print('Only in n_set: ' + str(n_set.difference(p_set,s_set)))
    print('Only in p_set but not n_set: ' + str(p_set.difference(n_set)))
    print('Only in s_set but not n_set: ' + str(s_set.difference(n_set)))

def print_vex(addrs):
    b = load_kernel_image(sys.argv[1],ARCH,BASE)
    print_vex_blocks(b,addrs)

def print_capstone(addrs):
    b = load_kernel_image(sys.argv[1],ARCH,BASE)
    for addr in addrs:
        caps = b.factory.block(addr).capstone
        #print type(caps.insns[0].insn)
        #print dir(caps.insns[0].insn)
        #print type(caps.insns[0].insn.mnemonic)
        print(caps.insns[0].insn.mnemonic)
        #print caps.insns[0].insn.insn_name
        print(caps)

def show_pickled_sig():
    with open(sys.argv[1],'rb') as f:
        sig = pickle.load(f)
        show_signature(sig)

def test_segments():
    segments = get_code_segments(sys.argv[1],BASE)
    print([(hex(x),hex(y),hex(z)) for (x,y,z) in segments])

if __name__ == '__main__':
    #print_cfg_graph(0xffffff8008dddbcc,0xffffff8008dddec8)
    #print_vex([0xFFFFFFC0003498A0])
    #print_capstone([0xffffff80088a1510])
    show_pickled_sig()
    #test_segments()