import tkinter as tk

class SizeDialog(tk.Frame):
    def __init__(self, parent):
        tk.Frame.__init__(self, parent, width=300, height=150)
        self.grid_propagate(0)
        self.grid_columnconfigure(1, weight=1)
        self.map_width      = tk.IntVar()
        self.map_width.set(20)
        self.map_height     = tk.IntVar()
        self.map_height.set(20)

        text_label = tk.Label(self, text="Enter the dimension of the room\nyou want to run simulation on")
        text_label.grid(row=0, column=0, columnspan=2, sticky='nsew')
        width_label         = tk.Label(self, text="Width [m] :")
        self.width_scale    = tk.Scale(self, from_=10, to=30, orient=tk.HORIZONTAL, variable=self.map_width)
        height_label        = tk.Label(self, text="Height [m] :")
        self.height_scale   = tk.Scale(self, from_=10, to=30, orient=tk.HORIZONTAL, variable=self.map_height)

        self.ok_button      = tk.Button(self, text="Run Radar Measurement Simulator")
        width_label.grid(row=1, column=0, sticky='w')
        self.width_scale.grid(row=1, column=1, sticky='we')
        height_label.grid(row=2, column=0, sticky='w')
        self.height_scale.grid(row=2, column=1, sticky='we')
        self.ok_button.grid(row=3, column=0, columnspan=2, sticky='nsew')

    def getValues(self):
        return [self.width_scale.get(), self.height_scale.get()]


import random
import requests

get = str(input("enter url of the img you want to download : "))

def download_image(url):
    print("downloading ...")
    name = random.randrange(1, 1000)
    full_name = str(name) + ".jpeg"
    r = requests.get(url, stream = True)
    with open (full_name, 'wb') as f:
        f.write(r.content)
    return "download completed and saved as :" + full_name

print(download_image(get))


"""
Formatting text by adjusting where line breaks occur in a paragraph.

"""

sample_text = '''
    The textwrap module can be used to format text for output in
    situations where pretty-printing is desired.  It offers
    programmatic functionality similar to the paragraph wrapping
    or filling features found in many text editors.
    '''

import textwrap

print(textwrap.fill(sample_text, width=50))

# Dedent the text

dedented_text = textwrap.dedent(sample_text)
print("Dedented Text: ")
print(dedented_text)

# Use the indent() function to add consistent prefix text to all of the lines in a string.

dedented_text = textwrap.dedent(sample_text)
wrapped = textwrap.fill(sample_text, width=50)
wrapped += '\n\nSecond paragraph after a blank line.'

final = textwrap.indent(wrapped, ">>> ")

print(final)

# Truncating The Long Text

shortened = textwrap.shorten(sample_text, width=100)
shortened_wrapped = textwrap.fill(shortened, width=50)

print('\nShortened:\n')
print(shortened_wrapped)

Text/textwrap — Formatting Text Paragraphs.py


import sys

def solution(s):
    idx = 0
    num = ''
    nums = []
    operators = []
    while True:
        if idx == len(s):
            nums.append(int(num))
            break

        if s[idx] == '+' or s[idx] == '-':
            nums.append(int(num))
            operators.append(s[idx])
            num = ''

        else : num += s[idx]

        idx += 1

    answer = nums[0]

    o = '+'
    for i in range(len(operators)):
        if operators[i] == '-' :
            o = '-'
            answer -= nums[i + 1]
        elif operators[i] == '+' and o == '-':
            answer -= nums[i + 1]
        else :
            answer += nums[i + 1]

    return answer

if __name__ == '__main__':
    s = sys.stdin.readline()[:-1]

    sys.stdout.write('%d\n'%solution(s))


import os

def save_model_h5(file_model, name):
        model_fn = name.lower().replace(" ", "_")
        root_path = os.path.dirname(os.path.dirname(__file__))
        full_path = os.path.join(root_path, 'static/model-upload', model_fn)

        file_model.save(full_path)
        return model_fn

def init_detection(feature, signal_img, label, proba, index_length, curr_index):
    feature_labels = {
        "rr_interval" : "RR Interval",
        "qt_interval" : "QT Interval",
        "qrs_complex" : "QRS Complex",
        "rr_interval,qt_interval" : "RR Interval & QT Interval",
        "rr_interval,qrs_complex" : "RR Interval & QRS Complex",
        "qt_interval,qrs_complex" : "QT Interval & QRS Complex",
        "rr_interval,qt_interval,qrs_complex" : "RR Interval, QT Interval & QRS Complex"
    }
    sampling_rates = {
        "250" : "250 hz",
        "128" : "128 hz"
    }
    signal = {}
    signal["feature"] = feature
    signal["index_length"] = index_length
    signal["curr_index"] = curr_index
    signal["signal_img"] = signal_img
    signal["predicted_label"] = label
    signal["predicted_proba"] = proba

    return feature_labels, signal, sampling_rates


import cv2
import time
import numpy as np
import pandas as pd

"""
prev = new = ''
cap = cv2.VideoCapture(0)

def getColorName(R,G,B):
    minimum = 10000
    for i in range(len(csv)):
        d = abs(R- int(csv.loc[i,"R"])) + abs(G- int(csv.loc[i,"G"]))+ abs(B- int(csv.loc[i,"B"]))
        if(d<=minimum):
            minimum = d
            cname = csv.loc[i,"color_name"]
    return cname

while True:
    index=["color_name","hex","R","G","B"]
    csv = pd.read_csv('colorsV3.csv', names=index, header=None,encoding='latin-1')
    ret, frame = cap.read()

    cv2.rectangle(frame, (309,229), (329,249), (0,255,0), 1)       #pixel range = x ==> 0-639 and y == 0 - 479
    roi = frame[230:249, 310:329]
    avg1 = np.average(roi, axis=0)
    avg2 = np.average(avg1, axis=0)
    avg2_int = avg2.astype(int)
    avg2_int = avg2_int[::-1]           #reversed for rgb
   # avg2_int_tup = tuple(avg2_int)
    r = avg2_int[0]
    g = avg2_int[1]
    b = avg2_int[2]

    print(r," ",g," ",b )
    time.sleep(1)
    '''new = getColorName(r,g,b)

    if (new != prev):
        print(new)
        prev = new
    frame = cv2.flip(frame,1)

    cv2.imshow('CAMERA',frame)

    if cv2.waitKey(1) & 0xFF == ord('q'):
        break
    '''

cap.release()

cv2.destroyAllWindows()

"""

index=["color_name","R","G","B"]
csv = pd.read_csv('colorsT2.csv', names=index, header=None,encoding='latin-1')
def getColorName(R,G,B):
    minimum = 10000
    for i in range(len(csv)):
        d = abs(R- int(csv.loc[i,"R"])) + abs(G- int(csv.loc[i,"G"]))+ abs(B- int(csv.loc[i,"B"]))
        if(d<=minimum):
            minimum = d
            cname = csv.loc[i,"color_name"]
    return cname

def detect_color(s1,s2,e1,e2):
    #print((e1-s1)*(e2-s2))
    red = 0
    blue = 0
    white = 0
    ph = 0
    count = 0
    for x in range( s1, e1-1,10) :
        for y in range( s2 , e2-1,10) :
            count+=1
            if frameHeight <= x+10 or frameWidth <= y+10:
                break
            roi = img[x:x+10, y:y+10]

            avg1 = np.average(roi, axis=0)
            avg2 = np.average(avg1, axis=0)
            avg2_int = avg2.astype(int)
            avg2_int = avg2_int[::-1]           #reversed for rgb
           # avg2_int_tup = tuple(avg2_int)
            r = avg2_int[0]
            g = avg2_int[1]
            b = avg2_int[2]

            new = getColorName(r,g,b)
            if(new =='Red'):
                red +=1
            elif(new== 'Blue'):
                blue +=1
            elif(new == 'Ph'):
                ph +=1
            else:
                white +=1

    print(red,"   ",blue,"    ",white,"     ",ph,"    ",count)

    if(red > blue and red > 10):
        print("Red detected")
    elif(blue > red and blue> 10 ):
        print("Blue detected")
    elif(ph>30):
        print("Ph paper only")
    else:
        print("no ph paper detected")

frameWidth = 640
frameHeight = 480
cap = cv2.VideoCapture(0)
cap.set(3, frameWidth)
cap.set(4, frameHeight)

def empty(a):
    pass

cv2.namedWindow("Parameters")
cv2.resizeWindow("Parameters",640,240)
cv2.createTrackbar("Threshold1","Parameters",15,255,empty)
cv2.createTrackbar("Threshold2","Parameters",10,255,empty)
cv2.createTrackbar("Area","Parameters",1000,30000,empty)

def stackImages(scale,imgArray):
    rows = len(imgArray)
    cols = len(imgArray[0])
    rowsAvailable = isinstance(imgArray[0], list)
    width = imgArray[0][0].shape[1]
    height = imgArray[0][0].shape[0]
    if rowsAvailable:
        for x in range ( 0, rows):
            for y in range(0, cols):
                if imgArray[x][y].shape[:2] == imgArray[0][0].shape [:2]:
                    imgArray[x][y] = cv2.resize(imgArray[x][y], (0, 0), None, scale, scale)
                else:
                    imgArray[x][y] = cv2.resize(imgArray[x][y], (imgArray[0][0].shape[1], imgArray[0][0].shape[0]), None, scale, scale)
                if len(imgArray[x][y].shape) == 2: imgArray[x][y]= cv2.cvtColor( imgArray[x][y], cv2.COLOR_GRAY2BGR)
        imageBlank = np.zeros((height, width, 3), np.uint8)
        hor = [imageBlank]*rows
        hor_con = [imageBlank]*rows
        for x in range(0, rows):
            hor[x] = np.hstack(imgArray[x])
        ver = np.vstack(hor)
    else:
        for x in range(0, rows):
            if imgArray[x].shape[:2] == imgArray[0].shape[:2]:
                imgArray[x] = cv2.resize(imgArray[x], (0, 0), None, scale, scale)
            else:
                imgArray[x] = cv2.resize(imgArray[x], (imgArray[0].shape[1], imgArray[0].shape[0]), None,scale, scale)
            if len(imgArray[x].shape) == 2: imgArray[x] = cv2.cvtColor(imgArray[x], cv2.COLOR_GRAY2BGR)
        hor= np.hstack(imgArray)
        ver = hor
    return ver

def getContours(img,imgContour):
    contours, hierarchy = cv2.findContours(img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
    for cnt in contours:
        area = cv2.contourArea(cnt)
        areaMin = cv2.getTrackbarPos("Area", "Parameters")
        if area > areaMin:
            cv2.drawContours(imgContour, cnt, -1, (255, 0, 255), 7)
            peri = cv2.arcLength(cnt, True)
            approx = cv2.approxPolyDP(cnt, 0.02 * peri, True)

            #time.sleep(0.1)

            if len(approx)<4 or len(approx)>6:
                break

            print(len(approx))

            x , y , w, h = cv2.boundingRect(approx)
            detect_color(x,y,x+w,y+h)
            x=x+10
            y=y+10
            w=w-20
            h=h-20

            cv2.rectangle(imgContour, (x , y ), (x + w , y + h ), (0, 255, 0), 5)

            cv2.putText(imgContour, "Points: " + str(len(approx)), (x + w + 20, y + 20), cv2.FONT_HERSHEY_COMPLEX, .7,
                        (0, 255, 0), 2)
            cv2.putText(imgContour, "Area: " + str(int(area)), (x + w + 20, y + 45), cv2.FONT_HERSHEY_COMPLEX, 0.7,
                        (0, 255, 0), 2)

            #time.sleep(0.2)

while True:
    success, img = cap.read()
    time.sleep(0.1)
    imgContour = img.copy()
    imgBlur = cv2.GaussianBlur(img, (7, 7), 1)
    imgGray = cv2.cvtColor(imgBlur, cv2.COLOR_BGR2GRAY)
    threshold1 = cv2.getTrackbarPos("Threshold1", "Parameters")
    threshold2 = cv2.getTrackbarPos("Threshold2", "Parameters")
    imgCanny = cv2.Canny(imgGray,threshold1,threshold2)
    kernel = np.ones((5, 5))
    imgDil = cv2.dilate(imgCanny, kernel, iterations=1)
    getContours(imgDil,imgContour)
    imgStack = stackImages(0.8,([img,imgCanny],
                                [imgDil,imgContour]))
    cv2.imshow("Result", imgStack)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

COLOR PICKER/Python test codes/v3_contour/rect_git_1.py


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
:mod:`battery`
==================

Created by hbldh <henrik.blidh@nedomkull.com>
Created on 2016-04-02

"""

from __future__ import division
from __future__ import print_function
from __future__ import absolute_import

import time

from pymetawear.discover import select_device
from pymetawear.client import MetaWearClient

address = select_device()
c = MetaWearClient(str(address), 'pygatt', debug=True)
print("New client created: {0}".format(c))

def battery_callback(data):
    """Handle a battery status tuple."""
    epoch = data[0]
    battery = data[1]
    print("[{0}] Voltage: {1}, Charge: {2}".format(
        epoch, battery[0], battery[1]))

print("Subscribe to battery notifications...")
c.settings.notifications(battery_callback)

time.sleep(1.0)

print("Trigger battery state notification...")
c.settings.read_battery_state()

time.sleep(2.0)

print("Unsubscribe to battery notifications...")
c.settings.notifications(None)

time.sleep(1.0)

c.disconnect()


import csv
import statistics
import numpy as np
import matplotlib.pyplot as plt

with open('activity.csv', 'r') as f:
    data = csv.reader(f)
    mydictWeekend = {}
    mydictWeekdays = {}
    weekendList = ('2012-10-06', '2012-10-07', '2012-10-13', '2012-10-14', '2012-10-20', '2012-10-21', '2012-10-27', '2012-10-28', '2012-11-03', '2012-11-04', '2012-11-10', '2012-11-11', '2012-11-17', '2012-11-18', '2012-11-24', '2012-11-25')

    for row in data:
        if row[1] in weekendList:
            if row[0] != 'NA' and row[0] != 'steps':
                if row[2] not in mydictWeekend:
                    mydictWeekend[row[2]] = [int(row[0])]
                else:
                    mydictWeekend[row[2]].append(int(row[0]))
        else:
            if row[0] != 'NA' and row[0] != 'steps':
                if row[2] not in mydictWeekdays:
                    mydictWeekdays[row[2]] = [int(row[0])]
                else:
                    mydictWeekdays[row[2]].append(int(row[0]))

pertimeWeekend = {}
mean_pertimeWeekend = {}

for key, value in mydictWeekend.items():
    pertimeWeekend[key] = sum(value)
    mean_pertimeWeekend[key] = round(statistics.mean(value),2)

plt.plot(list(mean_pertimeWeekend.keys()), list(mean_pertimeWeekend.values()))
plt.title('Weekends')
plt.xlabel('time')
plt.ylabel('steps each 5 time')
plt.show()

pertimeWeekdays = {}
mean_pertimeWeekdays = {}

for key, value in mydictWeekdays.items():
    pertimeWeekdays[key] = sum(value)
    mean_pertimeWeekdays[key] = round(statistics.mean(value),2)

plt.plot(list(mean_pertimeWeekdays.keys()), list(mean_pertimeWeekdays.values()))
plt.title('Weekdays')
plt.xlabel('time')
plt.ylabel('steps each 5 time')
plt.show()


#!/usr/bin/env python3
# =============================================================================
'''
per SUBJECT: Read the curated events and see whether there is any modulation
between conditions
'''
# =============================================================================
#~~~~~~~~~~
# modules
import argparse
from utils.generic import load_args
import numpy as np
import pandas as pd
import seaborn as sns
from mlxtend.evaluate import permutation_test
from scipy.stats import sem
import matplotlib.pyplot as plt
#~~~~~~~~~~
# =============================================================================
# ARGUMENTS AND PARAMETERS
# =============================================================================
parser = argparse.ArgumentParser(description='events classification per channel')
parser.add_argument('-ro', '--root_path', default='/media/christos/Transcend1/',
                    help='root path')
parser.add_argument('-s', '--subject', default='sub-05', help='subject')
parser.add_argument('-runs', '--runs', default='all', help='single/all')
parser.add_argument('-ru', '--run', default='run_1', help='run')
# get arguments
args = parser.parse_args()
# Update args
args=load_args.SetPaths(args).check_paths(args)
#~~~~~~~~~~
data_path=args.join(args.project_path,'Data','derivatives',args.subject)
#~~~~~~~~~~
target_events='ripples'

#~~~~~~~~~~
# Collect events of interest per subject for all runs.
events=[f for f in args.see(data_path) if target_events in f]
# Events sorted per channel
events.sort(key = lambda x: x.split('_')[1])
n_channels=len(events)

# load triggers
triggers=args.join(args.output_path,'all_runs')
trigger_timing=args.read(args.join(triggers,args.see(triggers)[0]), sep='\t')

event_types=trigger_timing.eventtype.unique().tolist()

# =============================================================================
# TRIGGER TIMING PER CONDITION
# =============================================================================
wordlist=trigger_timing[trigger_timing.trialtype=='wordlist']
sentloc=trigger_timing[trigger_timing.trialtype=='sentloc']
sentmain=trigger_timing[trigger_timing.trialtype=='sentmain']
jab=trigger_timing[trigger_timing.trialtype=='jab']

condition_df=[wordlist, sentloc, sentmain, jab]
conditions=['wordlist', 'sentloc', 'sentmain', 'jab']
# =============================================================================
# SENTENCE DURATION (FIRST WORD ONSET TO LAST WORD ONSET)
# =============================================================================
def count_events_while_sentence_on_screen(condition, events_per_channel):
    '''
    From fixation to last word offset per CHANNEL per PATIENT
    '''
    start=condition[condition.eventtype=='starfix']
    end=condition[condition.eventtype=='lastwordoffset']
    # check if they have the same len
    if len(start)!=len(end):
        raise AssertionError('Different number of triggers!')
    # get duration of sentence per trials
    duration_per_trial=np.vstack((start.time, end.time))
    events=[]
    # loop through trials

    for event in events_per_channel['time (s)']:
        for trial in range(0, len(start)):
            if (event>duration_per_trial[0][trial] and event<duration_per_trial[1][trial]):
                events.append(event)

    return len(events)

counts_per_condition_per_channel=[]

for idx,condition in enumerate(condition_df):
    print(f'{conditions[idx]}')
    events_per_condition=[]
    for channel in range(1,n_channels):
        # load events per channel
        events_per_channel=args.read(args.join(data_path,events[channel-1]), sep='\t')

        curr_events=count_events_while_sentence_on_screen(condition, events_per_channel)
        events_per_condition.append(curr_events)

    counts_per_condition_per_channel.append(events_per_condition)

def plot_distribution_of_events(df):
    if 'wordlist' in df.columns.tolist():
        colors=['blue','red']
        title=r'$\mathcal{Localizer \quad Block}$'
    else:
        colors=['green','black']
        title=r'$\mathcal{Main \quad Block}$'

    df.plot(color=colors)
    sns.despine(trim=True)
    plt.xlabel('#Channels')
    plt.ylabel('#Events')
    plt.title(title, fontsize=14)

# =============================================================================
# LOCALIZER BLOCK
# =============================================================================
wordlist_list=counts_per_condition_per_channel[0]
sentloc_list =counts_per_condition_per_channel[1]

df_localizer = pd.DataFrame(columns=['wordlist','sentloc'])
df_localizer.wordlist=wordlist_list
df_localizer.sentloc =sentloc_list

# Stats on the localizer block
p_value = permutation_test(wordlist_list, sentloc_list,
                           method='approximate',
                           num_rounds=10000,
                           seed=0)
print(p_value)

# =============================================================================
# MAIN BLOCK
# =============================================================================
sentmain_list=counts_per_condition_per_channel[2]
jab_list     =counts_per_condition_per_channel[3]

df_main = pd.DataFrame(columns=['sentmain','jab'])
df_main.sentmain=sentmain_list
df_main.jab     =jab_list

# Stats on the main block
p_value = permutation_test(sentmain_list, jab_list,
                           method='approximate',
                           num_rounds=10000,
                           seed=42)
print(p_value)


import json

import requests

groupUrl = "https://gitlab.com/api/v4/groups/{group}/projects?private_token={token}"
labelsUrl = "https://gitlab.com/api/v4/projects/{project_id}/labels?private_token={token}"

def main():
    print("Enter GitLab Private Token:")
    key = input()
    key = key.strip()

    print("Enter GitLab Group Name:")
    group = input()

    if (not group):
        print("group should not be empty")
        return

    group = group.replace("/","%2F")
    url = groupUrl.replace("{group}", group)

    url = url.replace("{token}", key)

    answer = requests.get(url)

    if (answer.status_code != 200):
        if('message' in answer):
            print(answer.json()['message'] + " for group: " + group)
        else:
            print(answer.json())
        return

    projects = answer.json()

    ids = []
    for count, project in enumerate(projects):
        name = "%2d %50s: %8d" % (count + 1, project['name'], project['id'])
        ids.append(project['id'])
        print(name)

    if (len(ids) < 1):
        print("No Repository found for group: " + group)
        return

    print("Choose Repository:")
    choosedId = int(input()) - 1
    repositoryId = ids[choosedId]
    url = labelsUrl.replace('{project_id}', str(repositoryId))
    url = url.replace("{token}", key)
    answer = requests.get(url)
    if (answer.status_code == 200):
        with open('labels.json') as file:
            labels = json.load(file)
        for label in labels:
            answer = requests.post(url, {"name": label[0], "color": label[1]}).json()
            if ('name' in answer or 'color' in answer):
                print("Label %10s with color %10s created" % (answer['name'], answer['color']))
            elif ('message' in answer):
                print(answer['message'] + ": " + label[0])
            else:
                print(answer)

if __name__ == "__main__":
    main()


"""
Initialy populate an empty django database with the required groups, and possibly dummy data for testing.

"""
import argparse
import os
import sys

import django

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Populate the database with initial values")
    parser.add_argument('--mode', nargs='?', const=1, type=str, default='debug', help='debug/production')
    parser.add_argument('--create-dummy-data', dest='createDummyData', action='store_true',
                        help='if activated dummy data is generated')
    parser.set_defaults(createDummyData=False)
    DUMMY, MODE = parser.parse_args().createDummyData, parser.parse_args().mode

    if MODE not in ["debug", "production"]:
        print("Please use --mode debug or --mode production")
        sys.exit(1)
else:
    MODE = 'debug'
    DUMMY = False

if MODE == 'debug':
    os.environ['DJANGO_SETTINGS_MODULE'] = 'BepMarketplace.settings_development'
elif MODE == 'production':
    os.environ['DJANGO_SETTINGS_MODULE'] = 'BepMarketplace.settings'

django.setup()

from django.contrib.auth.models import Group, User
from index.models import Track, UserMeta
from proposals.models import Proposal
from students.models import Application
from support.models import CapacityGroup
from timeline.models import TimePhase, TimeSlot
import random
from timeline.utils import get_timeslot
from datetime import date
from django.conf import settings
from proposals.utils import get_all_proposals

tracks = [
    ("Automotive", "AU"),
    ("Smart&Sustainable Society", "SSS"),
    ("Connected World", "CW"),
    ("Care&Cure", "C&C"),
]

Groups = (
    ("EES", "Electrical Energy Systems"),
    ("ECO", "Electro-Optical Communications"),
    ("EPE", "Electromechanics and Power Electronics"),
    ("ES", "Electronic Systems"),
    ("IC", "Integrated Circuits"),
    ("CS", "Control Systems"),
    ("SPS", "Signal Processing Systems"),
    ("PHI", "Photonic Integration"),
    ("EM", "Electromagnetics")
)
for group in Groups:
    c, created = CapacityGroup.objects.get_or_create(ShortName=group[0], FullName=group[1])
    if created:
        print("creating {}-{}".format(group[0], group[1]))

print("populating tables with production values")

# setup tracks
trackobjs = []
for track in tracks:
    t, created = Track.objects.get_or_create(Name=track[0])
    if created:
        print("creating track {}".format(track[0]))
        t.ShortName = track[1]
        t.save()
    trackobjs.append(t)

# setup all groups with permissions
g, created = Group.objects.get_or_create(name='type1staff')
if created:
    print("creating type1staff")
    # perms = Permission.objects.filter(content_type=ContentType.objects.get_for_model(Proposal))
    # for p in perms:
    #     g.permissions.add(p)
    # #perms = Permission.objects.filter(content_type=ContentType.objects.get_for_model(CategoryResult))
    # g.permissions.add(perms[0])
    # g.permissions.add(perms[1])
    g.save()

g, created = Group.objects.get_or_create(name='type2staff')
if created:
    print("creating type2staff")
    # #perms = Permission.objects.filter(content_type=ContentType.objects.get_for_model(Proposal))
    # g.permissions.add(perms[0])
    # g.permissions.add(perms[1])
    # g.permissions.add(perms[2])
    g.save()

g, created = Group.objects.get_or_create(name='type2staffunverified')
if created:
    print("creating type2staffunverified")
    # perms = Permission.objects.filter(content_type=ContentType.objects.get_for_model(Proposal))
    # g.permissions.add(perms[0])
    # g.permissions.add(perms[1])
    # g.permissions.add(perms[2])
    g.save()

g, created = Group.objects.get_or_create(name='type3staff')
if created:
    print("creating type3staff")
    # for model in all_models:
    #     perms = Permission.objects.filter(content_type=ContentType.objects.get_for_model(model))
    #     for p in perms:
    #         g.permissions.add(p)
    g.save()

g, created = Group.objects.get_or_create(name='type4staff')
if created:
    print("creating type4staff")

g, created = Group.objects.get_or_create(name='type5staff')
g.save()
if created:
    print("creating type5staff")

g, created = Group.objects.get_or_create(name='type6staff')
g.save()
if created:
    print("creating type6staff")

if not DUMMY:
    # End of non dummy generation
    sys.exit(0)

# generate test users, proposals and applictions for debug purpose
print("populating tables with debug objects")

NUMPROFS = 20
NUMPHDS = 30
NUMSTDS = 50
NUMPROPOSALS = 75

type1staff = Group.objects.get(name="type1staff")
type2staff = Group.objects.get(name="type2staff")
type3staff = Group.objects.get(name="type3staff")

profs = list(type1staff.user_set.all())
phds = list(type2staff.user_set.all())

print("creating timeslot and timephase")
ts = TimeSlot(Name="semester", Begin=date.today(), End=date.today().replace(year=date.today().year+1))
ts.save()
tp = TimePhase(Description=4, TimeSlot=ts, Begin=date.today(), End=date.today().replace(month=date.today().month + 3))
tp.save()

print("creating {} professors".format(NUMPROFS))
for i in range(0, NUMPROFS):
    try:
        prof = User.objects.create_user('professor{}'.format(i), 'professor{}@tue.nl'.format(i), 'marketplace')
        prof.first_name = "professor"
        prof.last_name = str(i)
        prof.groups.add(type1staff)
        prof.save()
        profs.append(prof)
    except:
        print(str(i) + " not created")
    prof = User.objects.get(username='professor{}'.format(i))
    try:
        mta = UserMeta()
        mta.Fullname = "Professor-" + str(i)
        mta.Studentnumber = 0
        mta.User = prof
        mta.save()
        print("usermeta prof" + str(i))
    except:
        print(str(i) + " prof usermeta not created")

print("creating {} phders".format(NUMPHDS))
for i in range(0, NUMPHDS):
    try:
        phd = User.objects.create_user('phd{}'.format(i), 'phd{}@tue.nl'.format(i), 'marketplace')
        phd.first_name = "phd"
        phd.last_name = str(i)
        phd.groups.add(type2staff)
        phd.save()
        phds.append(phd)
    except:
        print(str(i) + " not created")
    phd = User.objects.get(username='phd{}'.format(i))
    try:
        mta = UserMeta()
        mta.Fullname = "phd-" + str(i)
        mta.Studentnumber = 0
        mta.User = phd
        mta.save()
        print("usermeta phd" + str(i))
    except:
        print(str(i) + " phd usermeta not created")

stds = []
print("creating {} students".format(NUMSTDS))
for i in range(0, NUMSTDS):
    try:
        std = User.objects.create_user('std{}'.format(i), 'std{}@tue.nl'.format(i), 'marketplace')
        std.first_name = "std"
        std.last_name = str(i)
        std.save()
        stds.append(std)
    except:
        print(str(i) + " not created")
    std = User.objects.get(username='std{}'.format(i))
    try:
        mta = UserMeta()
        mta.Fullname = "student-" + str(i)
        mta.Studentnumber = str(i) + str(i) + str(i) + str(i) + str(i)
        mta.User = std
        mta.EnrolledBEP = True
        mta.save()
        mta.TimeSlot.add(ts)
        mta.save()
        print("usermeta" + str(i))
    except:
        print(str(i) + " usermeta not created")

print("creating the support user")
try:
    supp = User.objects.create_user('janedoe', 'j.doe.1@tue.nl', 'marketplace')
    supp.first_name = "Jane"
    supp.last_name = "Doe"
    supp.groups.add(type3staff)
    supp.save()
except:
    print("no type3 created")
supp = User.objects.get(username='janedoe')
try:
    mta = UserMeta()
    mta.Fullname = 'Jane Doe'
    mta.User = supp
    mta.save()
except:
    print('Supp usermeta not created')

print("creating track Head and assigning him")
try:
    trackh = User.objects.create_user('johndoe', 'j.doe.2@tue.nl', 'marketplace')
    trackh.first_name = "John"
    trackh.last_name = "Doe"
    trackh.groups.add(type1staff)
    trackh.save()
except:
    print("no trackhead created")
supp = User.objects.get(username='johndoe')
try:
    mta = UserMeta()
    mta.Fullname = 'John Doe'
    mta.User = supp
    mta.save()
except:
    print('Track usermeta not created')

for track in tracks:
    try:
        t = Track.objects.get(Name=track[0])
        t.Head = trackh
        t.save()
    except:
        print("no track created")

def flip(x):
    """

    :param x:
    :return:
    """
    return True if random.random() < x else False

# create usermeta's
for user in User.objects.all():
    try:
        meta = user.usermeta
    except UserMeta.DoesNotExist:
        meta = UserMeta(User=user)
        meta.save()
    meta.TimeSlot.add(ts)
    meta.save()

print("creating proposals")
for i in range(0, NUMPROPOSALS):
    try:
        p = Proposal()
        p.Title = "project{}".format(i)
        p.ResponsibleStaff = random.choice(profs)
        p.Group = CapacityGroup.objects.order_by('?').first()
        p.NumStudentsMin = random.randint(1, 2)
        p.NumStudentsMax = random.randint(p.NumStudentsMin, 5)
        p.GeneralDescription = "stuff about project. autogenerated with number {}".format(i)
        p.StudentsTaskDescription = "students have to do stuff woop woop"
        p.Track = random.choice(trackobjs)
        # p.Private = None
        # p.Image = random.choice(["niels.png", "crying.png"])
        # p.Status = random.choice(Proposal.StatusOptions)[0]
        p.Status = 4
        p.TimeSlot = get_timeslot()
        p.save()  # save already to activate the manytomany field of assistants
        numphd = random.choice([1, 2])
        ass1 = random.choice(phds)
        p.Assistants.add(ass1)
        if numphd == 2:
            phds.remove(ass1)
            ass2 = random.choice(phds)
            p.Assistants.add(ass2)
            phds.append(ass1)
        p.save()
        print('{} created'.format(p))
    except:
        print(str(i) + "th proposal not created")

print("generating applications")
secure_random = random.SystemRandom()
projects = get_all_proposals()
for i in range(0, NUMSTDS):
    for c in range(1, settings.MAX_NUM_APPLICATIONS+1):
        app = Application(Priority=c,
                          Proposal=secure_random.choice(projects),
                          Student=User.objects.get(username='std{}'.format(i)))
        app.save()


# -*- coding: utf-8 -*-
# Time: 2019/04/01 01:18
# Author: caiongbo
# File: main.py
# 功能：
'''
以中国新闻网社会新闻板块为抓取对象
http://www.chinanews.com/society.shtml
当前抓取了3月份所有数据新闻数据，后期进行数据分析
可以在news_spider.py中设置任意的时间段
'''

from multiprocessing import Pool
from news_spider import news, get_res, get_all_links, item_info, url_lists

def all_links():
    '''
    先从网页爬取所有的链接数据存入数据库，
    然后从数据库里读取所有的链接
    :return: 以所有链接的列表形式
    '''
    get_all_links()
    allLinks = []
    for data in url_lists.find():
        allLinks.append(data['links'])
    return  allLinks

if __name__ == '__main__':
    pool = Pool()  # 创建多线程池
    pool.map(get_res, all_links())


import numpy as np
import torch
import torch.utils.data as data_utils
from sklearn.preprocessing import minmax_scale
from torchvision import transforms

from src.data.Audio.CustomTensor import CustomTensorDataset
from src.data.Audio.TorchLoader import TorchLoader

def dataloader(args, log):
    audio = TorchLoader(path_source=args.source_folder, log=log, sample_rate=args.sample_rate)
    x_train, train_y, x_val, val_y, x_test, test_y = audio.load_data(
        source_path=args.source_folder).load_files().save_data(
        destination_path=args.source_folder).get_data()

    x_train = minmax_scale(x_train)
    x_val = minmax_scale(x_val)
    x_test = minmax_scale(x_test)

    x_train = np.reshape(x_train, (-1, args.in_channels, args.x, args.y))
    x_val = np.reshape(x_val, (-1, args.in_channels, args.x, args.y))
    x_test = np.reshape(x_test, (-1, args.in_channels, args.x, args.y))

    class PadNumpy(object):

        def __init__(self, type_padding):
            self._type_padding = type_padding

        def __call__(self, sample):
            real_shape = sample.shape
            if real_shape[2] == 49:
                sample = np.pad(sample, ((0, 0), (0, 0), (8, 7)), self._type_padding)
            if real_shape[2] == 40:
                sample = np.pad(sample, ((0, 0), (0, 0), (12, 12)), self._type_padding)
            if real_shape[1] == 25:
                sample = np.pad(sample, ((0, 0), (20, 19), (0, 0)), self._type_padding)
            if real_shape[2] == 99:
                sample = np.pad(sample, ((0, 0), (0, 0), (15, 14)), self._type_padding)
            if real_shape[2] == 80:
                sample = np.pad(sample, ((0, 0), (0, 0), (24, 24)), self._type_padding)
            if real_shape[1] == 50:
                sample = np.pad(sample, ((0, 0), (39, 39), (0, 0)), self._type_padding)
            return sample

    data_transform_pad = transforms.Compose([
        PadNumpy("constant"),
    ])

    train_dataset = CustomTensorDataset(torch.from_numpy(x_train), torch.from_numpy(x_train),
                                        transform=data_transform_pad)
    train_loader = data_utils.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=2)

    validation_dataset = CustomTensorDataset(torch.from_numpy(x_val), torch.from_numpy(x_val),
                                             transform=data_transform_pad)
    val_loader = data_utils.DataLoader(validation_dataset, batch_size=args.batch_size, shuffle=False, num_workers=2)

    test_dataset = CustomTensorDataset(torch.from_numpy(x_test), torch.from_numpy(x_test), transform=data_transform_pad)
    test_loader = data_utils.DataLoader(test_dataset, batch_size=args.batch_size, shuffle=False, num_workers=2)

    return train_loader, val_loader, test_loader


from wpaudit.providers.azure.facade.base import AzureFacade
from wpaudit.providers.azure.resources.base import AzureResources

class Roles(AzureResources):

    def __init__(self, facade: AzureFacade, subscription_id: str):
        super().__init__(facade)
        self.subscription_id = subscription_id

    async def fetch_all(self):
        for raw_role in await self.facade.rbac.get_roles(self.subscription_id):
            id, role = self._parse_role(raw_role)
            self[id] = role

    def _parse_role(self, raw_role):
        role_dict = {}
        role_dict['id'] = raw_role.name
        role_dict['name'] = raw_role.role_name
        role_dict['type'] = raw_role.type
        role_dict['description'] = raw_role.description
        role_dict['role_type'] = raw_role.role_type
        role_dict['permissions'] = raw_role.permissions
        role_dict['assignable_scopes'] = raw_role.assignable_scopes
        role_dict['additional_properties'] = raw_role.additional_properties
        role_dict['assignments_count'] = 0
        role_dict['assignments'] = {'users': [],
                                    'groups': [],
                                    'service_principals': []}  # this will be filled in `finalize()`
        return role_dict['id'], role_dict


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"rest - Functions for implementing a RESTful API for the TesteWeb."

__author__ = 'Vitor Henrique L. Mesquita <vitor@vitorhenrique.me>'

import json
import logging
import os
import sys

sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'persistence')))

from database import db_session, DB_PATH, DB_NAME, init_db
from flask import Flask, jsonify, request, jsonify, make_response
from flask_cors import CORS, cross_origin
from model import Product

COMPONENT = None
REST = '/rest'
app = Flask(__name__)
CORS(app)
def urljoin(*args):
    """Joins the arguments in the likes of os.path.join.
        :param args: The arguments to be joined.
    """
    return '/'.join(s for s in args if s.strip() != '')

class ApplicationComponent(object):
    def __init__(self):
        pass
    def insert_product(self, session, data):
        try:
            transformed_data = {}
            for x in data:
                transformed_data[x['name']] = x['value']
            product = Product(transformed_data['cod_mer'],
                              transformed_data['type_mer'],
                              transformed_data['name_mer'],
                              transformed_data['amt'],
                              transformed_data['price'],
                              transformed_data['type_b'])
            session.add(product)
            session.flush
            session.commit()
            return True
        except ValueError:
            return ValueError
    def get_all_product(self, session):
        return session.query(Product).all()

@app.route(urljoin(REST, "product/"), methods=['POST'])
def new_product():
    try:
        data = request.get_json()
        if COMPONENT.insert_product(db_session, data):
            return jsonify(True)
        else:
            return make_response(jsonify(False), 404)
    except:
            return make_response(jsonify(False), 404)
@app.route(urljoin(REST, "products/"), methods=['GET'])
def get_products():
    rows ={"data":[]}
    result = COMPONENT.get_all_product(db_session)
    for x in result:
        dictret = dict(x.__dict__)
        dictret.pop('_sa_instance_state', None)
        rows['data'].append(dictret)
    return jsonify(rows)

def main():
    database = os.path.join(DB_PATH, DB_NAME)
    if not os.path.exists(database):
        init_db()
    global COMPONENT
    COMPONENT = ApplicationComponent()
    app.run(host='0.0.0.0',port=8080, debug=True, use_reloader=False, threaded=True)

if __name__ == '__main__':
    main()


# https://uva.onlinejudge.org/index.php?option=com_onlinejudge&Itemid=8&category=124&page=show_problem&problem=423

###
### Config data
###

import os
from math import floor, ceil, log

example_test= """1

3 1 2
32.0 54.7 -2"""

exec(open("../../../_auxiliary/input_iterators.py").read())

line = input_iter()
#line = string_iter()

fobject = None
pend = None

check_with_oracle = False

###
### Run
###
test_cases = int(next(line))

for tc in range(test_cases):
	next(line)

	permutations = list(map(lambda x: int(x) - 1,next(line).split()))
	floats =  next(line).split()
	copyfloats = [x for x in floats]

	for p in range(len(permutations)):
		floats[permutations[p]]   = copyfloats[p]

	if tc != 0:
		print("")
	for x in floats:
		print(x)


import os

import fabrika
import yaml
from fabric import api as fabric_api, operations
from fabric.decorators import roles
from fabrika import deploy_tools
from fabrika.tasks.docker import SetupLoadBalancerBehindGateway

import configuration
from fabfile.app_configuration import configured_for

CONTAINER_HOSTNAME = 'node'
DEPENDENCIES = 'dependencies'
DNS_SERVER_2 = 'dns2'
DNS_SERVER_1 = 'dns1'
LOAD_BALANCER_NODES = 'nodes'
LOAD_BALANCER_PORT = 'load_balancer_port'
LOAD_BALANCER_HOST = 'load_balancer_host'
LOAD_BALANCER_MODE = 'mode'
NODE_PORT = 'node_port'
ENABLE_NSCD = 'enable_nscd'

FABRIC_ROLE_SEED_SANITY_TEST_FILE = 'seed_sanity_test_file'
FABRIC_ROLE_DOCKER_HOST = 'docker_host'
FABRIC_ROLE_LOAD_BALANCER = 'lb_config_node'
FABRIC_ROLE_ELASTIC_SEARCH_LOAD_BALANCER = 'elastic_search_lb_config_node'

NSCD_SOCKET_SOURCE = '/var/run/nscd'
NSCD_SOCKET_TARGET = '/tmp/nscd-extern'

load_balancer_task = SetupLoadBalancerBehindGateway()

@fabric_api.task()
def assert_deploying_newer_image(environment, full_image_name):
    _initialize_environment(environment)
    fabric_api.execute(deploy_tools.assert_deploying_newer_image, full_image_name, role=FABRIC_ROLE_DOCKER_HOST)

def _initialize_environment(environment):
    _parse_deployment_yaml(environment)
    env_properties = """
    roledefs  : {roledefs}
    hosts     : {hosts}
    properties: {properties}
    """
    print(env_properties.format(roledefs=fabric_api.env.roledefs, hosts=fabric_api.env.hosts,
                                properties=fabric_api.env.host_role_properties))

@fabric_api.task()
def full_deploy(environment, full_image_name, configuration_dir, run_migrate=False, enable_nscd=None):
    _initialize_environment(environment)
    fabric_api.execute(deploy_docker_image, environment, full_image_name, configuration_dir, enable_nscd=enable_nscd)
    fabric_api.execute(setup_elastic_search_service_load_balancer)
    fabric_api.execute(setup_lls_service_load_balancer)
    fabric_api.execute(seed_test_file)

def _associate_host_to_a_role(host, role):
    if role not in fabric_api.env.roledefs:
        fabric_api.env.roledefs[role] = []
    fabric_api.env.roledefs[role].append(host)

def _parse_deployment_yaml(environment):
    fabric_api.env.host_role_properties = {}
    deploy_cfg_file = os.path.join(os.getcwd(), "fabfile/deploy/{}.yml".format(environment))
    yaml_config = yaml.load(open(deploy_cfg_file, 'r'))
    for host in yaml_config:
        for role in yaml_config[host]['roles']:
            _associate_host_to_a_role(host, role)
            fabric_api.env.host_role_properties[(host, role)] = yaml_config[host]['roles'][role]

def set_load_balancer(role):
    load_balancer_config = fabric_api.env.host_role_properties[(fabric_api.env.host, role)]
    fabric_api.execute(load_balancer_task,
                       load_balancer_config[LOAD_BALANCER_HOST],
                       load_balancer_config[LOAD_BALANCER_PORT],
                       load_balancer_config[LOAD_BALANCER_NODES],
                       load_balancer_config[NODE_PORT],
                       load_balancer_config.get(LOAD_BALANCER_MODE))

@roles(FABRIC_ROLE_SEED_SANITY_TEST_FILE)
def seed_test_file():
    print(">>>>>>> seeding sanity test file on host {}".format(fabric_api.env.host_string))
    local_csv_file = os.path.join(os.getcwd(), "tests/samples/normal.csv")
    remote_sanity_test_file = os.path.join(configuration.data.VOLUME_MAPPINGS_FILE_UPLOAD_SOURCE, 'normal.csv')
    operations.put(local_csv_file, remote_sanity_test_file)

@roles(FABRIC_ROLE_LOAD_BALANCER)
def setup_lls_service_load_balancer():
    print(">>>>>>> setting up load balancer on host {}".format(fabric_api.env.host_string))
    set_load_balancer(FABRIC_ROLE_LOAD_BALANCER)

@roles(FABRIC_ROLE_ELASTIC_SEARCH_LOAD_BALANCER)
def setup_elastic_search_service_load_balancer():
    print(">>>>>>> setting up search service load balancer on host {}".format(fabric_api.env.host_string))
    set_load_balancer(FABRIC_ROLE_ELASTIC_SEARCH_LOAD_BALANCER)

def _process_nscd_mapping(volume_mappings, configuration, override):
    option_overridden = override is not None

    if option_overridden:
        if override:
            volume_mappings[NSCD_SOCKET_SOURCE] = NSCD_SOCKET_TARGET
        return volume_mappings

    if configuration.get(ENABLE_NSCD):
        volume_mappings[NSCD_SOCKET_SOURCE] = NSCD_SOCKET_TARGET
    return volume_mappings

@roles(FABRIC_ROLE_DOCKER_HOST)
def deploy_docker_image(environment, fully_qualified_image_name, app_container_config_path, enable_nscd=None,
                        stop_existing=True):
    print(">>>>>>> deploy_docker_image on host {}".format(fabric_api.env.host))
    node_deploy_configuration = fabric_api.env.host_role_properties[(fabric_api.env.host, FABRIC_ROLE_DOCKER_HOST)]

    volume_mappings = _process_nscd_mapping({}, node_deploy_configuration, enable_nscd)

    with configured_for(environment):
        file_upload_source = configuration.data.VOLUME_MAPPINGS_FILE_UPLOAD_SOURCE
        file_upload_target = configuration.data.VOLUME_MAPPINGS_FILE_UPLOAD_TARGET
        volume_mappings[file_upload_source] = file_upload_target

    deploy_task = fabrika.tasks.docker.DeployTask('list_loading_service', volume_mappings=volume_mappings)
    fabric_api.execute(deploy_task,
                       fully_qualified_image_name,
                       node_deploy_configuration.get(DEPENDENCIES),
                       node_deploy_configuration.get(DNS_SERVER_1),
                       node_deploy_configuration.get(DNS_SERVER_2),
                       node_deploy_configuration.get(CONTAINER_HOSTNAME),
                       app_container_config_path,
                       stop_existing=stop_existing)


import socket
import json
import sys
import random
import time
from goodmovesonly import good_moves

def receiveJSON(socket):
    fullreceive = False
    message = ''
    while not fullreceive:
        message += socket.recv(512).decode('utf8')
        try:
            data = json.loads(message)
            fullreceive = True
        except json.JSONDecodeError:
            pass
    return data

def sendJSON(socket, data):
    data = json.dumps(data).encode('utf8')
    totalsend = 0
    while totalsend < len(data):
        sent = socket.send(data[totalsend:])
        totalsend += sent

def subscribe(port,name):
    port = int(port)
    message = {"request": "subscribe","port": port,"name": name,"matricules": ["19022", "18164"]}
    s = socket.socket()
    s.connect(('localhost',3000))
    sendJSON(s, message)
    s.close()
    #boucle de réception de requêtes
    listening = True
    while listening == True:
        s = socket.socket()
        s.bind(('0.0.0.0', port))
        s.listen()
        client, address = s.accept()
        serverrequest = receiveJSON(client)
        if serverrequest['request'] == 'ping':
            sendJSON(client, {'response': 'pong'})
            listening = False

def SENDmove(server_request):
    # 'current': 0 -> black, 'current': 1 -> white
    playercolors = ['B','W']
    playerindice = server_request['state']['current']
    moves = good_moves(server_request['state']['board'],playercolors[playerindice])
    move = random.choice(moves)
    marbles = []
    for i in move[1]:
        if type(i) is tuple:
            marbles.append(list(i))
        else:
            direction = i
    moverequest = {"marbles": marbles,"direction": direction}
    return {"response": "move","move": moverequest}

defaultport = '5000'
defaultname = 'Random AI'
if len(sys.argv) == 2:
    try:
        isinteger = int(sys.argv[1])
        port = sys.argv[1]
        name = defaultname
    except ValueError:
        port = defaultport
        name = sys.argv[1]
elif len(sys.argv) == 3:
    port = sys.argv[1]
    name = sys.argv[2]
elif len(sys.argv) > 3:
    print('Invalid input')
    sys.exit()
else:
    port = defaultport
    name = defaultname
subscribe(port, name)
listening = True
while listening == True:
        s = socket.socket()
        s.bind(('0.0.0.0', int(port)))
        s.listen()
        client, address = s.accept()
        serverrequest = receiveJSON(client)
        print(serverrequest['request'])
        if serverrequest['request'] == 'ping':
            sendJSON(client, {'response': 'pong'})
        if serverrequest['request'] == 'play':
            start = time.time()
            sendJSON(client, SENDmove(serverrequest))
            print('Temps de réponse : {} s'.format(time.time()-start))


#@HEADER
# ************************************************************************
#
#                        Torchbraid v. 0.1
#
# Copyright 2020 National Technology & Engineering Solutions of Sandia, LLC
# (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S.
# Government retains certain rights in this software.
#
# Torchbraid is licensed under 3-clause BSD terms of use:
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# 3. Neither the name National Technology & Engineering Solutions of Sandia,
# LLC nor the names of the contributors may be used to endorse or promote
# products derived from this software without specific prior written permission.
#
# Questions? Contact Eric C. Cyr (eccyr@sandia.gov)
#
# ************************************************************************
#@HEADER

import torch
import torch.nn as nn
import torch.nn.functional as F
import unittest
import sys
import numpy as np
import statistics as stats

import torchbraid

import faulthandler
faulthandler.enable()

from mpi4py import MPI

class LSTMBlock(nn.Module):
  def __init__(self, input_size, hidden_size, num_layers):
    super(LSTMBlock, self).__init__()
    self.hidden_size = hidden_size
    self.num_layers = num_layers

    torch.manual_seed(20)
    lstm_cells = num_layers*[None]
    lstm_cells[0] = nn.LSTMCell(input_size, hidden_size)
    for i in range(num_layers-1):
      lstm_cells[i+1] = nn.LSTMCell(hidden_size, hidden_size)

    self.lstm_cells = nn.ModuleList(lstm_cells)

  def forward(self, x, h_prev, c_prev):
    h_cur = h_prev[0]
    c_cur = c_prev[0]
    x_cur = x

    hn = self.num_layers*[None]
    cn = self.num_layers*[None]
    for i in range(self.num_layers):
      hn[i], cn[i] = self.lstm_cells[i](x_cur, (h_prev[i], c_prev[i]))
      x_cur = hn[i]

    return (torch.stack(hn), torch.stack(cn))

def RNN_build_block_with_dim(input_size, hidden_size, num_layers):
  b = LSTMBlock(input_size, hidden_size, num_layers) # channels = hidden_size
  return b

def preprocess_input_data_serial_test(num_blocks, num_batch, batch_size, channels, sequence_length, input_size):
  torch.manual_seed(20)
  x = torch.randn(num_batch,batch_size,channels,sequence_length,input_size)

  data_all = []
  x_block_all = []
  for i in range(len(x)):
    image = x[i].reshape(-1, sequence_length, input_size)
    images_split = torch.chunk(image, num_blocks, dim=1)
    seq_split = []
    for blk in images_split:
      seq_split.append(blk)
    x_block_all.append(seq_split)
    data_all.append(image)

  return data_all, x_block_all

def preprocess_distribute_input_data_parallel(rank,num_procs,num_batch,batch_size,channels,sequence_length,input_size,comm):
  if rank == 0:
    torch.manual_seed(20)
    x = torch.randn(num_batch,batch_size,channels,sequence_length,input_size)

    # x_block_all[total_images][total_blocks]
    x_block_all = []
    for i in range(len(x)):
      image = x[i].reshape(-1,sequence_length,input_size)
      data_split = torch.chunk(image, num_procs, dim=1)
      seq_split = []
      for blk in data_split:
        seq_split.append(blk)
      x_block_all.append(seq_split)

    x_block = []
    for image_id in range(len(x_block_all)):
      x_block.append(x_block_all[image_id][rank])

    for block_id in range(1,num_procs):
      x_block_tmp = []
      for image_id in range(len(x_block_all)):
        x_block_tmp.append(x_block_all[image_id][block_id])
      comm.send(x_block_tmp,dest=block_id,tag=20)

    return x_block

  else:
    x_block = comm.recv(source=0,tag=20)

    return x_block

class TestRNNLayerParallel(unittest.TestCase):
  def test_forward_exact(self):
    self.forwardProp(max_levels=1,max_iters=1)

  def test_forward_approx(self):
    self.forwardProp(max_levels=3,max_iters=20)

  def test_backward_exact(self):
    self.backwardProp()

  def test_backward_approx(self):
    self.backwardProp(max_levels=3,max_iters=12,sequence_length=27,tol=1e-5)

  def test_backward_lstm(self):
    if MPI.COMM_WORLD.Get_size()==1:
      self.backwardProp_lstm()

  def copyParameterGradToRoot(self,m):
    comm     = m.getMPIComm()
    my_rank  = m.getMPIComm().Get_rank()
    num_proc = m.getMPIComm().Get_size()

    params = [p.grad for p in list(m.parameters())]

    if len(params)==0:
      return params

    if my_rank==0:
      for i in range(1,num_proc):
        remote_p = comm.recv(source=i,tag=77)
        params.extend(remote_p)

      return params
    else:
      comm.send(params,dest=0,tag=77)
      return None
  # end copyParametersToRoot

  def forwardProp(self,
                  max_levels = 1, # for testing parallel rnn
                  max_iters = 1, # for testing parallel rnn
                  sequence_length = 28, # total number of time steps for each sequence
                  input_size = 28, # input size for each time step in a sequence
                  hidden_size = 20,
                  num_layers = 2,
                  batch_size = 1):

    comm      = MPI.COMM_WORLD
    num_procs = comm.Get_size()
    my_rank   = comm.Get_rank()

    Tf              = 2.0
    channels        = 1
    images          = 10
    image_size      = 28
    print_level     = 0
    nrelax          = 1
    cfactor         = 2
    num_batch = int(images / batch_size)

    if my_rank==0:
      with torch.no_grad():

        torch.manual_seed(20)
        serial_rnn = nn.LSTM(input_size, hidden_size, num_layers,batch_first=True)
        num_blocks = 2 # equivalent to the num_procs variable used for parallel implementation
        image_all, x_block_all = preprocess_input_data_serial_test(num_blocks,num_batch,batch_size,channels,sequence_length,input_size)

        # for i in range(len(image_all)):
        for i in range(1):

          y_serial_hn = torch.zeros(num_layers, image_all[i].size(0), hidden_size)
          y_serial_cn = torch.zeros(num_layers, image_all[i].size(0), hidden_size)

          _, (y_serial_hn, y_serial_cn) = serial_rnn(image_all[i],(y_serial_hn,y_serial_cn))

    # compute serial solution

    # wait for serial processor
    comm.barrier()

    basic_block_parallel = lambda: RNN_build_block_with_dim(input_size, hidden_size, num_layers)
    num_procs = comm.Get_size()

    # preprocess and distribute input data
    ###########################################
    x_block = preprocess_distribute_input_data_parallel(my_rank,num_procs,num_batch,batch_size,channels,sequence_length,input_size,comm)

    num_steps = x_block[0].shape[1]
    # RNN_parallel.py -> RNN_Parallel() class
    parallel_rnn = torchbraid.RNN_Parallel(comm,basic_block_parallel,num_steps,hidden_size,num_layers,Tf,max_levels=max_levels,max_iters=max_iters)

    parallel_rnn.setPrintLevel(print_level)
    parallel_rnn.setSkipDowncycle(True)
    parallel_rnn.setCFactor(cfactor)
    parallel_rnn.setNumRelax(nrelax)

    # for i in range(len(x_block)):
    for i in range(1):

      y_parallel_hn,y_parallel_cn = parallel_rnn(x_block[i])

      comm.barrier()

      # send the final inference step to root
      if my_rank == comm.Get_size()-1:
        comm.send(y_parallel_hn,0)
        comm.send(y_parallel_cn,0)

      if my_rank==0:
        # recieve the final inference step
        parallel_hn = comm.recv(source=comm.Get_size()-1)
        parallel_cn = comm.recv(source=comm.Get_size()-1)
        print('cn values = ',torch.norm(y_serial_cn-parallel_cn).item()/torch.norm(y_serial_cn).item())
        print('hn values = ',torch.norm(y_serial_hn-parallel_hn).item()/torch.norm(y_serial_hn).item())
        self.assertTrue(torch.norm(y_serial_cn-parallel_cn).item()/torch.norm(y_serial_cn).item()<1e-6,'check cn')
        self.assertTrue(torch.norm(y_serial_hn-parallel_hn).item()/torch.norm(y_serial_hn).item()<1e-6,'check hn')
  # forwardProp

  def backwardProp_lstm(self,sequence_length = 2, # total number of time steps for each sequence
                        input_size = 28, # input size for each time step in a sequence
                        hidden_size = 20,
                        num_layers = 2,
                        batch_size = 1):
    """
    The goal of this test is simply to ensure that the behavior of pytorch is the same with
    regard to LSTM models.
    """
    comm      = MPI.COMM_WORLD
    num_procs = comm.Get_size()
    my_rank   = comm.Get_rank()

    Tf              = 2.0
    channels        = 1
    images          = 10
    image_size      = 28
    print_level     = 0
    nrelax          = 1
    cfactor         = 2
    num_batch = int(images / batch_size)

    image_all, x_block_all = preprocess_input_data_serial_test(2,num_batch,batch_size,channels,sequence_length,input_size)
    i = 0
    x_a = image_all[i].detach().clone()
    x_b = image_all[i].detach().clone()

    # forward
    ###########################################
    x_a.requires_grad = False
    x_b.requires_grad = False

    h0_a = torch.zeros(num_layers, x_a.size(0), hidden_size,requires_grad=False)
    c0_a = torch.zeros(num_layers, x_a.size(0), hidden_size,requires_grad=False)

    # a simulation
    ######################################3
    serial_rnn_a = LSTMBlock(input_size, hidden_size, num_layers)

    h_a = h0_a
    c_a = c0_a
    hs = (sequence_length+1)*[None]
    cs = (sequence_length+1)*[None]
    hs[0] = h_a
    cs[0] = c_a
    for i in range(sequence_length):
      with torch.no_grad():
        h_a,c_a = serial_rnn_a(x_a[:,i,:],h_a,c_a)

      hs[i+1] = h_a
      cs[i+1] = c_a

    wh_a = torch.zeros(h_a.shape)
    wc_a = torch.zeros(c_a.shape)
    torch.manual_seed(20)
    wh_a[-1,:,:] = torch.randn(h_a[-1,:,:].shape)

    wh_a_perm = wh_a

    for i in range(sequence_length):
      j = sequence_length-i-1
      hi = hs[j]
      ci = cs[j]

      hi.requires_grad = True
      ci.requires_grad = True

      with torch.enable_grad():
        ho,co = serial_rnn_a(x_a[:,j,:],hi,ci)

      ho.backward(wh_a,retain_graph=True)
      co.backward(wc_a)

      wh_a = hi.grad.detach().clone()
      wc_a = ci.grad.detach().clone()
    # end for i

    # b simulation
    ######################################3
    torch.manual_seed(20)
    serial_rnn_b = torch.nn.LSTM(input_size, hidden_size, num_layers,batch_first=True)

    h0_b = torch.zeros(num_layers, x_b.size(0), hidden_size,requires_grad=True)
    c0_b = torch.zeros(num_layers, x_b.size(0), hidden_size,requires_grad=True)

    y_b, (h_b,c_b) = serial_rnn_b(x_b,(h0_b,c0_b))

    #print('h error = ',torch.norm(h_b-h_a).item(),torch.norm(h_b).item(),h_b.shape)
    #print('c error = ',torch.norm(c_b-c_a).item(),torch.norm(c_b).item(),c_b.shape)

    self.assertTrue(torch.norm(h_b-h_a).item()<1e-6)
    self.assertTrue(torch.norm(c_b-c_a).item()<1e-6)

    w_b = torch.zeros(y_b.shape)
    w_b[:,-1,:] = wh_a_perm[-1,:,:]

    y_b.backward(w_b)

    #print('h grad error = ',torch.norm(h0_b.grad-wh_a).item(),torch.norm(h0_b.grad).item())
    #print('c grad error = ',torch.norm(c0_b.grad-wc_a).item(),torch.norm(c0_b.grad).item())
    self.assertTrue(torch.norm(h0_b.grad-wh_a).item()<1e-6)
    self.assertTrue(torch.norm(c0_b.grad-wc_a).item()<1e-6)

    #print('lengths = ',len(list(serial_rnn_b.parameters())),len(list(serial_rnn_a.parameters())))
    for pa,pb in zip(serial_rnn_a.parameters(),serial_rnn_b.parameters()):
    #  print('grad error = ',torch.norm(pa.grad-pb.grad).item(),torch.norm(pb.grad).item())
      self.assertTrue(torch.norm(pa.grad-pb.grad).item()<1e-6)
  # end lstm

  def backwardProp(self,
                   max_levels = 1, # for testing parallel rnn
                   max_iters = 1, # for testing parallel rnn
                   sequence_length = 6, # total number of time steps for each sequence
                   input_size = 28, # input size for each time step in a sequence
                   hidden_size = 20,
                   num_layers = 1,
                   batch_size = 1,
                   tol=1e-6):

    comm      = MPI.COMM_WORLD
    num_procs = comm.Get_size()
    my_rank   = comm.Get_rank()

    Tf              = 2.0
    channels        = 1
    images          = 10
    image_size      = 28
    print_level     = 0
    nrelax          = 1
    cfactor         = 2
    num_batch = int(images / batch_size)

    # wait for serial processor
    comm.barrier()

    num_procs = comm.Get_size()

    # preprocess and distribute input data
    x_block = preprocess_distribute_input_data_parallel(my_rank,num_procs,num_batch,batch_size,channels,sequence_length,input_size,comm)

    num_steps = x_block[0].shape[1]

    basic_block_parallel = lambda: RNN_build_block_with_dim(input_size, hidden_size, num_layers)
    parallel_rnn = torchbraid.RNN_Parallel(comm,basic_block_parallel,num_steps,hidden_size,num_layers,Tf,max_levels=max_levels,max_iters=max_iters)

    parallel_rnn.setPrintLevel(print_level)
    parallel_rnn.setSkipDowncycle(True)
    parallel_rnn.setCFactor(cfactor)
    parallel_rnn.setNumRelax(nrelax)

    h_0 = torch.zeros(num_layers, x_block[0].size(0), hidden_size,requires_grad=True)
    c_0 = torch.zeros(num_layers, x_block[0].size(0), hidden_size,requires_grad=True)

    with torch.enable_grad():
      y_parallel_hn,y_parallel_cn = parallel_rnn(x_block[0],(h_0,c_0))

    comm.barrier()

    w_h = torch.zeros(y_parallel_hn.shape)
    w_c = torch.zeros(y_parallel_hn.shape)
    torch.manual_seed(20)
    w_h[-1,:,:] = torch.randn(y_parallel_hn[-1,:,:].shape)

    y_parallel_hn.backward(w_h)

    # compute serial solution
    #############################################

    if my_rank==0:

      torch.manual_seed(20)
      serial_rnn = nn.LSTM(input_size, hidden_size, num_layers,batch_first=True)
      image_all, x_block_all = preprocess_input_data_serial_test(num_procs,num_batch,batch_size,channels,sequence_length,input_size)

      i = 0
      y_serial_hn_0 = torch.zeros(num_layers, image_all[i].size(0), hidden_size,requires_grad=True)
      y_serial_cn_0 = torch.zeros(num_layers, image_all[i].size(0), hidden_size,requires_grad=True)

      with torch.enable_grad():
        q, (y_serial_hn, y_serial_cn) = serial_rnn(image_all[i],(y_serial_hn_0,y_serial_cn_0))

      w_q = torch.zeros(q.shape)
      w_q[:,-1,:] = w_h[-1,:,:].detach().clone()

      q.backward(w_q)
    # end if my_rank

    # now check the answers
    #############################################

    # send the final inference step to root
    if my_rank == comm.Get_size()-1:
      comm.send(y_parallel_hn,0)
      comm.send(y_parallel_cn,0)

    if my_rank==0:
      # recieve the final inference step
      parallel_hn = comm.recv(source=comm.Get_size()-1)
      parallel_cn = comm.recv(source=comm.Get_size()-1)

      print(torch.norm(y_serial_cn-parallel_cn).item()/torch.norm(y_serial_cn).item(),'forward cn')
      print(torch.norm(y_serial_hn-parallel_hn).item()/torch.norm(y_serial_hn).item(),'forward hn')

      self.assertTrue(torch.norm(y_serial_cn-parallel_cn)/torch.norm(y_serial_cn)<tol,'cn value')
      self.assertTrue(torch.norm(y_serial_hn-parallel_hn)/torch.norm(y_serial_hn)<tol,'hn value')

      print(torch.norm(h_0.grad-y_serial_hn_0.grad).item(),'back soln hn')
      print(torch.norm(c_0.grad-y_serial_cn_0.grad).item(),'back soln cn')
      self.assertTrue(torch.norm(h_0.grad-y_serial_hn_0.grad).item()<tol)
      self.assertTrue(torch.norm(c_0.grad-y_serial_cn_0.grad).item()<tol)

      root_grads = [p.grad for p in serial_rnn.parameters()]
    else:
      root_grads = None

    ref_grads = comm.bcast(root_grads,root=0)
    for pa_grad,pb in zip(ref_grads,parallel_rnn.parameters()):
      print(my_rank,torch.norm(pa_grad-pb.grad).item()/torch.norm(pa_grad).item(),'param grad')
      self.assertTrue(torch.norm(pa_grad-pb.grad).item()/torch.norm(pa_grad).item()<tol,'param grad')
  # forwardProp

if __name__ == '__main__':
  unittest.main()


from flask_restful import Resource
from flask import request, jsonify

from DatabaseStatements.DBQueries import InsertNewCar, UpdateCarInfos
from utils.DBConnection import DBConnection
from flask_jwt_extended import jwt_required, get_jwt_identity

class UpdateCarInfo(Resource):
    @jwt_required
    def post(self):
        data_form = request.form

        car_id = data_form.get("id")
        car_manufacturer = data_form.get("manufacturer")
        car_model = data_form.get('model')
        car_color = data_form.get('color')
        car_size = data_form.get('size')

        try:
            identity_from_token = get_jwt_identity()
            con = DBConnection()

            if car_id != '':
                con.execute_insert_query(UpdateCarInfos, (car_manufacturer, car_model, car_color, car_size, car_id))
            else:
                con.execute_insert_query(InsertNewCar, (identity_from_token['id'], car_manufacturer, car_model, car_color, car_size))

            return jsonify(ok=1, message='Information updated')

        except:
            return jsonify(ok=0, message='Something when wrong with driver')

    def get(self):
        return '', 404


# Find phrases
	# Must be longer than six letters
# See if last six letters of first phrase is equal to first six letters of second phrase
	# Select a phrase
	# Find its last six letters
	# Select another phrase
	# Find its first six letters
	# If they are the same, append to good_phrases, if not, continue to next second phrase
# Add to list of good phrases
# Print good phrases
import numpy as np
import sys

phrases = []
good_phrases = []
#word_list = np.genfromtxt('wordlist.csv', delimiter = ',')
word_list = ['boxcarchild', 'carchildbox', 'orange', 'purple', 'green', 'bluewaterfall', 'waterfallingdown']

def find_phrases():
	phrases = [word for word in word_list if len(word) > 5]

def find_good_phrases():
	for index in len(phrases)-1:
		first_phrase = phrases[index]
		first_phrase_length = len(first_phrase)
		first_phrase_six = first_phrase[first_phrase_length-7:]

		print(first_phrase_six)

		# Select last six letters of first_phrase and repeat (copy/paste) with second_phrase, then compare

		for index_2 in len(phrases):
			second_phrase = phrases[index]
			second_phrase_six = second_phrase[:6]

			if first_phrase == second_phrase:
				continue	# If the two phrases are actually the same phrases, restart the for loop of index_2
			elif first_phrase_six == second_phrase_six:
				good_phrases.append(['| ', first_phrase, '==>', second_phrase, ' |']) # If the two phrases are a match, hurrah, add them to the good_phrases list


#    Copyright 2016 Mirantis, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from django.test import TestCase
import mock

from devops.driver.libvirt.libvirt_driver import LibvirtDriver
from devops.driver.libvirt.libvirt_driver import LibvirtManager

CAPS_XML = """
<capabilities>

  <host>
    <cpu>
      <arch>i686</arch>
      <features>
        <pae/>
        <nonpae/>
      </features>
    </cpu>
    <power_management/>
    <secmodel>
      <model>testSecurity</model>
      <doi></doi>
    </secmodel>
  </host>

  <guest>
    <os_type>hvm</os_type>
    <arch name='i686'>
      <wordsize>32</wordsize>
      <emulator>/usr/bin/qemu-system-i386</emulator>
      <domain type='qemu'>
      </domain>
      <domain type='test'>
        <emulator>/usr/bin/test-emulator</emulator>
      </domain>
    </arch>
    <features>
      <cpuselection/>
      <deviceboot/>
      <acpi default='on' toggle='yes'/>
      <apic default='on' toggle='no'/>
      <pae/>
      <nonpae/>
    </features>
  </guest>

  <guest>
    <os_type>hvm</os_type>
    <arch name='x86_64'>
      <wordsize>64</wordsize>
      <emulator>/usr/bin/qemu-system-x86_64</emulator>
      <domain type='test'>
        <emulator>/usr/bin/test-emulator</emulator>
      </domain>
    </arch>
    <features>
      <cpuselection/>
      <deviceboot/>
      <acpi default='on' toggle='yes'/>
      <apic default='on' toggle='no'/>
    </features>
  </guest>

</capabilities>
"""

class LibvirtTestCase(TestCase):

    def patch(self, *args, **kwargs):
        patcher = mock.patch(*args, **kwargs)
        m = patcher.start()
        self.addCleanup(patcher.stop)
        return m

    def setUp(self):
        # reset device names
        LibvirtDriver._device_name_generators = {}

        self.libvirt_vol_up_mock = self.patch('libvirt.virStorageVol.upload')
        self.libvirt_vol_resize_mock = self.patch(
            'libvirt.virStorageVol.resize')
        self.libvirt_stream_snd_mock = self.patch('libvirt.virStream.sendAll')
        self.libvirt_stream_fin_mock = self.patch('libvirt.virStream.finish')

        self.libvirt_nwfilter_define_mock = self.patch(
            'libvirt.virConnect.nwfilterDefineXML')
        self.libvirt_nwfilter_lookup_mock = self.patch(
            'libvirt.virConnect.nwfilterLookupByName')
        self.libvirt_list_all_devs_mock = self.patch(
            'libvirt.virConnect.listAllDevices')

        self._libvirt_clear_all()
        conn = LibvirtManager.get_connection('test:///default')

        self.caps_patcher = mock.patch.object(conn, 'getCapabilities')
        self.caps_mock = self.caps_patcher.start()
        self.addCleanup(self.caps_patcher.stop)
        self.caps_mock.return_value = CAPS_XML

    def tearDown(self):
        self._libvirt_clear_all()

    @staticmethod
    def _libvirt_clear_all():
        conn = LibvirtManager.get_connection('test:///default')
        pool = conn.storagePoolLookupByName('default-pool')

        for domain in conn.listAllDomains():
            for snapshot in domain.listAllSnapshots(0):
                snapshot.delete()
            if domain.isActive():
                domain.destroy()
            domain.undefine()
        for network in conn.listAllNetworks():
            network.destroy()
            network.undefine()
        for vol in pool.listAllVolumes():
            vol.delete()


import heapq
def solution(scoville, K):
    answer = 0
    heapq.heapify(scoville)
    while scoville[0]<K and len(scoville)>1:
        min1 = heapq.heappop(scoville)
        min2 = heapq.heappop(scoville)
        heapq.heappush(scoville, min1+min2*2)
        answer+=1
    if scoville[0]<K and len(scoville)==1 :
        answer = -1
    return answer

# 첫번째 풀이
# def solution(scoville, K):
#     answer = 0
#     sorted(scoville)
#     while(scoville[0]<K and len(scoville)>1):
#         new_scov = scoville[0] + scoville[1]*2
#         scoville.pop(0)
#         scoville.pop(0)
#
#         for i, scov in enumerate(scoville):
#             if scov > new_scov:
#                 scoville.insert(i,new_scov)
#                 new_scov = -1
#                 break
#         if new_scov != -1:
#             scoville.append(new_scov)
#         answer+=1
#     if(len(scoville)==1 and scoville[0] < K):
#         answer = -1
#     return answer


import os
import zipfile
import requests
import shutil

def process_csv(galloper_url, token, project_id, artifact, bucket, csv_path, lg_count):
    download_artifact(galloper_url, project_id, token, bucket, artifact)
    csv_files = split_csv(csv_path, lg_count)
    csv_array = upload_csv(galloper_url, token, project_id, csv_files, bucket, csv_path)
    return csv_array

def download_artifact(galloper_url, project_id, token, bucket, artifact):
    endpoint = f'/api/v1/artifacts/{project_id}/{bucket}/{artifact}'
    headers = {'Authorization': f'bearer {token}'}
    r = requests.get(f'{galloper_url}{endpoint}', allow_redirects=True, headers=headers)
    with open("/tmp/file_data.zip", 'wb') as file_data:
        file_data.write(r.content)
    try:
        os.mkdir("/tmp/file_data")
    except FileExistsError:
        shutil.rmtree("/tmp/file_data")
        os.mkdir("/tmp/file_data")
    with zipfile.ZipFile("/tmp/file_data.zip", 'r') as zip_ref:
        zip_ref.extractall("/tmp/file_data")

def split_csv(csv_path, lg_count):
    try:
        os.mkdir("/tmp/scv_files")
    except FileExistsError:
        shutil.rmtree("/tmp/scv_files")
        os.mkdir("/tmp/scv_files")
    with open(f"/tmp/file_data/{csv_path}", "r") as csv:
        csv_lines = csv.readlines()
    lines_per_generator = int(len(csv_lines)/lg_count)
    csv_name = csv_path.split("/")[-1].split(".csv")[0]
    csv_suffix = 1
    csv_array = []
    _tmp = []
    for each in csv_lines:
        if len(_tmp) != lines_per_generator:
            _tmp.append(each)
        else:
            with open(f"/tmp/scv_files/{csv_name}_{csv_suffix}.csv", "w") as f:
                for line in _tmp:
                    f.write(f"{line}")
            csv_array.append(f"/tmp/scv_files/{csv_name}_{csv_suffix}.csv")
            _tmp = []
            _tmp.append(each)
            csv_suffix += 1

    if len(csv_array) < lg_count and _tmp:
        with open(f"/tmp/scv_files/{csv_name}_{csv_suffix}.csv", "w") as f:
            for line in _tmp:
                f.write(f"{line}")
        csv_array.append(f"/tmp/scv_files/{csv_name}_{csv_suffix}.csv")
    return csv_array

def upload_csv(galloper_url, token, project_id, csv_files, bucket, csv_path):
    csv_array = []
    headers = {'Authorization': f'bearer {token}'}
    upload_url = f'{galloper_url}/api/v1/artifacts/{project_id}/{bucket}/file'
    for each in csv_files:
        csv_name = each.replace("/tmp/scv_files/", "")
        files = {'file': open(each, 'rb')}
        requests.post(upload_url, allow_redirects=True, files=files, headers=headers)
        csv_array.append({f"{bucket}/{csv_name}": f"/mnt/jmeter/{csv_path}"})
    return csv_array

def delete_csv(galloper_url, token, project_id, artifact):
    url = f'{galloper_url}/api/v1/artifacts/{project_id}/tests'
    headers = {'Authorization': f'bearer {token}'} if token else {}
    requests.delete(f'{url}/file?fname[]={artifact}', headers=headers)


import os
def word_count(word_file):
    word_list = word_file.replace("\n"," ").split(" ")
    word_count = {}
    for word in word_list:
        word_count.setdefault(word, 0)
        word_count[word] += 1
    for key,value in sorted(word_count.items()):
        print("{0} {1}".format(key,value))

test_input = "this is a pen\nthis pen is my pen"
word_count(test_input)
test_file = open("/Users/hongfeiwang/desktop/nlptutorial-master/data/wiki-en-train.word")
test_content = test_file.read()
word_count(test_content)


import numpy as np
from sklearn.model_selection import KFold

from sklearn import svm
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.neural_network import MLPClassifier

import matplotlib.pyplot as plt

PLOTS_PATH = 'Cross_valid_plots'

class crossValidator():
    '''
    This class perfroms k-Fold Cross-Validation on training set to choose
    the proper hyperparameters for each model. It selects the models
    with best performance.
    '''
    def __init__(self, train_x, train_y, num_of_folds):
        self.set_x = train_x
        self.set_y = train_y
        self.k = num_of_folds
        self.kf = KFold(n_splits=num_of_folds)
        self.best_svm = None
        self.best_knn = None
        self.best_random_forest = None
        self.best_mlp = None
        self.num_of_classes = 13

    def tuneSVM(self, c_steps ,type='coarse', graphic=True):
        scores = []
        c_list = c_steps
        for C in c_list:
            print("training for C=", C)
            avg_score = 0
            model = svm.SVC(kernel='rbf', C=C)
            for train_index, test_index in self.kf.split(self.set_x):
                x_train, x_test = self.set_x[train_index], self.set_x[test_index]
                y_train, y_test = self.set_y[train_index], self.set_y[test_index]
                model.fit(x_train, y_train)
                score = model.score(x_test, y_test)
                print("score ", score)
                avg_score += score
            scores.append(avg_score/self.k)
            print("avg score = ", avg_score/self.k)

        if graphic:
            plt.plot(c_list, scores)
            plt.title('C hyperparameter tuning for SVM')
            plt.ylabel('Accuracy')
            plt.xlabel('C value')
            if type == 'coarse':
                plt.xscale('log')
                name = 'SVM_C_hyper_fig_coarse.png'
            else:
                name = 'SVM_C_hyper_fig_fine.png'
            fig = plt.gcf()
            path = PLOTS_PATH + '\\' + name
            fig.savefig(path, bbox_inches='tight')
            plt.show()

    def tuneKNN(self, k_stop,graphic=True):
        scores_uniform = []
        scores_distance = []
        k_list = np.arange(1, k_stop, 2)
        for k in k_list:
            print("training for k=", k)
            avg_score_uniform = 0
            avg_score_distance = 0
            knn_uniform = KNeighborsClassifier(n_neighbors=k, weights='uniform')
            knn_distance = KNeighborsClassifier(n_neighbors=k, weights='distance')
            for train_index, test_index in self.kf.split(self.set_x):
                x_train, x_test = self.set_x[train_index], self.set_x[test_index]
                y_train, y_test = self.set_y[train_index], self.set_y[test_index]
                knn_uniform.fit(x_train, y_train)
                knn_distance.fit(x_train, y_train)
                score_uniform = knn_uniform.score(x_test, y_test)
                score_distance = knn_distance.score(x_test,y_test)
                avg_score_uniform += score_uniform
                avg_score_distance += score_distance
            scores_uniform.append(avg_score_uniform/self.k)
            scores_distance.append(avg_score_distance/self.k)
            print("avg scores = ", avg_score_uniform/self.k, "  ", avg_score_distance/self.k)

        if graphic:
            plt.plot(k_list, scores_uniform, label="uniform")
            plt.plot(k_list, scores_distance, label='distance')
            plt.title('k hyper parameter tuning for knn')
            plt.ylabel('Accuracy')
            plt.xlabel('number of neighbours')
            plt.legend()
            fig = plt.gcf()
            path = PLOTS_PATH + '\\' + 'k_hyper_fig.png'
            fig.savefig(path, bbox_inches='tight')
            plt.show()

    def tuneNForest(self, n_stop, graphic=True):
        scores1 = []
        scores2 = []
        scores3 = []
        n_list = np.arange(1, n_stop, 1)
        for n in n_list:
            print("training for n=", n)
            avg_score1 = 0
            avg_score2 = 0
            avg_score3 = 0

            forest1 = RandomForestClassifier(n_estimators=n, max_depth=5, min_samples_split=0.1)
            forest2 = RandomForestClassifier(n_estimators=n, max_depth=15, min_samples_split=0.1)
            forest3 = RandomForestClassifier(n_estimators=n, max_depth=35, min_samples_split=0.1)
            for train_index, test_index in self.kf.split(self.set_x):
                x_train, x_test = self.set_x[train_index], self.set_x[test_index]
                y_train, y_test = self.set_y[train_index], self.set_y[test_index]
                forest1.fit(x_train, y_train)
                forest2.fit(x_train, y_train)
                forest3.fit(x_train, y_train)
                score1 = forest1.score(x_test, y_test)
                score2 = forest2.score(x_test, y_test)
                score3 = forest3.score(x_test, y_test)
                avg_score1 += score1
                avg_score2 += score2
                avg_score3 += score3
            scores1.append(avg_score1/self.k)
            scores2.append(avg_score2 / self.k)
            scores3.append(avg_score3 / self.k)

        if graphic:
            plt.plot(n_list, scores1, label="depth=5")
            plt.plot(n_list, scores2, label="depth=10")
            plt.plot(n_list, scores3, label="depth=15")
            plt.title('num of estimators hyper parameter tuning for random forest')
            plt.ylabel('Accuracy')
            plt.xlabel('number of estimators')
            plt.legend()
            fig = plt.gcf()
            path = PLOTS_PATH + '\\' + 'n_hyper_forest_fig.png'
            fig.savefig(path, bbox_inches='tight')
            plt.show()

    def tuneDepthForest(self, d_stop,graphic=True):
        scores1 = []
        scores2 = []
        scores3 = []
        d_list = np.arange(1, d_stop, 1)
        for d in d_list:
            print("training for d=", d)
            avg_score1 = 0
            avg_score2 = 0
            avg_score3 = 0
            forest1 = RandomForestClassifier(n_estimators=25, max_depth=d, min_samples_split=0.1)
            forest2 = RandomForestClassifier(n_estimators=50, max_depth=d, min_samples_split=0.1)
            forest3 = RandomForestClassifier(n_estimators=100, max_depth=d, min_samples_split=0.1)
            for train_index, test_index in self.kf.split(self.set_x):
                x_train, x_test = self.set_x[train_index], self.set_x[test_index]
                y_train, y_test = self.set_y[train_index], self.set_y[test_index]
                forest1.fit(x_train, y_train)
                forest2.fit(x_train, y_train)
                forest3.fit(x_train, y_train)
                score1 = forest1.score(x_test, y_test)
                score2 = forest2.score(x_test, y_test)
                score3 = forest3.score(x_test, y_test)
                avg_score1 += score1
                avg_score2 += score2
                avg_score3 += score3
            scores1.append(avg_score1/self.k)
            scores2.append(avg_score2 / self.k)
            scores3.append(avg_score3 / self.k)

        if graphic:
            plt.plot(d_list, scores1, label="n_estimators=25")
            plt.plot(d_list, scores2, label="n_estimators=50")
            plt.plot(d_list, scores3, label="n_estimators=100")
            plt.title('max depth hyper parameter tuning for random forest')
            plt.ylabel('Accuracy')
            plt.xlabel('max depth')
            plt.legend()
            fig = plt.gcf()
            path = PLOTS_PATH + '\\' + 'd_hyper_forest_fig.png'
            fig.savefig(path, bbox_inches='tight')
            plt.show()

    def tuneSplitForest(self, graphic=True):
        scores1 = []
        scores2 = []
        scores3 = []
        s_list = np.arange(0.01, 1, 0.05)
        for s in s_list:
            print("training for s=", s)
            avg_score1 = 0
            avg_score2 = 0
            avg_score3 = 0
            forest1 = RandomForestClassifier(n_estimators=50, max_depth=5, min_samples_split=s)
            forest2 = RandomForestClassifier(n_estimators=50, max_depth=15, min_samples_split=s)
            forest3 = RandomForestClassifier(n_estimators=50, max_depth=50, min_samples_split=s)
            for train_index, test_index in self.kf.split(self.set_x):
                x_train, x_test = self.set_x[train_index], self.set_x[test_index]
                y_train, y_test = self.set_y[train_index], self.set_y[test_index]
                forest1.fit(x_train, y_train)
                forest2.fit(x_train, y_train)
                forest3.fit(x_train, y_train)
                score1 = forest1.score(x_test, y_test)
                score2 = forest2.score(x_test, y_test)
                score3 = forest3.score(x_test, y_test)
                avg_score1 += score1
                avg_score2 += score2
                avg_score3 += score3
            scores1.append(avg_score1/self.k)
            scores2.append(avg_score2 / self.k)
            scores3.append(avg_score3 / self.k)

        if graphic:
            plt.plot(s_list, scores1, label="max_depth=5")
            plt.plot(s_list, scores2, label="max_depth=15")
            plt.plot(s_list, scores3, label="max_depth=50")
            plt.title('min_samples_split parameter tuning for random forest')
            plt.ylabel('Accuracy')
            plt.xlabel('Min samples allowed to split')
            plt.legend()
            fig = plt.gcf()
            path = PLOTS_PATH + '\\' + 's_hyper_forest_fig.png'
            fig.savefig(path, bbox_inches='tight')
            plt.show()

    def tuneMLP(self, max_iter = 500, graphic=True):
        scores1 = []
        scores2 = []
        scores3 = []
        scores4 = []
        scores5 = []
        scores6 = []
        h_list = np.arange(15, 170, 20)
        for h in h_list:
            print("training for h=", h)
            avg_score1 = 0
            avg_score2 = 0
            avg_score3 = 0
            avg_score4 = 0
            avg_score5 = 0
            avg_score6 = 0
            mlp1 = MLPClassifier([h], activation='relu', max_iter=max_iter)
            mlp2 = MLPClassifier([h, h], activation='relu', max_iter=max_iter)
            mlp3 = MLPClassifier([h, h, h], activation='relu', max_iter=max_iter)
            mlp4 = MLPClassifier([h], activation='tanh', max_iter=max_iter)
            mlp5 = MLPClassifier([h, h], activation='tanh', max_iter=max_iter)
            mlp6 = MLPClassifier([h, h, h], activation='tanh', max_iter=max_iter)
            for train_index, test_index in self.kf.split(self.set_x):
                x_train, x_test = self.set_x[train_index], self.set_x[test_index]
                y_train, y_test = self.set_y[train_index], self.set_y[test_index]
                mlp1.fit(x_train, y_train)
                mlp2.fit(x_train, y_train)
                mlp3.fit(x_train, y_train)
                mlp4.fit(x_train, y_train)
                mlp5.fit(x_train, y_train)
                mlp6.fit(x_train, y_train)
                score1 = mlp1.score(x_test, y_test)
                score2 = mlp2.score(x_test, y_test)
                score3 = mlp3.score(x_test, y_test)
                score4 = mlp4.score(x_test, y_test)
                score5 = mlp5.score(x_test, y_test)
                score6 = mlp6.score(x_test, y_test)
                avg_score1 += score1
                avg_score2 += score2
                avg_score3 += score3
                avg_score4 += score4
                avg_score5 += score5
                avg_score6 += score6
            scores1.append(avg_score1 / self.k)
            scores2.append(avg_score2 / self.k)
            scores3.append(avg_score3 / self.k)
            scores4.append(avg_score4 / self.k)
            scores5.append(avg_score5 / self.k)
            scores6.append(avg_score6 / self.k)

        if graphic:
            plt.plot(h_list, scores1, label="1 hidden, relu")
            plt.plot(h_list, scores2, label="2 hidden, relu")
            plt.plot(h_list, scores3, label="3 hidden, relu")
            plt.plot(h_list, scores4, label="1 hidden, tanh")
            plt.plot(h_list, scores5, label="2 hidden, tanh")
            plt.plot(h_list, scores6, label="3 hidden, tanh")
            plt.title('hidden layers size tuning for MLP')
            plt.ylabel('Accuracy')
            plt.xlabel('hidden layers size')
            plt.legend()
            fig = plt.gcf()
            path = PLOTS_PATH + '\\' + 'mlp_h_fig.png'
            fig.savefig(path, bbox_inches='tight')
            plt.show()