#!/usr/bin/env python
import random
import os
import pyttsx3
import speech_recognition as sr
import datetime
import shutil
import glob
import pandas as pd
from pyaudio import PyAudio
from tkinter.filedialog import askdirectory
from tkinter import Tk
import os
import hashlib
from pathlib import Path
import datetime
import glob
import os
import pyttsx3
import speech_recognition as sr
import datetime
engine = pyttsx3.init('sapi5')
voices = engine.getProperty('voices')
engine.setProperty('voice', voices[1].id)
def speak(audio: object) -> object:
    engine.say(audio)
    engine.runAndWait()
def takeCommand():
    r = sr.Recognizer()

    with sr.Microphone() as source:

        print("Listening...")
        r.pause_threshold = 1
        audio = r.listen(source)

    try:
        print("Recognizing...")
        query = r.recognize_google(audio, language='en-in')
        print(f"User said: {query}\n")

    except Exception as e:
        print(e)
        print("Unable to Recognize your voice.")
        return "None"

    return query
def wishMe():
    hour = int(datetime.datetime.now().hour)
    if hour >= 0 and hour < 12:
        speak("Good Morning  !")

    elif hour >= 12 and hour < 18:
        speak("Good Afternoon !")

    else:
        speak("Good Evening  !")
def display_instructions():
    pass
def introduction_and_instructions():
    speak("Hello....!")
    wishMe()
    speak("Welcome to automated file management system")
    speak("This project deals with  memory allocation and deallocation as well as basic file operations using voice commands which was given by user")
    speak("As this is a voice assistant........")
    speak("please give the commands when you see listening.... in the screen")
    speak("it is my primary responsibility to display the user manual")
    '''display_instructions()'''
    speak("make sure you give proper commands to me .......so that i can compute your requests")
    speak("now...Handing over to you")
'''class Automatic_File_Management:
    def init(self):
        pass
    def get_folder_statistics(self, folder):
        d = {}
        creation = []
        last_modification = []
        size = []
        file_name = []
        l = os.listdir(format(folder))
        for ele in l:
            file_name.append(ele)
            c = datetime.datetime.fromtimestamp(os.path.getctime(f'{folder}/{ele}'))
            m = datetime.datetime.fromtimestamp(os.path.getmtime(f'{folder}/{ele}'))
            creation.append(str(c).split()[0])
            last_modification.append(str(m).split()[0])
            size.append((os.path.getsize(f'{folder}/{ele}') / (1024 * 1024)))
        d["File"] = file_name
        d['Created'] = creation
        d['last_modified'] = last_modification
        d['Size(MB)'] = size
        df = pd.DataFrame(d)
        return df
    def get_files_with_maximum_storage(self, df: object) -> object:
        return df.sort_values('Size(MB)', ascending=False)
    def get_files_within_time(self, df, time):
        year = time.split('-')[0]
        month = time.split('-')[1]
        day = time.split('-')[2]
        sorted_statistics = df.sort_values('last_modified')
        sorted_statistic_upto_date = df[df['last_modified'] < year]
        maximum = self.get_files_with_maximum_storage(sorted_statistic_upto_date)
        return maximum
    def get_actual_file_location(self, file_name):
        file_locations = []
        desktop_path = 'C:/Users/91938/Desktop/**/{}'.format(file_name)
        documen_path = 'C:/Users/91938/Documents/**/{}'.format(file_name)
        downloads_path = 'C:/Users/91938/Downloads/**/{}'.format(file_name)
        music_path = 'C:/Users/91938/Music/**/{}'.format(file_name)
        pictures_path = 'C:/Users/91938/Pictures/**/{}'.format(file_name)
        favorites_path = 'C:/Users/91938/Favorites/**/{}'.format(file_name)
        video_path = 'C:/Users/91938/Videos/**/{}'.format(file_name)
        Desktop_files = glob.glob(desktop_path, recursive=True)
        Document_files = glob.glob(documen_path, recursive=True)
        Downloads_files = glob.glob(downloads_path, recursive=True)
        Favorites_files = glob.glob(favorites_path, recursive=True)
        Music_files = glob.glob(music_path, recursive=True)
        Pictures_files = glob.glob(pictures_path, recursive=True)
        video_files = glob.glob(video_path, recursive=True)
        overall_files = [Desktop_files, Document_files, Downloads_files, Favorites_files, Music_files, Pictures_files,
                         video_files]
        for ele in overall_files:
            if len(ele) > 1:
                for ele1 in ele:
                    file_locations.append(ele1)
        return file_locations
    def deallocate(self, file_name):
        directories = ["Desktop", "Downloads", "Documents", "Videos", "Pictures", "Favorites", "Music"]
        for dir in directories:
            if file_name.find(dir) == 1:
                d = dir
        os.remove(file_name)
        print("Succesfully Deleted")
    def get_disk_storage(self):
        disk_size = shutil.disk_usage('C:/')
        disk_left_size = disk_size.used
        disk_total = disk_size.total
        return disk_left_size,disk_total
    def is_memory_allocation_possible(self,file):
        disk_left_size,disk_total=self.get_disk_storage()
        file_size = os.path.getsize(file) // ( 2 ** 30)
        if file_size < disk_total:
            return True
        else:
            return False'''
class Automatic_File_Management:
    def init(self):
        pass
    def listdirs(self, root):
        l = []
        for path, subdirs, files in os.walk(root):
            for name in files:
                l.append(os.path.join(path, name))
        return l
    def get_folder_statistics(self, folder):
        d = {}
        creation = []
        last_modification = []
        size = []
        file_name = []
        l = os.listdir(folder)
        for ele in l:
            if os.path.isdir(os.path.join(folder,ele)):
                ld = self.listdirs(os.path.join(folder,ele))
                for i in ld:
                    file_name.append(i)
                    c = datetime.datetime.fromtimestamp(os.path.getctime(i))
                    m = datetime.datetime.fromtimestamp(os.path.getmtime(i))
                    creation.append(str(c).split()[0])
                    last_modification.append(str(m).split()[0])
                    size.append((os.path.getsize(i) / (1024 * 1024)))
            file_name.append(ele)
            c = datetime.datetime.fromtimestamp(os.path.getctime(f'{folder}/{ele}'))
            m = datetime.datetime.fromtimestamp(os.path.getmtime(f'{folder}/{ele}'))
            creation.append(str(c).split()[0])
            last_modification.append(str(m).split()[0])
            size.append((os.path.getsize(f'{folder}/{ele}') / (1024 * 1024)))
        d["File"] = file_name
        d['Created'] = creation
        d['last_modified'] = last_modification
        d['Size(MB)'] = size
        df = pd.DataFrame(d)
        return df
    def get_files_with_maximum_storage(self, df: object) -> object:
        return df.sort_values('Size(MB)', ascending=False)
    def get_files_within_time(self, df, time):
        year = time.split('-')[0]
        month = time.split('-')[1]
        day = time.split('-')[2]
        sorted_statistics = df.sort_values('last_modified')
        sorted_statistic_upto_date = df[df['last_modified'] < year]
        maximum = self.get_files_with_maximum_storage(sorted_statistic_upto_date)
        return maximum
    def get_actual_file_location(self, file_name):
        file_locations = []
        desktop_path = 'C:/Users/91938/Desktop/**/{}'.format(file_name)
        documen_path = 'C:/Users/91938/Documents/**/{}'.format(file_name)
        downloads_path = 'C:/Users/91938/Downloads/**/{}'.format(file_name)
        music_path = 'C:/Users/91938/Music/**/{}'.format(file_name)
        pictures_path = 'C:/Users/91938/Pictures/**/{}'.format(file_name)
        favorites_path = 'C:/Users/91938/Favorites/**/{}'.format(file_name)
        video_path = 'C:/Users/91938/Videos/**/{}'.format(file_name)
        Desktop_files = glob.glob(desktop_path, recursive=True)
        Document_files = glob.glob(documen_path, recursive=True)
        Downloads_files = glob.glob(downloads_path, recursive=True)
        Favorites_files = glob.glob(favorites_path, recursive=True)
        Music_files = glob.glob(music_path, recursive=True)
        Pictures_files = glob.glob(pictures_path, recursive=True)
        video_files = glob.glob(video_path, recursive=True)
        overall_files = [Desktop_files, Document_files, Downloads_files, Favorites_files, Music_files, Pictures_files,
                         video_files]
        for ele in overall_files:
            if len(ele) > 1:
                for ele1 in ele:
                    file_locations.append(ele1)
        return file_locations
    def deallocate(self, file_name):
        directories = ["Desktop", "Downloads", "Documents", "Videos", "Pictures", "Favorites", "Music"]
        for dir in directories:
            if file_name.find(dir) == 1:
                d = dir
        os.remove(file_name)
        print("Succesfully Deleted")
    def get_disk_storage(self):
        disk_size = shutil.disk_usage('C:/')
        disk_left_size = disk_size.used
        disk_total = disk_size.total
        return disk_left_size,disk_total
    def is_memory_allocation_possible(self,file):
        disk_left_size, disk_total = self.get_disk_storage()
        file_size = os.path.getsize(file) // ( 2 ** 30)
        if file_size < disk_total:
            return True
        else:
            return False
    def check_file_with_extenstion_if_it_is_present(self, file):
        df = pd.read_csv("C:/Users/91938/Desktop/Automated OS/allfiles.csv")
        l = list(df["File"])
        txt = []
        ppt = []
        pdf = []
        docx = []
        java = []
        python = []
        excel_and_csv = []
        ipy = []
        jpg_or_jpeg = []
        png = []
        c = []
        for ele in l:
            if ele.endswith(".txt"):
                txt.append(ele)
            elif ele.endswith(".c"):
                c.append(ele)
            elif ele.endswith(".png"):
                png.append(ele)
            elif ele.endswith(".jpg") or ele.endswith(".jpeg"):
                jpg_or_jpeg.append(ele)
            elif ele.endswith(".java"):
                java.append(ele)
            elif ele.endswith(".py"):
                python.append(ele)
            elif ele.endswith(".ipynb"):
                ipy.append(ele)
            elif ele.endswith(".xlsx") or ele.endswith(".csv"):
                excel_and_csv.append(ele)
            elif ele.endswith(".docx"):
                docx.append(ele)
            elif ele.endswith(".pdf"):
                pdf.append(ele)
            elif ele.endswith(".pptx"):
                ppt.append(ele)
        if file.endswith(".txt"):
            r = []
            for ele in txt:
                if ele.endswith(file):
                    r.append(1)
            return len(r)

        elif file.endswith(".pdf"):
            r = []
            for ele in pdf:
                if ele.endswith(file):
                    r.append(1)
            return len(r)
        elif file.endswith(".pptx"):
            r = []
            for ele in ppt:
                if ele.endswith(file):
                    r.append(1)
            return len(r)
        elif file.endswith(".docx"):
            r = []
            for ele in docx:
                if ele.endswith(file):
                    r.append(1)
            return len(r)
        elif file.endswith(".java"):
            r = []
            for ele in java:
                if ele.endswith(file):
                    r.append(1)
            return len(r)
        elif file.endswith(".py"):
            r = []
            for ele in python:
                if ele.endswith(file):
                    r.append(1)
            return len(r)
        elif file.endswith(".ipynb"):
            r = []
            for ele in ipy:
                if ele.endswith(file):
                    r.append(1)
            return len(r)
        elif file.endswith(".xlsx") or file.endswith(".csv"):
            r = []
            for ele in excel_and_csv:
                if ele.endswith(file):
                    r.append(1)
            return len(r)
        elif file.endswith(".jpeg") or file.endswith(".jpg"):
            r = []
            for ele in jpg_or_jpeg:
                if ele.endswith(file):
                    r.append(1)
            return len(r)
        elif file.endswith(".png"):
            r = []
            for ele in png:
                if ele.endswith(file):
                    r.append(1)
            return len(r)
        elif file.endswith(".c"):
            r = []
            for ele in c:
                if ele.endswith(file):
                    r.append(1)
            return len(r)
class txtfiles:
    def _init_(self):
        pass

    def write_file(self, txt, file_name=None):
        if file_name != None:
            f = open(file_name, "w")
            f.writelines(txt)
            f.close()
        else:
            file_name = "C:/Users/{}/Automated OS/Files/Text/{}.txt".format(os.getlogin(),
                                                                            random.choice([i for i in range(1000)]))
            f = open(file_name, "w")
            f.writelines(txt)
            f.close()

    def read_file(self, file_name):
        file_name
        f = open(file_name, "r")
        l = []
        for i in f.readlines():
            l.append(i)
        f.close()
        return l

    def create_file(self, txt, file_name=None):

         if file_name != None:
            file_name1 = "C:/Users/91938/Desktop/Automated OS/Files/Text/{}.txt".format(file_name)
            f = open(file_name1, "w")
            f.writelines(txt)
            f.close()

    def update_file(self, txt, file_name):
        if file_name != None:
            file_name1  = "C:/Users/91938/Desktop/Automated OS/Files/Text/{}.txt".format(file_name)
            f = open(file_name1, "a")
            f.writelines(txt)
            f.close()
        else:
            speak("new file being created ")
            self.create_file(self, txt, file_name)

    def delete_file(self, file_name):
        if file_name != None:
            os.remove("C:/Users/91938/Desktop/Automated OS/Files/Text/{}.txt".format(file_name))
        else:
            speak("no file exists with name {} to delete".format(file_name))

    def copy(self, file_name, dest=None):

        if dest != None:
            shutil.copy(file_name, dest)
        else:
            path = 'C:/Users/91938/Desktop/Automated OS/Files/Text/{}-{}.txt'.format(file_name,"Copy")
            shutil.copy(file_name,'C:/Users/91938/Desktop/Automated OS/Files/Text')

    def move(self, file, dest):
        shutil.move(file, dest)
class Cluster_the_Files:
    def _init_(self):
        pass

    def get_extention(self, file_name):
        return str(file_name).split('.')[-1]

    def create_copy_of_that_file(self, filename):
        path = str(filename).split('.')[-2]
        copied_file = path + '-Copy'
        copy = copied_file + "." + self.get_extention(filename)
        shutil.copyfile(filename, copy)
        return copy

    def cluster_files_from_dir(self, dir):
        path = "C:/Users/91938/" + dir + "/"
        textfiles = glob.glob(path + "*.txt", recursive=True)
        docxfiles = glob.glob(path + "*.docx", recursive=True)
        pdffiles = glob.glob(path + "*.pdf", recursive=True)
        csvfiles = glob.glob(path + "*.csv", recursive=True)
        excelfiles = glob.glob(path + "*.xlsx", recursive=True)
        pptfiles = glob.glob(path + "*.ppt", recursive=True)
        javafiles = glob.glob(path + "*.java", recursive=True)
        pythonfiles = glob.glob(path + "*.py", recursive=True)
        pngfiles = glob.glob(path + "*.png", recursive=True)
        jpgfiles = glob.glob(path + "*.jpg", recursive=True)
        ipynbfiles = glob.glob(path + "*.ipynb", recursive=True)
        cfiles = glob.glob(path + "*.c", recursive=True)
        for ele in textfiles:
            ele1 = self.create_copy_of_that_file(ele)
            shutil.copy(ele1, r"C:\Users\91938\Desktop\Automated OS1\Files\Text")
        for ele in docxfiles:
            ele1 = self.create_copy_of_that_file(ele)
            shutil.copy(ele1, r"C:\Users\91938\Desktop\Automated OS1\Files\docx")
        for ele in pptfiles:
            ele1 = self.create_copy_of_that_file(ele)
            shutil.copy(ele1, r"C:\Users\91938\Desktop\Automated OS1\Files\PPTS")
        for ele in pngfiles:
            ele1 = self.create_copy_of_that_file(ele)
            shutil.copy(ele1, r"C:\Users\91938\Desktop\Automated OS1\Files\PNG")
        for ele in javafiles:
            ele1 = self.create_copy_of_that_file(ele)
            shutil.copy(ele1, r"C:\Users\91938\Desktop\Automated OS1\Files\Java Scripts")
        for ele in pythonfiles:
            ele1 = self.create_copy_of_that_file(ele)
            shutil.copy(ele1, r"C:\Users\91938\Desktop\Automated OS1\Files\Python Scripts")
        for ele in excelfiles:
            ele1 = self.create_copy_of_that_file(ele)
            shutil.copy(ele1, r"C:\Users\91938\Desktop\Automated OS1\Files\Excels and CSV's")
        for ele in csvfiles:
            ele1 = self.create_copy_of_that_file(ele)
            shutil.copy(ele1, r"C:\Users\91938\Desktop\Automated OS1\Files\Excels and CSV's")
        for ele in jpgfiles:
            ele1 = self.create_copy_of_that_file(ele)
            shutil.copy(ele1, r"C:\Users\91938\Desktop\Automated OS1\Files\jpeg or jpg")
        for ele in ipynbfiles:
            ele1 = self.create_copy_of_that_file(ele)
            shutil.copy(ele1, r"C:\Users\91938\Desktop\Automated OS1\Files\IPY NoteBooks")
        for ele in pdffiles:
            ele1 = self.create_copy_of_that_file(ele)
            shutil.copy(ele1, r"C:\Users\91938\Desktop\Automated OS1\Files\PDFS")
        print("all files are clustered from {}".format(dir))

    def cluster_files(self):
        self.cluster_files_from_dir("3D Objects")
        self.cluster_files_from_dir("Contacts")
        self.cluster_files_from_dir("Cookies")
        self.cluster_files_from_dir("Desktop")
        self.cluster_files_from_dir("Documents")
        self.cluster_files_from_dir("Downloads")
        self.cluster_files_from_dir("Favorites")
        self.cluster_files_from_dir("Links")
        self.cluster_files_from_dir("Music")
        self.cluster_files_from_dir("My Documents")
        self.cluster_files_from_dir("Pictures")
        self.cluster_files_from_dir("PycharmProjects")
        self.cluster_files_from_dir("Saved Games")
        self.cluster_files_from_dir("Recent")
        self.cluster_files_from_dir("Videos")
class Delete_duplicate_content:
    def _init_(self):
        pass

    def get_subdirectories(self, path):
        return os.listdir(path)

    def delete_duplicate_files_from_folder(self,path):

            file_list = os.listdir(path)

            unique = dict()

            for file in file_list:
                file_name = Path(os.path.join(path, file))

                if file_name.is_file():
                    fileHash = hashlib.md5(open(file_name, 'rb').read()).hexdigest()

                    if fileHash not in unique:
                        unique[fileHash] = file_name

                    else:
                        os.remove(file_name)
                        print("successfully  deleted at {}".format(path))

                else:
                    print("No Duplicates in {}".format(path))
    def delete_duplicate_files(self):
        self.delete_duplicate_files_from_folder("C:/Users/91938/Desktop/Automated OS1/Files/C Scripts")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Desktop/Automated OS1/Files/docx")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Desktop/Automated OS1/Files/Excels and CSV's")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Desktop/Automated OS1/Files/IPY NoteBooks")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Desktop/Automated OS1/Files/Java Scripts")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Desktop/Automated OS1/Files/jpeg or jpg")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Desktop/Automated OS1/Files/PDFS")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Desktop/Automated OS1/Files/PNG")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Desktop/Automated OS1/Files/PPTS")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Desktop/Automated OS1/Files/Python Scripts")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Desktop/Automated OS1/Files/Text")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Desktop")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Documents")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Downloads")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Pictures")
        self.delete_duplicate_files_from_folder("C:/Users/91938/Videos")

obj1 = txtfiles()
obj2 = Automatic_File_Management()
obj3 = Delete_duplicate_content()
obj4 = Cluster_the_Files()

speak("Hello....!")
wishMe()
speak("Welcome to automated file management system")
speak("This project deals with  memory allocation and deallocation as well as basic file operations using voice commands which was given by user")
display_instructions()
speak("make sure you give proper commands to me .......so that i can compute your requests")
speak("Clustering all the files from DISK")
obj4.cluster_files()
speak("Clustering is Completed")
speak("Deleting duplicate files")
obj3.delete_duplicate_files()
speak("Duplicate files are deleted")
speak("As this is a voice assistant........")
speak("please give the commands when you see listening.... in the screen")
speak("it is my primary responsibility to display the user manual")
speak("now...Handing over to you")
obj1 = txtfiles()
obj2 = Automatic_File_Management()
obj3 = Delete_duplicate_content()
obj4 = Cluster_the_Files()
while True:
    query = takeCommand().lower()
    if "hello" in query:
        speak("hello user")
    elif 'exit' in query:
        speak("ok")
        speak("thank you")
        exit()
    elif "how are you" in query:
        speak("iam fine")
    elif "delete" in query:
        speak("please tell me the name of the file you want me to delete")
        q6 = takeCommand().lower()
        obj1.delete_file(q6)
        speak("deleted the file" + q6)
    elif "copy" in query:
        speak("tell me the name of the file you want me to copy")
        q7 = takeCommand().lower()
        speak("do you want to specify the directory to which the file needs to be copied ?")
        q8 = takeCommand()
        if "yes" in q8:
            speak("what is your destination file path with out extention?")
            q9 = takeCommand()
            q = q9[0].upper()+q9[1:]
            obj1.copy(q7, q)
            obj3.delete_duplicate_files()
        elif "no" in q8:
            obj1.copy(q7)
            speak("Alert an duplicate object is been created ")
            obj3.delete_duplicate_files()
    elif "update" in query:
        speak("mention the name of the file which you want to update without extention")
        q10 = takeCommand()
        speak("tell me the text which needs to be added into the file")
        q11 = takeCommand()
        obj1.update_file(q11, q10)
        obj3.delete_duplicate_files()
    elif "move" in query:
        speak("please mention the filename which needs to be moved")
        q12 = takeCommand().lower()
        print(q12)
        speak("mention the directory to which the file needs to be moved")
        q13 = takeCommand().lower()

        dest1  = q13[0].upper()+q13[1:]
        print(dest1,type(dest1))
        dest = "C:/Users/91938/{}".format(dest1)
        src = "C:/Users/91938/Desktop/Automated OS/Files/Text/{}.txt".format(q12)
        obj1.move(src, dest)
    elif "open" in query:
        speak("please tell  me the name of file which you want to view or read contents")
        q4 = takeCommand().lower()
        speak("do you want me to share the content on screen ? please say yes or no")
        q5 = takeCommand().lower()
        file = "C:/Users/91938/Desktop/Automated OS/Files/Text/{}.txt".format(q4)
        if "yes" or "yeah" in q5:
            content = obj1.read_file(file)
            for ele in content:
                print(ele)
                speak(ele)
        elif "no" in q5:
            content = obj1.read_file(file)
            for ele in content:
                speak(ele)
    elif "create" in query:
        speak("do you want me to create a text file or not ?")
        speak("please answer yes or no")
        q1 = takeCommand().lower()
        if "yes" in q1:
            speak("please mention the name of the file you want to save it as without extention")
            q2 = takeCommand().lower()
            speak("enter the content you wanted to speak")
            q3 = takeCommand().lower()
            obj1.create_file(q3, q2)
            speak("Checking if the available storage")
            if obj2.is_memory_allocation_possible("C:/Users/91938/Desktop/Automated OS/Files/Text/{}.txt".format(q2)):
                speak("file is allocated")
                speak("file saved")
                obj3.delete_duplicate_files()
            else:
                speak("deallocation is required....")
                mdf = obj2.get_files_within_time("2021-08-01")
                speak("all files are displayed from different directories with maximum storages of files in it")
                for d in ["Desktop", "Downloads", "Documents", "Videos", "Pictures", "Favorites", "Music"]:
                    print("--------------------------------------------------------------------------------------------------------------------")
                    print("                                                 {}                                                                 ".format(dir))
                    print("--------------------------------------------------------------------------------------------------------------------")
                    print(obj2.get_files_with_maximum_storage(d))
                    print("--------------------------------------------------------------------------------------------------------------------")
                    print("--------------------------------------------------------------------------------------------------------------------")
                speak("pick up the directory from the above")
                dir2 = takeCommand().lower()
                dir2 = dir2[0].upper()+dir2[1:]
                speak("enter the index of the file from the choosen directory")
                index = takeCommand().lower()
                df = obj2.get_files_with_maximum_storage(dir2)
                file_location = obj2.get_actual_file_location(str(df["File"][index]))
                if len(file_location) > 1:
                    speak("the file you wanted to dealloacte is located at multiple locations....")
                    for ele in file_location:
                        obj2.deallocate(ele)
                    speak("deallocation is done")
                else:
                    obj2.deallocate(file_location[0])
                    obj3.delete_duplicate_files()
                    speak("deallocation is done")
        else:
            speak("sorry....im designed in a way to save the text files")
    elif "folder statistics" in query:
        speak("please enter the name of the folder or directory  from the disk")
        q14 = takeCommand().lower()
        dir  = q14[0].upper()+q14[1:]
        dir1 = "C:/Users/91938/{}".format(dir)
        statistics = obj2.get_folder_statistics(dir1)
        speak("The folder statistics will be displayed on screem")
        print("-------------------------------------------------------------------------------------------------------------------")
        print("                                                {}                                                                    ".format(dir))
        print("-------------------------------------------------------------------------------------------------------------------")
        print(statistics)
        print("-------------------------------------------------------------------------------------------------------------------")
        print("-------------------------------------------------------------------------------------------------------------------")
    elif "file location" in query:
        speak("please tell enter the name  of file with extention as the extention is important")
        q15 = input("Enter the name of the file with extention : ")
        locations = obj2.get_actual_file_location(q15)
        speak("the location of the files are being displayed in screen")
        print(locations)
    elif "disk statistics" in query:
        disk_left, disk_total = obj2.get_disk_storage()
        speak("Your disk statistics are ")
        speak("you have used {} out of {} GB in C Drive".format(((disk_total - disk_left) // (2 * 30)),(disk_total // (2 * 30))))

    elif 'duplicate' in query:
        speak("enter the name of file with extention you wanted to duplicate")
        q16 = input("enter the name of the file you wanted to create a duplicate with extention : ")
        locations1 = obj2.get_actual_file_location(q16)
        if len(locations1) > 1:
            speak("the file you wanted to replicate is located at multiple locations....")
            speak("do you still want to make duplicate of that which occupies more storage?please answer yes or no")
            q17 = takeCommand().lower()
            if q17 == "yes":
                for ele in locations1:
                    obj4.create_copy_of_that_file(ele)
                speak("duplicates are created")
            elif q17 == "no":
                speak("pick up the index which denotes a single location of the given file")
                for i in range(len(locations1)):
                    print("{} - > {}".format(i+1,locations1[i]))
                q18 = takeCommand().lower()
                obj4.create_copy_of_that_file(locations1[int(q18)-1])
                speak("duplicate is created")
        else:
            obj4.create_copy_of_that_file(locations1[0])
            speak("duplicate is created")
        obj3.delete_duplicate_files()


import re
from asyncio import sleep
from random import choice
from typing import List, Union, Optional, Tuple

import discord
from redbot.core import Config, checks, commands
from redbot.core.bot import Red
from redbot.core.utils.chat_formatting import escape, pagify, warning

from cog_shared.swift_libs.formatting import tick
from cog_shared.swift_libs.menus import confirm, PaginatedMenu
from cog_shared.swift_libs.time import FutureTime
from rndactivity.placeholders import Placeholders
from rndactivity.shared import translate, log
from swift_i18n import Humanize

TWITCH_REGEX = re.compile(r"<?(?:https?://)?twitch\.tv/[^ ]+>?", re.IGNORECASE)
Delay = FutureTime().min_time.minutes(5).default_period.minutes()

class ActivityTypeConverter(commands.Converter):
    async def convert(self, ctx, argument):
        argument = argument.lower()
        if argument not in ("playing", "watching", "listening", "streaming"):
            raise commands.BadArgument
        return getattr(discord.ActivityType, argument)

@translate.cog("help.cog_class")
class RNDActivity(commands.Cog):
    def __init__(self, bot: Red):
        super().__init__()
        self.bot = bot
        self.config = Config.get_conf(self, identifier=2042511098, force_registration=True)
        self.config.register_global(statuses=[], delay=600)
        self._running_tasks = [self.bot.loop.create_task(self.status_task())]

    def cog_unload(self):
        for task in self._running_tasks:
            task.cancel()

    @staticmethod
    def extract_status(data: Union[str, dict]) -> Tuple[int, str]:
        if isinstance(data, str):
            return 0, data
        return data.get("type", 0), data["game"]

    def format_status(self, status: Union[str, dict], shard: int = 0, **kwargs) -> discord.Activity:
        game_type = 0
        url = None
        if isinstance(status, dict):
            game_type: int = status.get("type", 0)
            url: Optional[str] = status.get("url")
            status: str = status.get("game")

        formatted = Placeholders.parse(status, self.bot, shard, **kwargs)
        # noinspection PyArgumentList
        return discord.Activity(name=formatted, url=url, type=discord.ActivityType(game_type))

    async def update_status(self, statuses: List[str]):
        if not statuses:
            return
        status = choice(statuses)
        for shard in self.bot.shards.keys():
            try:
                game = self.format_status(status, shard=shard)
            except KeyError as e:
                log.exception(
                    f"Encountered invalid placeholder while attempting to parse status "
                    f"#{statuses.index(status) + 1}, skipping status update.",
                    exc_info=e,
                )
                return
            await self.bot.change_presence(
                activity=game, status=self.bot.guilds[0].me.status, shard_id=shard
            )

    async def status_task(self):
        await self.bot.wait_until_ready()
        while True:
            try:
                await self.update_status(list(await self.config.statuses()))
            except Exception as e:
                log.exception("Failed to update bot status", exc_info=e)
            await sleep(int(await self.config.delay()))

    ##############################################

    # noinspection PyTypeChecker
    @commands.group()
    @checks.is_owner()
    @checks.bot_in_a_guild()
    @translate.command("help.root", placeholders=Placeholders(None, 0))
    async def rndactivity(self, ctx: commands.Context):
        pass

    @rndactivity.command(name="delay")
    @translate.command("help.delay")
    async def rndactivity_delay(self, ctx: commands.Context, *, duration: Delay):
        await self.config.delay.set(duration.total_seconds())
        await ctx.send(tick(translate("delay_set", duration=Humanize(duration))))

    @rndactivity.command(name="add", usage="[activity_type=playing] <status>")
    @translate.command("help.add")
    async def rndactivity_add(
        self, ctx: commands.Context, game_type: Optional[ActivityTypeConverter], *, game: str
    ):
        stream = None
        if game_type is None:
            game_type = discord.ActivityType.playing
        elif game_type == discord.ActivityType.streaming:
            game = game.split(" ")
            stream, game = game.pop(), " ".join(game)
            if not TWITCH_REGEX.match(stream):
                raise commands.BadArgument(translate("no_stream_provided"))
        activity = {
            "type": game_type.value,
            "game": game,
            "url": stream.strip("<>") if stream else None,
        }

        try:
            self.format_status(activity, error_deprecated=True)
        except KeyError as e:
            await ctx.send(warning(translate("parse_fail_invalid_placeholder", placeholder=str(e))))
        else:
            async with self.config.statuses() as statuses:
                statuses.append(activity)
                await ctx.send(tick(translate("added_status", id=len(statuses))))

    @rndactivity.command(name="parse")
    @translate.command("help.parse")
    async def rndactivity_parse(self, ctx: commands.Context, *, status: str):
        shard = getattr(ctx.guild, "shard_id", 0)

        try:
            result = self.format_status(status, shard=shard, error_deprecated=True)
        except KeyError as e:
            await ctx.send(
                warning(
                    translate(
                        "parse_fail_invalid_placeholder",
                        placeholder=escape(str(e), mass_mentions=True),
                    )
                )
            )
        else:
            await ctx.send(
                translate(
                    "parse_result",
                    input=escape(status, mass_mentions=True),
                    result=escape(result.name, mass_mentions=True),
                )
            )

    @rndactivity.command(name="remove", aliases=["delete"])
    @translate.command("help.remove")
    async def rndactivity_remove(self, ctx: commands.Context, status: int):
        async with self.config.statuses() as statuses:
            if len(statuses) < status:
                return await ctx.send(warning(translate("non_existant_status", id=status)))

            removed = statuses.pop(status - 1)
            if not statuses:
                await self.bot.change_presence(
                    activity=None, status=getattr(ctx.me, "status", None)
                )
        removed = escape(self.extract_status(removed)[1], mass_mentions=True)
        await ctx.send(tick(translate("status_removed", id=status, status=removed)))

    @rndactivity.command(name="list")
    @translate.command("help.list")
    async def rndactivity_list(self, ctx: commands.Context):
        statuses = list(await self.config.statuses())
        if not statuses:
            return await ctx.send(warning(translate("no_setup_statuses", prefix=ctx.prefix)))

        pages = []
        for status in statuses:
            index = statuses.index(status) + 1
            game_type = 0
            stream_url = None
            if isinstance(status, dict):
                game_type = status.get("type", 0)
                stream_url = status.get("url")
                status = status.get("game")
            status = escape(status, formatting=True, mass_mentions=True)
            if stream_url:
                status = f"[{status}]({stream_url})"
            pages.append(
                f"**#{index}** \N{EM DASH} **{translate('game_type')[game_type]}** {status}"
            )

        colour = await ctx.embed_colour()
        pages = [discord.Embed(colour=colour, description=x) for x in pagify("\n".join(pages))]
        await PaginatedMenu(
            pages=pages, bot=ctx.bot, channel=ctx.channel, member=ctx.author
        ).prompt()

    @rndactivity.command(name="clear")
    @translate.command("help.clear")
    async def rndactivity_clear(self, ctx: commands.Context):
        amount = len(await self.config.statuses())
        if await confirm(ctx, content=translate("clear_confirm", amount=amount)):
            await self.config.statuses.set([])
            await self.bot.change_presence(activity=None, status=self.bot.guilds[0].me.status)
            await ctx.send(tick(translate("clear_success", amount=amount)))
        else:
            await ctx.send(translate("ok_then"))


import csv
import nltk
import numpy as np
import pickle
import pymorphy2
import re
import os
from math import log
from nltk import ngrams
from nltk.corpus import stopwords
from nltk.text import Text
from operator import itemgetter
from sklearn.feature_extraction.text import CountVectorizer

m = pymorphy2.MorphAnalyzer()
stopwords = stopwords.words("russian")
vectorizer = CountVectorizer()

def get_docs():
    with open('quora_question_pairs_rus.csv', 'r', encoding='utf-8') as t:
        corpus = csv.reader(t)
        text = list(corpus)
    if not os.path.exists('docs.pickle'):
        docs = []
        for i, line in enumerate(text):
            if i != 0 and i < 10002:
                docs.append(line[2])
        with open("docs.pickle", "wb") as p:
                pickle.dump(docs, p)
    else:
        with open("docs.pickle", "rb") as p:
            docs = pickle.load(p)
    return docs

def get_corpus(docs):
    if not os.path.exists('corpus.pickle'):
        corpus = []
        for i, sent in enumerate(docs):
            if i < 10001:
                corpus.append(' '.join(clean_text(sent)))
            else:
                break
            with open("corpus.pickle", "wb") as d:
                pickle.dump(corpus, d)
    else:
        with open("corpus.pickle", "rb") as d:
            corpus = pickle.load(d)
    return corpus

def clean_text(text):
    text = re.sub(r'[^\w\s]', '', text)
    text = re.sub(r'\d', '', text)
    text = re.sub(r'[A-Za-z]', '', text)
    text = [m.normal_forms(x)[0] for x in text.split() if x not in stopwords]
    return text

def bm25_matrix(all_ns, docs_len, tf_matrix, corpus, doc_words, avgdl, N):
    k = 2.0
    b = 0.75
    bm_matrix = []
    for i, word in enumerate(doc_words):
        bm_matrix.append([])
        for doc in corpus:
            bm = 0
            if word in doc:
                w_i = doc_words.index(word)
                d_i = corpus.index(doc)
                doc_len = docs_len[doc]
                tf = tf_matrix[w_i][d_i]/doc_len
                n = all_ns[word]
                idf = log((N-n+0.5)/(n+0.5))
                bm = idf*((tf*(k+1))/(tf+k*(1-b+(b*(doc_len/avgdl)))))
            bm_matrix[i].append(bm)
    return np.array(bm_matrix)

def upload_bm25_matrix(all_ns, docs_len, tf_matrix, corpus, doc_words, avgdl, N):
    if os.path.exists('bm25.pickle'):
        with open("bm25.pickle", "rb") as d:
            bm_matrix = pickle.load(d)
    else:
        bm_matrix = bm25_matrix(all_ns, docs_len, tf_matrix, corpus, doc_words, avgdl, N)
        with open("bm25.pickle", "wb") as d:
            pickle.dump(bm_matrix, d)
    return bm_matrix

def vect_bm25(query, k, doc_words, all_ns, N):
    q_words = clean_text(query)
    counts = {}
    for word in q_words:
        if word not in counts:
            counts[word] = 1
        else:
            counts[word] += 1
    v_query = []
    for word in doc_words:
        bm = 0
        if word in counts:
            doc_len = len(q_words)
            n = all_ns[word]
            tf = counts[word]/doc_len
            idf = log((N+1)/(n+1))
            bm = idf*((tf*(k+1))/(tf+k))
        v_query.append(bm)
    v_query = np.array(v_query)
    return v_query

def bm25_search(query):
    docs = get_docs()
    corpus = get_corpus(docs)
    N = len(corpus)
    X = vectorizer.fit_transform(corpus)
    f_matrix = X.toarray()
    tf_matrix = np.transpose(f_matrix)
    doc_words = vectorizer.get_feature_names()
    all_ns = {}
    for word in doc_words:
        w_i = doc_words.index(word)
        all_ns[word] = np.count_nonzero(tf_matrix[w_i])
    docs_len = {}
    whole_len = 0
    for doc in corpus:
        doc_len = len(doc.split())
        docs_len[doc] = doc_len
        whole_len += doc_len
    avgdl = whole_len/N
    doc_words = vectorizer.get_feature_names()
    bm_matrix = upload_bm25_matrix(all_ns, docs_len, tf_matrix, corpus, doc_words, avgdl, N)
    k = 2.0
    b = 0.75
    q_vect = vect_bm25(query, k, doc_words, all_ns, N)
    doc_score = q_vect.dot(bm_matrix)
    response = list(zip(docs, doc_score))
    response = sorted(response,key=itemgetter(1), reverse = True)
    return response


#!/usr/bin/env python3

import os
import connexion
import asyncio
from aiohttp import web

from .extensions import db_queue, db, listen_for_writes
import api_server.config as config

loop = asyncio.get_event_loop()

async def db_init(loop):
    try:
        await db.set_bind(config.DATABASE_URL)
        await db.gino.create_all()
    except:
        raise Exception

def main():
    options = {
        "swagger_ui": True
        }
    connexion_app = connexion.AioHttpApp(__name__, options=options, only_one_api=True)
    connexion_app.add_api('openapi.yaml',
                arguments={'title': 'url-shortener'},
                pythonic_params=True,
                pass_context_arg_name='request')

    loop.run_until_complete(db_init(loop))

    async def start_background_tasks(app):
        app['database_writes'] = asyncio.create_task(listen_for_writes(app))

    async def cleanup_background_tasks(app):
        app['database_writes'].cancel()
        await app['database_writes']

    connexion_app.app.on_startup.append(start_background_tasks)
    connexion_app.app.on_cleanup.append(cleanup_background_tasks)

    connexion_app.run(port=8080, debug=True)

if __name__ == '__main__':
    main()


from django.urls import reverse
from django.forms.models import model_to_dict
from django.utils.translation import ugettext as _

from messages import tasks as notifier
from teams.models import (
    Team, TeamMember, Application, Project
)
from teams.permissions import roles_user_can_assign, save_role
from utils import translation
from utils.rpc import Error, Msg, RpcRouter

class TeamsApiClass(object):
    def create_application(self, team_id, msg, user):
        if not user.is_authenticated():
            return Error(_('You should be authenticated.'))

        try:
            if not team_id:
                raise Team.DoesNotExist
            team = Team.objects.get(pk=team_id)
        except Team.DoesNotExist:
            return Error(_('Team does not exist'))

        try:
            TeamMember.objects.get(team=team, user=user)
            return Error(_(u'You are already a member of this team.'))
        except TeamMember.DoesNotExist:
            pass

        if team.is_open():
            TeamMember(team=team, user=user).save()
            return Msg(_(u'You are now a member of this team.'))
        elif team.is_by_application():

            if msg.strip() == '':
                return Error(_(u'The "About you" field is required in order to apply.'))
            try:
                application = team.applications.get(user=user)
                if application.status == Application.STATUS_PENDING:
                    return Error(_(u'You have already applied to this team.'))
                elif application.status == Application.STATUS_DENIED:
                    return Error(_(u'Your application has been denied.'))
                elif application.status == Application.STATUS_MEMBER_REMOVED:
                    return Error(_(u'You have been removed from this team.'))
                elif application.status == Application.STATUS_MEMBER_LEFT:
                    # the user chose to participate, so we can already approve it
                    application.note = msg
                    application.approve(author=application.user, interface='web UI')
                    return Msg(_(u"Your application has been approved. "
                         u"You are now a member of this team"))
            except Application.DoesNotExist:
                application = Application(team=team, user=user)
            application.note = msg
            application.save(author=user, interface='web UI')
            notifier.application_sent.delay(application.pk)

            return Msg(_(u"Your application has been submitted. "
                         u"You will be notified of the team administrator's response"))
        else:
            return Error(_(u"You can't join this team by application."))

    def promote_user(self, team_id, member_id, role, user):
        try:
            team = Team.objects.for_user(user).get(pk=team_id)
        except Team.DoesNotExist:
            return Error(_(u'Team does not exist.'))

        if not team.is_manager(user):
            return Error(_(u'You are not manager of this team.'))

        if not role in dict(TeamMember.ROLES):
            return Error(_(u'Incorrect team member role.'))

        try:
            tm = TeamMember.objects.get(pk=member_id, team=team)
        except TeamMember.DoesNotExist:
            return Error(_(u'Team member does not exist.'))

        if tm.user == user:
            return Error(_(u'You can\'t promote yourself.'))

        tm.role = role
        tm.save()
        return Msg(_(u'Team member role changed.'))

TeamsApi = TeamsApiClass()

def _project_to_dict(p):
    d  = model_to_dict(p, fields=["name", "slug", "order", "description", "pk",
                                  "workflow_enabled"])
    d.update({
        "pk":p.pk,
        "url": reverse("teams:project_video_list", kwargs={
            "slug":p.team.slug,
            "project_slug": p.slug,
        })
    })
    return d

class TeamsApiV2Class(object):
    def test_api(self, message, user):
        return Msg(u'Received message: "%s" from user "%s"' % (message, unicode(user)))

    def member_role_info(self, team_slug, member_pk, user):
        team = Team.objects.get(slug=team_slug)
        member = team.members.get(pk=member_pk)
        roles =  roles_user_can_assign(team, user, member.user)
        # massage the data format to make it easier to work with
        # over the client side templating
        verbose_roles = [{"val":x[0], "name":x[1]} for x in TeamMember.ROLES if x[0] in roles]
        narrowings = member.narrowings.all()

        current_languages = [n.language for n in narrowings if n.language]
        current_projects = [n.project for n in narrowings if n.project]

        projects = []
        for p in Project.objects.for_team(team):
            data = dict(pk=p.pk, name=p.name)
            if p in current_projects:
                data['selected'] = "selected"
            projects.append(data)

        langs = []
        writeable_languages = team.get_writable_langs()
        for code, name in [l for l in translation.SUPPORTED_LANGUAGE_CHOICES if l[0] in writeable_languages]:
            lang = {
                'selected': True if code in current_languages else False,
                'code': code,
                'name': name,
            }
            langs.append(lang)

        langs.sort(key=lambda l: unicode(l['name']))

        return {
            'current_role': member.role,
            'roles': verbose_roles,
            'languages': langs,
            'projects': projects,
        }

    def save_role(self, team_slug, member_pk, role, projects, languages, user=None):
        team = Team.objects.get(slug=team_slug)
        member = team.members.get(pk=member_pk)

        if role == 'admin':
            languages = None

        projects = map(int, projects or [])
        res = save_role(team, member, role, projects, languages, user)
        if res:
            return { 'success': True }
        else:
            return { 'success': False,
                     'errors': [_('You cannot assign that role to that member.')] }

TeamsApiV2 = TeamsApiV2Class()

rpc_router = RpcRouter('teams:rpc_router', {
    'TeamsApi': TeamsApi,
    'TeamsApiV2': TeamsApiV2,
})


# Weichao Qiu @ 2018
# Visualize arm dataset

import unreal.virtual_db as vdb
import imageio as io
import matplotlib.pyplot as plt
import numpy as np

joint_name = vdb.meta.arm_skel

def fov2f(fov, width):
    # fov = 2 * arctan(w / 2f)
    fov = fov * np.pi / 180
    return width / (2 * np.tan(fov / 2))

def f2fov(f, width):
    # fov = 2 * arctan(w / 2f)
    fov = 2 * np.arctan(width / (2 * f))
    fov = fov * 180 / np.pi
    return fov

def get_vertex_2d_raw(dataset, ids, cam, actor_name):
    vertexs = dataset.get_d3_vertex(ids)
    #actor_name = "OWI-arm-baked-pivot_2";
    if isinstance(vertexs[0], dict): #fit to the new data generation tool
        vertex = vertexs[0][actor_name]
    else:
        vertex = vertexs[0]
    points3d = np.array([[v['X'], v['Y'], v['Z']] for v in vertex])
    points2d = cam.project_to_2d(points3d)
    return points2d

def get_vertex_2d(dataset, frame_id, cam_name, actor_name):
    cams = [cam_name]
    ids = [frame_id]

    images = dataset.get_image(cams, ids); image_dir = images[0]
    cams_info = dataset.get_cam_info(cams, ids); cam_info = cams_info[0]

    loc = cam_info['Location']
    rot = cam_info['Rotation']
    cam = vdb.d3.CameraPose(loc['X'], loc['Y'], loc['Z'],
        rot['Pitch'], rot['Yaw'], rot['Roll'],
        cam_info['FilmWidth'], cam_info['FilmHeight'], cam_info['FilmWidth']/2)

    points2d = get_vertex_2d_raw(dataset, ids, cam, actor_name)
    return points2d, image_dir

def get_joint_2d_raw(dataset, ids, cam, actor_name):
    #joint_name = vdb.meta.arm_skel
    joints = dataset.get_d3_skeleton(ids)
    #actor_name = "OWI-arm-baked-pivot_2";
    joint = joints[0][actor_name]['WorldJoints'] # This usage might change
    # joint = dict({k.lower(): v for (k, v) in joint.items()}) # Convert key to lower case
    # Convert to np array
    points3d = np.array([[joint[v]['X'], joint[v]['Y'], joint[v]['Z']] for v in joint_name])
    points2d = cam.project_to_2d(points3d)
    return points2d

def get_joint_2d(dataset, frame_id, cam_name, actor_name):
    ids = [frame_id]
    cams = [cam_name]

    images = dataset.get_image(cams, ids); image_dir = images[0]

    # Project to 2D joint
    # ---------------------------------------------------------------------
    cams_info = dataset.get_cam_info(cams, ids); cam_info = cams_info[0]

    loc = cam_info['Location']
    rot = cam_info['Rotation']
    cam = vdb.d3.CameraPose(loc['X'], loc['Y'], loc['Z'],
        rot['Pitch'], rot['Yaw'], rot['Roll'],
        cam_info['FilmWidth'], cam_info['FilmHeight'], cam_info['FilmWidth']/2)

    #print(points3d)
    points2d = get_joint_2d_raw(dataset, ids, cam, actor_name)

    return points2d, image_dir

def get_joint_vertex_2d(dataset, frame_id, cam_name, actor_name):
    ids = [frame_id]
    cams = [cam_name]

    images = dataset.get_image(cams, ids); image_dir = images[0]

    # Project to 2D joint
    # ---------------------------------------------------------------------
    cams_info = dataset.get_cam_info(cams, ids); cam_info = cams_info[0]

    if cam_info.__contains__('Fov'):
        f = fov2f(cam_info['Fov'], cam_info['FilmWidth'])
    else:
        f = fov2f(90, cam_info['FilmWidth'])

    loc = cam_info['Location']
    rot = cam_info['Rotation']
    cam = vdb.d3.CameraPose(loc['X'], loc['Y'], loc['Z'],
        rot['Pitch'], rot['Yaw'], rot['Roll'],
        cam_info['FilmWidth'], cam_info['FilmHeight'], f)

    joint_2d = get_joint_2d_raw(dataset, ids, cam, actor_name)
    vertex_2d = get_vertex_2d_raw(dataset, ids, cam, actor_name)
    return joint_2d, vertex_2d, image_dir

def show_frame(dataset, frame_id, cam_name, actor_name):
    ids = [frame_id]
    cams = [cam_name]

    points2d, _ = get_joint_2d(dataset, frame_id, cam_name, actor_name)

    # Visualize 2D joint
    # ---------------------------------------------------------------------
    images = dataset.get_image(cams, ids); image = images[0]

    plt.subplot(121)
    plt.imshow(io.imread(image))
    plt.plot(points2d[:,0], points2d[:,1], '*')
    for edge in vdb.meta.arm_skel_edge:
        plt.plot(points2d[edge,0], points2d[edge,1], 'r-')

    # Load 3D vertex data and project to 2d
    # ---------------------------------------------------------------------
    points2d = get_vertex_2d(dataset, frame_id, cam_name, actor_name)

    plt.subplot(122)
    plt.imshow(io.imread(image))
    plt.plot(points2d[:,0], points2d[:,1], '*')

    # Show plot
    plt.show()

def main():
    db_root_dir = '/mnt/c/qiuwch/ue4_project/CarAct/OWIMap'

    cams = ['FusionCameraActor_1', 'FusionCameraActor2',
        'FusionCameraActor3', 'FusionCameraActor4']

    dataset = vdb.Dataset(db_root_dir)
    ids = dataset.get_ids() # The frame ids we have.
    print('Number of images %d' % len(ids))

    show_frame(dataset, ids[0], cams[3])

if __name__ == '__main__':
    main()


import time
import board
import digitalio
import usb_hid
from adafruit_hid.keyboard import Keyboard
from adafruit_hid.keycode import Keycode
from custom_keycodes import CustomKeycode

# Define GPIO pins for buttons
button_pins = [
    board.GP0,   # Button 0
    board.GP1,   # Button 1
    board.GP2,   # Button 2
    board.GP3,   # Button 3
    board.GP4,   # Button 4
    board.GP6,   # Button 5
    board.GP7,   # Button 6
    board.GP26,  # Button 7
    board.GP27   # Button 8
]

# Define key combinations for each button
button_key_combinations = [
    CustomKeycode.CLOSE_BRACE,                  # Button 0  }
    CustomKeycode.LESS_THAN,                    # Button 1  <
    CustomKeycode.GREATER_THAN,                 # Button 2  >
    [Keycode.LEFT_ALT, Keycode.F5],             # Button 3  Debug
    [Keycode.F11],                              # Button 4  Step into
    [Keycode.F5],                               # Button 5  Resume program
    CustomKeycode.OPEN_BRACE,                   # Button 6  {
    [Keycode.F10],                              # Button 7  Step over
    [Keycode.LEFT_ALT, CustomKeycode.ENTER]     # Button 8  Alt + Enter
]

# Button init
buttons = [digitalio.DigitalInOut(pin) for pin in button_pins]
for button in buttons:
    button.direction = digitalio.Direction.INPUT
    button.pull = digitalio.Pull.UP

button_state = {i: False for i in range(len(button_pins))}
keyboard = Keyboard(usb_hid.devices)
keyboard.release_all()

print("MacroPad initialized")

while True:
    for i, button in enumerate(buttons):
        if not button.value:
            # Pressed
            if not button_state[i]:
                for keycode in button_key_combinations[i]:
                    keyboard.press(keycode)
                print(f"Button {i} pressed")
                button_state[i] = True
        else:
            # Released
            if button_state[i]:
                for keycode in button_key_combinations[i]:
                    keyboard.release(keycode)
                print(f"Button {i} released")
                button_state[i] = False

    time.sleep(0.05)


from datetime import datetime
from typing import Dict, List

from pydantic import BaseModel

class CreateUrl(BaseModel):
    url: str

class ClickInfo(BaseModel):
    timestamp: datetime
    referrer: str

class UrlBasics(BaseModel):
    id: str
    url: str
    created_at: datetime

class UrlEntry(UrlBasics):
    clicks: List[ClickInfo] = []

    def dict(self, *args, **kwargs):
        url_entry_dict = super().dict(*args, **kwargs)
        url_entry_dict["created_at"] = url_entry_dict["created_at"].isoformat()
        for click in url_entry_dict["clicks"]:
            click["timestamp"] = click["timestamp"].isoformat()
        return url_entry_dict

class StatisticSummary(UrlBasics):
    total_clicks: int
    clicks_per_day: Dict[str, int]
    referrers: Dict[str, int]


import os, datetime
from ucf_converter import *

from command_parameters import *

current_directory = os.getcwd().replace('Script', 'Trees')
current_directory = os.path.join(current_directory, datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S'))
os.makedirs(current_directory)

from mlc_script_log import *
logger = Logger(current_directory, "")
logging.info("Current directory: " + current_directory)

arff_filename = os.path.join(current_directory, "features.arff")

# private import:
import mlc_configurator
from mlc_configurator import *
import decision_tree_generator
from decision_tree_generator import generateDecisionTree
from decision_tree_generator import generate_subset_of_ARFF
import mlc_test
from mlc_test import *
from arff_generator import *

device_name = "LSM6DSOX"    ## list of supported devices available with mlc_configurator.get_devices()

# device settings
#mlc_odr = "26 Hz"                        ## list of allowed values available with mlc_configurator.get_mlc_odr(device_name)
#input_type = "accelerometer+gyroscope"   ## list of allowed values available with mlc_configurator.get_mlc_input_type(device_name)
#accelerometer_fs = "2 g"                 ## list of allowed values available with mlc_configurator.get_accelerometer_fs(device_name)
#accelerometer_odr = "26 Hz"              ## list of allowed values available with mlc_configurator.get_accelerometer_odr(device_name)
#gyroscope_fs = "2000 dps"                ## list of allowed values available with mlc_configurator.get_gyroscope_fs(device_name)
#gyroscope_odr = "26 Hz"                  ## list of allowed values available with mlc_configurator.get_gyroscope_odr(device_name)
#window_length = 39                       ## Window length (supported values: from 1 to 255)

#####
# ARGUMENTS MANAGE
# Parsing argument
cmd_checking = cmd_parameters()
window_length, mlc_odr, accelerometer_odr, accelerometer_fs, gyroscope_fs, gyroscope_odr, input_type, _name = cmd_checking.check_args_value()

#############################
ucf_filename = os.path.join(current_directory, _name +".ucf")
h_filename = os.path.join(current_directory, _name +".h")

result_names  = [] # leave empty here
result_values = [] # leave empty here
datalog_results = [] # leave empty here

# Load class names (folder names) from Logs folder
direct = os.listdir("../Logs/")
print ("available classes = ", direct)

#filenames = next(os.walk(direct), (None, None, []))[1] # 2: Get all files | 1: Get all directories
class_names = list(filter(lambda file: not("DS_Store") in file, direct))

# For each class (folder), load all data (files in the folder)
datalogs = []
datalogs_split_by_class = []
for class_name in class_names:
    datalogs_i = os.listdir("../Logs/" + class_name +"/")
    print(class_name, " --> data logs: ", datalogs_i)
    datalogs_split_by_class.append(datalogs_i)
    for datalog_i in datalogs_i:
        datalogs.append("../Logs/" + class_name + "/" + datalog_i)
        datalog_results.append(class_name);
print("All data logs: ", datalogs)
print("All data logs 0: ", datalogs[0])

# Assign results and values for decision tree 1:
result_names.append(  class_names )
result_values.append( list(range(0, len(class_names), 1)) )
# Assign results and values for decision tree 2:
result_names.append(  [] )
result_values.append( [] )
# Assign results and values for decision tree 3:
result_names.append(  [] )
result_values.append( [] )
# Assign results and values for decision tree 4:
result_names.append(  [] )
result_values.append( [] )
# Assign results and values for decision tree 5:
result_names.append(  [] )
result_values.append( [] )
# Assign results and values for decision tree 6:
result_names.append(  [] )
result_values.append( [] )
# Assign results and values for decision tree 7:
result_names.append(  [] )
result_values.append( [] )
# Assign results and values for decision tree 8:
result_names.append(  [] )
result_values.append( [] )

dectree_filenames = []
for i in range(0,8):
    if not result_names[i]:
        break
    else:
        dectree_filenames.append(os.path.join(current_directory, "dectree{}.txt".format(i+1)))
n_decision_trees = i
logging.info('n_decision_trees = %d' % (n_decision_trees))

############

# filters configuration
# the list of available filters can be obtained with mlc_configurator.get_filter_names(input_type)
filters_list = []
#filters_list.append(mlc_configurator.mlc_filter("filter_1", "HP_Acc_XYZ"))
#filters_list.append(mlc_configurator.mlc_filter("filter_2", "BP_Acc_V", coef_a2=-0.5, coef_a3=0.5, coef_gain=1))
#filters_list.append(mlc_configurator.mlc_filter("filter_3", "IIR1_Acc_V", coef_b1=1, coef_b2=0.25, coef_a2=0.75))
#filters_list.append(mlc_configurator.mlc_filter("filter_4", "IIR2_Acc_V", coef_b1=0.59, coef_b2=1.19, coef_b3=0.59, coef_a2=-1.01, coef_a3=0.36))

# features configuration
# full list of available features can be obtained with mlc_configurator.get_feature_names()
# full list of available inputs can be obtained with   mlc_configurator.get_mlc_inputs(device_name, input_type)
features_list = []

arff_calculus = Arff_generator()

composantes = ["Acc_X", "Acc_Y", "Acc_Z", "Gyr_X", "Gyr_Y", "Gyr_Z"]
if input_type == 'accelerometer_only':
    composantes = ["Acc_X", "Acc_Y", "Acc_Z"]
arff_calculus.set_composantes(composantes)

for c in composantes:
  features_list.append(mlc_configurator.mlc_feature("MEAN", c))
  #features_list.append(mlc_configurator.mlc_feature("MINIMUM", c))
  #features_list.append(mlc_configurator.mlc_feature("MAXIMUM", c))
  #features_list.append(mlc_configurator.mlc_feature("PEAK_TO_PEAK", c))
  #features_list.append(mlc_configurator.mlc_feature("VARIANCE", c))
  #features_list.append(mlc_configurator.mlc_feature("ENERGY", c))

"""
features_list.append(mlc_configurator.mlc_feature("ZERO_CROSSING", "Acc_V", 0.5))
features_list.append(mlc_configurator.mlc_feature("POSITIVE_ZERO_CROSSING", "Acc_V", 0.5))
features_list.append(mlc_configurator.mlc_feature("NEGATIVE_ZERO_CROSSING", "Acc_V", 0.5))
features_list.append(mlc_configurator.mlc_feature("PEAK_DETECTOR", "Acc_V", 0.5))
features_list.append(mlc_configurator.mlc_feature("POSITIVE_PEAK_DETECTOR", "Acc_V", 0.5))
features_list.append(mlc_configurator.mlc_feature("NEGATIVE_PEAK_DETECTOR", "Acc_V", 0.5))
features_list.append(mlc_configurator.mlc_feature("ZERO_CROSSING", "Gyr_V", 0.5))
features_list.append(mlc_configurator.mlc_feature("POSITIVE_ZERO_CROSSING", "Gyr_V", 0.5))
features_list.append(mlc_configurator.mlc_feature("NEGATIVE_ZERO_CROSSING", "Gyr_V", 0.5))
features_list.append(mlc_configurator.mlc_feature("PEAK_DETECTOR", "Gyr_V", 0.5))
features_list.append(mlc_configurator.mlc_feature("POSITIVE_PEAK_DETECTOR", "Gyr_V", 0.5))
features_list.append(mlc_configurator.mlc_feature("NEGATIVE_PEAK_DETECTOR", "Gyr_V", 0.5))
"""

# Some examples of features on filtered inputs:
#features_list.append(mlc_configurator.mlc_feature("MEAN","Acc_X_filter_1"))
#features_list.append(mlc_configurator.mlc_feature("ENERGY", "Acc_V_filter_2"))
#features_list.append(mlc_configurator.mlc_feature("ZERO_CROSSING", "Acc_V_filter_3", 0.5))

mlc_configurator.arff_generator( arff_calculus=arff_calculus,
               device_name = device_name,
               datalogs = datalogs,
               results = datalog_results,
               mlc_odr = mlc_odr,
               input_type = input_type,
               accelerometer_fs = accelerometer_fs,
               accelerometer_odr = accelerometer_odr,
               gyroscope_fs = gyroscope_fs,
               gyroscope_odr = gyroscope_odr,
               n_decision_trees = n_decision_trees,
               window_length = window_length,
               filters_list = filters_list,
               features_list = features_list,
               arff_filename = arff_filename,
               current_directory = current_directory)

###########################

if (n_decision_trees == 1):
    dectree_accuracy, dectree_nodes = decision_tree_generator.generateDecisionTree(
                                                        arff_filename = arff_filename,
                                                        dectree_filename = dectree_filenames[0],
                                                        arff_calculus = arff_calculus)
else:
    for i in range(n_decision_trees) :
        arff_filename_i = arff_filename + str(i+1)
        decision_tree_generator.generate_subset_of_ARFF( arff_filename = arff_filename,
                                                         arff_subset_filename = arff_filename_i,
                                                         classes_subset = result_names[i] )
        logging.info("\n# Decision Tree %d:" %(i+1))
        dectree_accuracy, dectree_nodes = decision_tree_generator.generateDecisionTree(
                                               arff_filename = arff_filename_i,
                                               dectree_filename = dectree_filenames[i],
                                               arff_calculus = arff_calculus )

###################

# Meta-classifiers
# metaclassifierX_values contains the end counter values of the meta classifier associated to the decision tree 'X'
# 4 end counter values are available in LSM6DSOX (the first 4 values in "metaclasifierX_values")
# 8 end counter values are available in LSM6DSRX/ISM330DHCX (the 8 values in "metaclasifierX_values")
# values allowed for end counters are from 0 to 14
metaclassifier1_values = "0,0,0,0,0,0,0,0"
metaclassifier2_values = "0,0,0,0,0,0,0,0"
metaclassifier3_values = "0,0,0,0,0,0,0,0"
metaclassifier4_values = "0,0,0,0,0,0,0,0"
metaclassifier5_values = "0,0,0,0,0,0,0,0"
metaclassifier6_values = "0,0,0,0,0,0,0,0"
metaclassifier7_values = "0,0,0,0,0,0,0,0"
metaclassifier8_values = "0,0,0,0,0,0,0,0"
metaclassifier_values = [metaclassifier1_values, metaclassifier2_values, metaclassifier3_values, metaclassifier4_values, metaclassifier5_values, metaclassifier6_values, metaclassifier7_values, metaclassifier8_values]

mlc_configurator.ucf_generator( arff_calculus = arff_calculus,
                                device_name = device_name,
                                arff_filename = arff_filename,
                                dectree_filenames = dectree_filenames,
                                result_names = result_names,
                                result_values = result_values,
                                metaclassifier_values = metaclassifier_values,
                                ucf_filename = ucf_filename,
                                current_directory = current_directory )

convert_ucf_to_h(_name, ucf_filename, h_filename)


#!/usr/bin/env python

import jpype
import jaydebeapi
import os
import warnings

commithash = os.environ['CI_COMMIT_SHA']
mysecret = os.environ['BENCHDBPASS']
avatica = os.environ['JAVA_AVATICA_CLASSPATH']

if "classpath" in locals() or "classpath" in globals():
    classpath = classpath + ":" + str(avatica)
else:
    classpath = str(avatica)

classpath = classpath + ':' + '/usr/share/java/postgresql.jar'

if not jpype.isJVMStarted():
    assert(avatica in classpath)
    jpype.startJVM(jpype.getDefaultJVMPath(), "-Djava.class.path=" + classpath)
else:
    warnings.warn("JVM already started. Keeping old JVM configuration.")

conn = jaydebeapi.connect('org.postgresql.Driver', 'jdbc:postgresql://db:5432/postgres', ['runner', mysecret])
curs = conn.cursor()

class NULL:
    def __str__(self):
        return "NULL"
    __repr__ = __str__

def escape(s):
    for c in "\"'":
        s = s.replace(c, "_")
    s = s.replace('\u221e', "Inf")
    s = ''.join([i if ord(i) < 128 else '?' for i in s]).encode('ascii')
    return s

def escape2(s):
    if s is None:
        return NULL()
    else:
        return str(escape(s))

curs.execute("INSERT INTO results_repo(commithash, repourl, branch) VALUES" +str((commithash, os.environ['CI_PROJECT_URL'], escape(os.environ['CI_COMMIT_REF_NAME']))))

class DEFAULT:
    def __str__(self):
        return "DEFAULT"

    __repr__ = __str__

host='http://localhost:8081'
proteus = jaydebeapi.connect(jclassname="org.apache.calcite.avatica.remote.Driver",
                   url="jdbc:avatica:remote:url=" + host + ";serialization=PROTOBUF",
                   driver_args={})

cursprot = proteus.cursor()
cursprot.execute('SELECT * FROM SessionTimings')
for t in cursprot.fetchall():
    t = tuple(int(x) if isinstance(x, (int, long)) else escape2(x) for x in t)
    try:
        curs.execute('INSERT INTO SessionTimings VALUES ' + str((os.environ['CI_PROJECT_URL'], commithash) + t))
    except:
        print('INSERT INTO SessionTimings VALUES ' + str((os.environ['CI_PROJECT_URL'], commithash) + t))
        raise


import requests

class FlightSearch:

    def __init__(self, some_apikey: str, some_date_from: str, some_date_to: str, some_IATA_from: str, some_IATA_to: str,
                 some_api_url: str):
        self.apikey = some_apikey
        self.date_from = some_date_from
        self.date_to = some_date_to
        self.fly_from = some_IATA_from
        self.fly_to = some_IATA_to
        self.url = some_api_url

    def make_headers(self):
        self.headers = {
            "apikey": self.apikey,
            "Content-Encoding": "gzip"
        }
        return self.headers

    def make_parameters(self):
        self.parameters = {
            "fly_from": self.fly_from,
            "fly_to": self.fly_to,
            "date_from": self.date_from,
            "date_to": self.date_to
        }
        return self.parameters

    def make_requests(self):
        """This method makes a request to KIWI.com API """
        response = requests.get(url=self.url, headers=self.headers, params=self.parameters)
        response.raise_for_status()
        kiwi_data = response.json()
        return kiwi_data


import unittest
import tempfile
import os
import shutil

from utils.vasp import (find_vasp_directories)

class TestFindVaspDirectories(unittest.TestCase):
    def setUp(self):
        self.temp_dir = tempfile.mkdtemp()
        self.create_test_files()

    def tearDown(self):
        shutil.rmtree(self.temp_dir)

    def create_test_files(self):
        dir1_path = os.path.join(self.temp_dir, "dir1")
        os.makedirs(dir1_path)

        vasp_log_path = os.path.join(dir1_path, "asdf.jpg")
        with open(vasp_log_path, "w") as file:
            file.write("This is a jpg file")

        incar_path = os.path.join(dir1_path, "pasdf.txt")
        with open(incar_path, "w") as file:
            file.write("This is random text file")

        dir2_path = os.path.join(self.temp_dir, "dir2")
        os.makedirs(dir2_path)

        vasp_log_path = os.path.join(dir2_path, "vasp.log")
        with open(vasp_log_path, "w") as file:
            file.write("This is vasp.log")

        potcar_path = os.path.join(dir2_path, "POTCAR")
        with open(potcar_path, "w") as file:
            file.write("This is POTCAR")

        dir3_path = os.path.join(self.temp_dir, "dir3")
        os.makedirs(dir3_path)

        incar_path = os.path.join(dir3_path, "INCAR")
        with open(incar_path, "w") as file:
            file.write("This is INCAR")

        kpoints_path = os.path.join(dir3_path, "KPOINTS")
        with open(kpoints_path, "w") as file:
            file.write("This is KPOINTS")

        outcar_path = os.path.join(dir3_path, "OUTCAR")
        with open(outcar_path, "w") as file:
            file.write("This is OUTCAR")

    def test_find_vasp_directories(self):
        parent_dir = self.temp_dir
        filenames = ["vasp.log", "INCAR", "POTCAR", "CONTCAR", "KPOINTS", "OUTCAR"]
        all_present = False
        extract_tarballs = True

        directories = find_vasp_directories(parent_dir, filenames, all_present, extract_tarballs)

        self.assertEqual(len(directories), 2)

        expected_dirs = ["dir2", "dir3"]
        for dir_name in expected_dirs:
            self.assertIn(dir_name, [os.path.basename(dir) for dir in directories])

    def test_find_vasp_directories_negative(self):
        # Create a temporary empty directory to test the negative case
        empty_dir = tempfile.mkdtemp()

        # Call the function with empty directory and all_present=True
        parent_dir = empty_dir
        filenames = ["vasp.log", "INCAR", "POTCAR", "CONTCAR", "KPOINTS", "OUTCAR"]
        all_present = True
        extract_tarballs = True

        directories = find_vasp_directories(parent_dir, filenames, all_present, extract_tarballs)

        # Assert that the function returns an empty list as there are no directories that meet the criteria
        self.assertEqual(len(directories), 0)

if __name__ == "__main__":
    unittest.main()


import atexit
import os
import csv
import time
import numpy as np
import torch
import pandas as pd
import pyesapi as api
import json
from ax.modelbridge.random import RandomModelBridge
from ax import Experiment
from typing import Dict, cast
from ax.core.data import Data
from ax.core.arm import Arm
from ax.core.types import TTrialEvaluation
from ax.core.generator_run import GeneratorRun
from ax.core.objective import Objective
from ax.core.optimization_config import OptimizationConfig
from ax.modelbridge.registry import Models
from bo_planner.opt_backend import OptSimulator, OptRunner
from bo_planner.metric import PlanQualityMetric
import bo_planner.opt_space as opt_space
from bo_planner.data_storage import DataStorage
import bo_planner.utils as utils

def main(n_batch, patient_id=None,
         course_id="C1", plan_id="sobol",
         model_select='FULLYBAYESIAN',
         goal_fn="./config/PQM/Rectum/PQM.csv",
         obj_fn="./config/prescriptions/Rectum/34D.csv",
         const_fn = "./config/prescriptions/Rectum/rectum_constraint.txt",
         n_init=1,
         batch_size=3,
         opt_dir=None,
         dvh_dir=None,
         **kwargs
         ):
    """Main routine for the automated planning.
    The total number of queried samples are `n_init + n_batch * batch_size`
    Args:
        n_batch: total number of trial batches
        patient_id: patient ID
        course_id: course ID
        plan_id: Plan ID
        model_select: BO model to use
        goal_fn: filename defining the clinical goals for PQM
        obj_fn: filename defining the objective space for TPS
        n_init: initial trial numbers, currently using Sobol sequences
        batch_size: size of each trial batch
    Return:
        None
    """
    config_fn = kwargs.pop("env_fn", "./config/env_config.json")
    with open(config_fn, "r") as buf:
         config = json.load(buf)
    tkwargs = {
        "dtype": torch.double,
        "device": torch.device("cuda" if torch.cuda.is_available() else "cpu")}
    goal_df = pd.read_csv(goal_fn)
    f_df, p_dict, f_dict = opt_space.parse_obj(obj_fn)
    c_str = opt_space.load_constraints(const_fn)
    struct_roi = {i.split("_")[0] for i in goal_df["Structure"]}
    # add BODY for further extractions of total dvh
    struct_roi.add("BODY")
    app = api.CustomScriptExecutable.CreateApplication("BayOpt")
    atexit.register(app.Dispose)
    # initialize simulator
    patient = app.OpenPatientById(patient_id)
    patient.BeginModifications()
    plan = patient.CoursesLot(course_id).PlanSetupsLot(plan_id)
    plan.SetCalculationModel(api.CalculationType.PhotonVolumeDose, config["DCA"])
    plan.SetCalculationModel(api.CalculationType.PhotonIMRTOptimization, config["OA"])
    plan.SetCalculationModel(api.CalculationType.PhotonVMATOptimization, config["OA"])
    plan.SetCalculationModel(api.CalculationType.PhotonLeafMotions, config["LMC"])
    plan.SetCalculationOption(config["DCA"], "UseGPU", "Yes")
    plan.SetCalculationOption(config["OA"], "UseGPU", "Yes")
    opt_simulator = OptSimulator(plan, struct_roi, fixed_objectives=f_dict)
    opt_runner = OptRunner(opt_simulator)
    opt_prefix = "./opt_res/{}".format(patient_id) + opt_dir
    dvh_prefix = "./opt_res/{}".format(patient_id) + dvh_dir
    if not os.path.exists(opt_prefix):
        os.makedirs(opt_prefix)
    if not os.path.exists(dvh_prefix):
        os.makedirs(dvh_prefix)
    save_data = DataStorage(opt_prefix, dvh_prefix, p_dict)  # data storage
    pars_random = []

    # initialize search space
    search_space = opt_space.form_space(p_dict, c_strs=c_str)
    plan_quality = PlanQualityMetric("PQM", goal_df, noise_sd=0)
    optimization_config = OptimizationConfig(objective=Objective(metric=plan_quality))
    print(f"doing {n_init + n_batch * batch_size} evaluations")
    experiment = Experiment(name="saasbo_planning",
                            search_space=search_space,
                            optimization_config=optimization_config,
                            runner=opt_runner)
    sobol = Models.SOBOL(search_space=experiment.search_space)
    experiment = opt_space.init_data(patient_id, experiment, n_init, mode="sobol", file='', generator=sobol)
    for trial_name, trial in experiment.trials.items():  # extract data from initial experiment
        pars_random.append(list(trial.arms[0].parameters.values()))
        save_data.save_dvh(trial.arms)   # Store the DVHs generated by SOBOL during initialization
    data = experiment.fetch_data()
    save_data.save_init_data(data, pars_random)   # save init data

    for i in range(n_batch):
        if model_select == 'FULLYBAYESIAN':
            model = Models.FULLYBAYESIAN(
               experiment=experiment,
               data=data,
               num_samples=256,
               warmup_steps=512,
               gp_kernel="matern",
               torch_device=tkwargs["device"],
               torch_dtype=tkwargs["dtype"],
               verbose=False,
               disable_progbar=True)
        elif model_select == 'GPEI':
            model = Models.GPEI(
                experiment=experiment,
                data=data,
            )
        elif model_select == 'SOBOL':
            model = Models.SOBOL(
                experiment=experiment,
                data=data,
            )
        generator_run = model.gen(batch_size)
        trial = experiment.new_batch_trial(generator_run=generator_run)
        trial.run()
        trial.complete()
        data = Data.from_multiple_data([data, trial.fetch_data()])
        new_value = trial.fetch_data().df["mean"].max()
        if model_select == 'FULLYBAYESIAN':
            save_data.save_lenscal_saas(model)
        elif model_select == 'GPEI':
            save_data.save_lenscale_gpei(model)
        save_data.save_opt_data(data, trial, batch_size, n_init, i)
        save_data.save_dvh(trial.arms)
        print(f"Iteration: {i}, Best in iteration {new_value:.3f}, Best so far: {data.df['mean'].max():.3f}")
    save_data.close_lenscale_file()

    best_params, arm_name = utils.extract_best_params(data, experiment)
    # last step: dose calculation and re-optimization with the best params
    data, trial = utils.calculate_plan(experiment, arm_name, best_params, data)
    save_data.save_cal_data(data, best_params)
    save_data.save_dvh(trial.arms, 'reopt-cal1')

    # reopt with calculated dose
    data, trial = utils.reopt_calculated_plan(experiment, best_params, data)
    save_data.save_cal_data(data, best_params)
    save_data.close_pars_file()
    save_data.save_dvh(trial.arms, 'reopt-cal2')
    app.SaveModifications()
    app.ClosePatient()
    app.Dispose()
    print("optimization completed")

if __name__ == "__main__":
    with open("./config/pat_names.txt", "r") as buf:
         pat_list = list(buf.readline().strip().split(" "))
    for pa in pat_list:
        main(n_batch=2, patient_id=pa, plan_id="test", model_select='SOBOL', opt_dir="/util_random/", dvh_dir="/dvh_random/")
        print(pa + ' sobol is ok !')
        main(n_batch=2, patient_id=pa, plan_id="test", model_select='GPEI', opt_dir="/util_gpei/", dvh_dir="/dvh_gpei/")
        print(pa + ' gpei is ok !')
        main(n_batch=2, patient_id=pa, plan_id="test", model_select='FULLYBAYESIAN', opt_dir="/util_saas/", dvh_dir="/dvh_saas/")
        print(pa + " saasbo is ok !")


import fnmatch
import click
import collections
import mlflow
import json
import time
import os.path

from ast import literal_eval
from tempfile import TemporaryDirectory
from tqdm import tqdm

from aim import Run, Image, Text, Audio

IMAGE_EXTENSIONS = ('jpg', 'bmp', 'jpeg', 'png', 'gif', 'svg')
HTML_EXTENSIONS = ('html',)
TEXT_EXTENSIONS = (
    'txt',
    'log',
    'py',
    'js',
    'yaml',
    'yml',
    'json',
    'csv',
    'tsv',
    'md',
    'rst',
    'jsonnet',
)

# Audio is not handled in mlflow but including here just in case
AUDIO_EXTENSIONS = (
    'flac',
    'mp3',
    'wav',
)

class RunHashCache:
    def __init__(self, repo_path, no_cache=False):
        self._cache_path = os.path.join(repo_path, 'mlflow_logs_cache')
        self._needs_refresh = False

        if no_cache and os.path.exists(self._cache_path):
            os.remove(self._cache_path)

        try:
            with open(self._cache_path) as FS:
                self._cache = json.load(FS)
        except Exception:
            self._cache = {}

    def get(self, run_id):
        return self._cache.get(run_id)

    def __setitem__(self, key: str, val: str):
        if not self._cache.get(key) == val:
            self._cache[key] = val
            self._needs_refresh = True

    def __getitem__(self, key: str):
        return self._cache[key]

    def refresh(self):
        if self._needs_refresh:
            with open(self._cache_path, 'w') as FS:
                json.dump(self._cache, FS)

def get_mlflow_experiments(client, experiment):
    if experiment is None:
        # process all experiments
        experiments = client.search_experiments()
    else:
        try:
            ex = client.get_experiment(experiment)
        except mlflow.exceptions.MlflowException:
            ex = client.get_experiment_by_name(experiment)
        if not ex:
            raise RuntimeError(f'Could not find experiment with id or name "{experiment}"')
        experiments = (ex,)

    return experiments

def get_aim_run(repo_inst, run_id, run_name, experiment_name, run_cache):
    if run_cache.get(run_id):
        aim_run = Run(
            run_hash=run_cache[run_id],
            repo=repo_inst,
            system_tracking_interval=None,
            capture_terminal_logs=False,
            experiment=experiment_name,
        )
    else:
        aim_run = Run(
            repo=repo_inst,
            system_tracking_interval=None,
            capture_terminal_logs=False,
            experiment=experiment_name,
        )
        run_cache[run_id] = aim_run.hash
    aim_run.name = run_name
    return aim_run

def collect_run_params(aim_run, mlflow_run):
    aim_run['mlflow_run_id'] = mlflow_run.info.run_id
    aim_run['mlflow_experiment_id'] = mlflow_run.info.experiment_id
    aim_run.description = mlflow_run.data.tags.get("mlflow.note.content")

    # Collect params & tags
    # MLflow provides "string-ified" params values and we try to revert that
    aim_run['params'] = _map_nested_dicts(_try_parse_str, mlflow_run.data.params)
    aim_run['tags'] = {
        k: v for k, v in mlflow_run.data.tags.items() if not k.startswith('mlflow')
    }

def collect_artifacts(aim_run, mlflow_run, mlflow_client, exclude_artifacts):
    if '*' in exclude_artifacts:
        return

    run_id = mlflow_run.info.run_id

    artifacts_cache_key = '_mlflow_artifacts_cache'
    artifacts_cache = aim_run.meta_run_tree.get(artifacts_cache_key) or []

    __html_warning_issued = False
    with TemporaryDirectory(prefix=f'mlflow_{run_id}_') as temp_path:
        artifact_loc_stack = [None]
        while artifact_loc_stack:
            loc = artifact_loc_stack.pop()
            artifacts = mlflow_client.list_artifacts(run_id, path=loc)

            for file_info in artifacts:
                if file_info.is_dir:
                    artifact_loc_stack.append(file_info.path)
                    continue

                if file_info.path in artifacts_cache:
                    continue
                else:
                    artifacts_cache.append(file_info.path)

                if exclude_artifacts:
                    exclude = False
                    for expr in exclude_artifacts:
                        if fnmatch.fnmatch(file_info.path, expr):
                            exclude = True
                            break
                    if exclude:
                        continue

                downloaded_path = mlflow_client.download_artifacts(run_id, file_info.path, dst_path=temp_path)
                if file_info.path.endswith(HTML_EXTENSIONS):
                    if not __html_warning_issued:
                        click.secho(
                            'Handler for html file types is not yet implemented.', fg='yellow'
                        )
                        __html_warning_issued = True
                    continue
                elif file_info.path.endswith(IMAGE_EXTENSIONS):
                    aim_object = Image
                    kwargs = dict(
                        image=downloaded_path,
                        caption=file_info.path
                    )
                    content_type = 'image'
                elif file_info.path.endswith(TEXT_EXTENSIONS):
                    with open(downloaded_path) as fh:
                        content = fh.read()
                    aim_object = Text
                    kwargs = dict(
                        text=content
                    )
                    content_type = 'text'
                elif file_info.path.endswith(AUDIO_EXTENSIONS):
                    audio_format = os.path.splitext(file_info.path)[1].lstrip('.')
                    aim_object = Audio
                    kwargs = dict(
                        data=downloaded_path,
                        caption=file_info.path,
                        format=audio_format
                    )
                    content_type = 'audio'
                else:
                    click.secho(
                        f'Unresolved or unsupported type for artifact {file_info.path}', fg='yellow'
                    )
                    continue

                try:
                    item = aim_object(**kwargs)
                except Exception as exc:
                    click.echo(
                        f'Could not convert artifact {file_info.path} into aim object - {exc}', err=True
                    )
                    continue
                aim_run.track(item, name=loc or 'root', context={'type': content_type})

            aim_run.meta_run_tree[artifacts_cache_key] = artifacts_cache

def collect_metrics(aim_run, mlflow_run, mlflow_client, timestamp=None):
    for key in mlflow_run.data.metrics.keys():
        metric_history = mlflow_client.get_metric_history(mlflow_run.info.run_id, key)
        if timestamp:
            metric_history = list(filter(lambda m: m.timestamp >= timestamp, metric_history))

        for m in metric_history:
            aim_run.track(m.value, step=m.step, name=m.key)

def convert_existing_logs(repo_inst, tracking_uri, experiment=None, excluded_artifacts=None, no_cache=False):
    client = mlflow.tracking.client.MlflowClient(tracking_uri=tracking_uri)

    experiments = get_mlflow_experiments(client, experiment)
    run_cache = RunHashCache(repo_inst.path, no_cache)
    for ex in tqdm(experiments, desc=f'Parsing mlflow experiments in {tracking_uri}', total=len(experiments)):
        runs = client.search_runs(ex.experiment_id)

        for run in tqdm(runs, desc=f'Parsing mlflow runs for experiment `{ex.name}`', total=len(runs)):
            # get corresponding `aim.Run` object for mlflow run
            aim_run = get_aim_run(repo_inst, run.info.run_id, run.info.run_name, ex.name, run_cache)
            # Collect params and tags
            collect_run_params(aim_run, run)

            # Collect metrics
            collect_metrics(aim_run, run, client)

            # Collect artifacts
            collect_artifacts(aim_run, run, client, excluded_artifacts)

    run_cache.refresh()

def _wait_forever(watcher):
    try:
        while True:
            time.sleep(24 * 60 * 60)  # sleep for a day
    except KeyboardInterrupt:
        watcher.stop()

def _map_nested_dicts(fun, tree):
    if isinstance(tree, collections.abc.Mapping):
        return {k: _map_nested_dicts(fun, subtree) for k, subtree in tree.items()}
    else:
        return fun(tree)

def _try_parse_str(s):
    assert isinstance(s, str), f'Expected a string, got {s} of type {type(s)}'
    try:
        return literal_eval(s.strip())
    except:  # noqa: E722
        return s


import json
import logging
import uuid

import aiohttp.web

from .jsonencoder import JSONEncoder

logger = logging.getLogger(__name__)

class WebSocketRPCProtocol:

    def __init__(self):
        self.service_container = {}
        self.service_allowed_methods = {}
        self.disconnect_handlers = {}
        self.clients_by_id = {}

    async def connect(self, request):
        socket = aiohttp.web.WebSocketResponse()
        await socket.prepare(request)

        client_id, client_data = await self.handshake(socket)
        self.clients_by_id[client_id] = socket

        session_data = {
            **request.cookies,
            **client_data
        }

        for service_name, service in self.service_container.items():
            await service.on_connected(client_id, session_data)

        await self.receive_messages(socket, client_id)

        return socket

    async def handshake(self, socket):
        async for msg in socket:
            if msg.type == aiohttp.WSMsgType.text:
                rpc_body = json.loads(msg.data)
                logger.debug("rpc body: %s", rpc_body)

                if rpc_body["type"] != "hello":
                    raise Exception("invalid hello message")

                client_id = rpc_body.get("client_id", str(uuid.uuid1()))
                hello_message = {
                    "type": "hello",
                    "client_id": client_id
                }

                await socket.send_str(json.dumps(hello_message))
                return client_id, {k: v for k, v in rpc_body.items() if k not in ("type", "client_id")}

            elif msg.type == aiohttp.WSMsgType.close:
                raise Exception("connection closed without explanation")

        raise Exception("no handshake received")

    async def receive_messages(self, socket, client_id):
        async for msg in socket:
            if msg.type == aiohttp.WSMsgType.text:
                try:
                    message_body = json.loads(msg.data)
                    if message_body["type"] != "rpc":
                        raise Exception("unhandled message of type %s" % message_body["type"])

                    rpc_body = message_body["rpc"]
                    service_name = rpc_body["service"]
                    method_name = rpc_body["method"]
                    service = self.service_container[service_name]

                    if method_name in self.service_allowed_methods[service_name]:
                        await getattr(service, method_name)(client_id, **rpc_body.get("params", {}))
                    else:
                        logger.warning("method %s is not allowed for service %s", method_name, service_name)
                except Exception as e:
                    logger.exception(str(e))
            elif msg.type == aiohttp.WSMsgType.close:
                pass

        for service_name, service in self.service_container.items():
            await service.on_disconnected(client_id)

    def register_service(self, service_instance, allowed_methods):
        service_name = service_instance.__class__.__name__
        self.service_container[service_name] = service_instance
        self.service_allowed_methods[service_name] = allowed_methods

    async def send_event(self, client_id, event_data):
        socket = self.clients_by_id[client_id]

        message = {
            "type": "event",
            "event": event_data }

        await socket.send_str(json.dumps(message, cls = JSONEncoder))


from ase.cluster.decahedron import Decahedron
from ase.io import write

size = [3,4,5,6]

for n in size:
    atoms = Decahedron('Au',
                       p=n,  # natoms on 100 face normal to 5-fold axis
                       q=n,  # natoms 0n 100 parallel to 5-fold axis
                       r=0,  # depth of the Marks re-entrance?
                       latticeconstant=4.16,
                       )

    length = len(atoms) # check length

    print(length)

    atoms.center(vacuum=10, axis=(0, 1, 2), about=None)

    write(f'Au{length}_deca.dat', atoms, format="lammps-data") # write NP to file


def get_shop_info(keyword):
    client_id = "JagKFffWu4Fj_OkksX_R"
    client_secret = "LD2ZmSF2j8"
    encText = urllib.parse.quote(keyword)
    shop_url = "https://openapi.naver.com/v1/search/shop?query=" +encText+ "&display=100&start=1"
    request = urllib.request.Request(shop_url)
    request.add_header("X-Naver-Client-Id", client_id)
    request.add_header("X-Naver-Client-Secret", client_secret)
    response = urllib.request.urlopen(request)
    rescode = response.getcode()
    if rescode == 200:
        response_body = response.read()
        json_str = response_body.decode('utf-8')
    else:
        response_body.decode('utf-8')
        print("Error Code: " + rescode)
    json_data = json.loads(json_str)
    df = pd.json_normalize(json_data['items'])
    df['title'] = df['title'].str.replace('<b>', '').str.replace('</b>', '')

    lprice_mean = df.loc[df['lprice'].notnull(), 'lprice'].astype(int).mean()

    mallName_cnt = df['mallName'].str.split(expand=True).stack().value_counts()
    mallName_dup_cnt = dict(mallName_cnt[:10])

    brand_cnt = df['brand'].str.split(expand=True).stack().value_counts()
    brand_dup_cnt = dict(brand_cnt[:10])

    return lprice_mean, mallName_dup_cnt, brand_dup_cnt,

print()
print("팔려고 하는 물품 상위 100개의 가격 및 상위 10개의 브랜드 명 과 쇼핑몰 플랫폼(쿠팡, 네이버) 확인 하기")
keyword = input("keyword 를 입력해주세요.\n : ")
lprice_mean, mallName_dup_cnt, brand_dup_cnt = get_shop_info(keyword)

print(f"상위 100개의 평균 가격 입니다. \n = {lprice_mean} 원")
print(f"상위 10개 쇼핑몰명 중복횟수 입니다. \n = {mallName_dup_cnt}")
print(f"상위 10개 브랜드명 중복횟수 입니다. \n = {brand_dup_cnt}")


# -*- coding: utf-8 -*-
"""
Created on Fri Feb 15 15:51:12 2019

@author: hjiang
"""

"""
The string "PAYPALISHIRING" is written in a zigzag pattern on a given number of rows like this:
    (you may want to display this pattern in a fixed font for better legibility)

P   A   H   N
A P L S I I G
Y   I   R
And then read line by line: "PAHNAPLSIIGYIR"

Write the code that will take a string and make this conversion given a number of rows:

string convert(string s, int numRows);
Example 1:

Input: s = "PAYPALISHIRING", numRows = 3
Output: "PAHNAPLSIIGYIR"
Example 2:
            01234567890123
Input: s = "PAYPALISHIRING", numRows = 4
Output: "PINALSIGYAHRPI"
Explanation:

P     I    N  0,6,12
A   L S  I G  1,5,7,11,13
Y A   H R
P     I
明眼人一看就知道，这个肯定是有公式的。我自己推导的公式和JustDoIT的一样：

(1)第一行和最后一行下标间隔都是interval = n*2-2;

(2)中间行的间隔是周期性的,第i行的间隔是: interval–2*i, 2*i, interval–2*i, 2*i, interval–2*i, 2*i, …

代码如下：
---------------------
作者：负雪明烛
来源：CSDN
原文：https://blog.csdn.net/fuxuemingzhu/article/details/80830509
版权声明：本文为博主原创文章，转载请附上博文链接！
"""
# Time:  O(n)
# Space: O(n)

class Solution:
    def convert(self, s, numRows):
        """
        :type s: str
        :type numRows: int
        :rtype: str
        """
        if numRows == 1: return s
        ans = ""
        interval = 2 * (numRows - 1)
        for i in range(0, len(s), interval):#第0行
            ans += s[i]
        for row in range(1, numRows - 1):
            inter = 2 * row
            i = row
            while i < len(s):
                ans += s[i]
                inter = interval - inter# interval-2*i; interval-(interval-2*i); ...
                i += inter
        for i in range(numRows - 1, len(s), interval):#最后一行
            ans += s[i]
        return ans


import random

class Ai_random:
    #check = None
    length = 0
    type = 0
    omok = None
    map = None

    def __init__(self, omok, length, t):
        self.length = length
        self.type = t
        self.omok = omok

    def put(self):
        while(True):
            x = random.randrange(0, self.length)
            y = random.randrange(0, self.length)
            if (self.omok.isPossable(x, y, self.type)):
                return self.omok.putStone(x, y, self.type)
            num = 0
            for i in range(0, self.length):
                for j in range(0, self.length):
                    if (self.omok.isPossable(i, j, self.type)):
                        num += 1
            if (num == 0):
                return -1