#usage of the tests:
# ```
#   from functions import *
#   test_final('your__testfile_name')
# ```

#mongoDB setup
import pymongo
from pymongo import MongoClient
from bson.objectid import ObjectId
client = MongoClient()
client = MongoClient('localhost', 27017)

#3 list XML's
import glob
import json
def list_xmls(glob_paths,out_file=None):
    files = []
    for path in glob_paths:
        files.extend(glob.glob(path+"*.tei.xml"))
    print("{} files".format(len(files)))
    for i,f in enumerate(files):
        #replace the "\" returned in the glob paths
        files[i] = f.replace('\\','/')

    files.reverse()
    caches = set()
    results = []
    #remove duplicates
    for item in files:
        prefix = item.rsplit('v', 2)[0]
        # check whether prefix already exists
        if prefix not in caches:
            results.append(item)
            caches.add(prefix)
    results.reverse()

    return(results)

#4 create fulltexts array
from contextlib import suppress
import grobid_tei_xml
from os import path

def create_texts(files, out_file=None):
    texts = []
    for file_id,file_path in enumerate(files):

        title = None
        authors = None
        text = None
        doi = None
        citation_count = None
        abstract = None

        with suppress(Exception):
            with open(file_path, 'r', encoding="utf8") as d_file:
                doc = grobid_tei_xml.parse_document_xml(d_file.read())
        with suppress(Exception):
            text = doc.body
        with suppress(Exception):
            title= doc.header.title
        with suppress(Exception):
            authors= ", ".join([a.full_name for a in doc.header.authors])
        with suppress(Exception):
            doi= str(doc.header.doi)
        with suppress(Exception):
            citation_count= str(len(doc.citations))
        with suppress(Exception):
            abstract= doc.abstract

        texts.append({
            "ID":file_id,
            "path":file_path,
            "text":text,
            "title":title,
            "authors":authors,
            "doi":doi,
            "citation_count":citation_count,
            "abstract":abstract
        })
        print("{} files without duplicates".format(len(texts)),end='\r')
    print('')
    if (out_file != None):
        with open('./'+out_file+'.json', 'w') as fout:
            json.dump(texts, fout)
    return(texts)

from nltk.tokenize import sent_tokenize
def create_phrses(texts,phrases_db_name):
    phr_db = client[phrases_db_name]
    db_all_phrases = phr_db.all_phrases
    all_phrases_on_db = db_all_phrases.count_documents({})
    all_phrases = []
    cont_file = 0
    for text in texts:
        cont_file = cont_file+1
        for section in text:
            if (section in ["text","title","abstract"]):
                if (text[section]!= None):
                    #print(text[section])
                    for phrase in sent_tokenize(text[section]):
                        phr = {
                            "_id":len(all_phrases),
                            "path":text["path"],
                            "phrase":phrase.lower(),
                            "lenght":len(phrase),
                            "section":section
                        }
                        phrase_id = db_all_phrases.insert_one(phr)
                        all_phrases.append(phrase_id)
                        print("file: {} ----,{} phrases in total".format(cont_file,len(all_phrases)),end='\r')
    return(all_phrases)

#search for texts containing any strings in a set
from collections import namedtuple
from langdetect import detect
import langdetect.lang_detect_exception as ldError

def search_texts(search_set,fields,db_name,phrases_db_name):
    db = client[db_name]
    phr_db = client[phrases_db_name]
    phrases = db.phrases
    phrases_on_db = phrases.count_documents({})
    if (phrases_on_db == 0):
        print("The database is empty. running your query...")
        matches = run_query(search_set,fields,db_name,phrases_db_name);
    else:
        valid_ans = bool(False)
        while (not valid_ans):
            redo = str(input("Database already has {} phrases, do you want to retry query? Y-n".format(phrases_on_db)))
            if (redo in {"Y","y"}):
                matches =run_query(search_set,fields,db_name,phrases_db_name);
                valid_ans = True
            elif(redo in {"N","n"}):
                cursor = phrases.find({})
                matches = [phrase['phrase'] for phrase in cursor]
                valid_ans = True
            else:
                print("Please answer with Y,y(Yes) or N,n(No)...")
    print('')
    return(matches)

def run_query(search_set,sections,db_name,phrases_db_name):
    db = client[db_name]
    phrases = db.phrases
    phrases.drop()
    phr_db = client[phrases_db_name]
    all_phrases = phr_db.all_phrases
    matches = []
    for phrase in all_phrases.find():
        if (phrase["section"] in sections):
            match_words = []
            for word in search_set:
                if (re.search(r"\b" + re.escape(word) + r"\b", phrase["phrase"].lower())):
                    if(len(match_words)>0):
                        phrase_id = phrases.update_one({"_id" :int(phrase["_id"])},{"$set" : {"match_word":match_words}})
                    else:
                        remove_non_alpha = re.compile('[^a-zA-Z \-]')
                        ratio =  len(remove_non_alpha.sub('',phrase["phrase"].lower()))/len(phrase["phrase"].lower())
                        if (ratio>0.8):
                            try:
                                lang = detect(phrase["phrase"])
                                if (lang=='en'):
                                    match = phrase
                                    match["a_id"] = match["_id"]
                                    match["_id"] = len(matches)
                                    match["match_word"] = [word]
                                    phrase_id = phrases.insert_one(match)
                                    #matches.append(Phrase(path=text["path"],phrase=phrase.lower(),lenght=len(phrase)))
                                    matches.append(phrase_id)
                                    print("{} phrase matches".format(len(matches)),end='\r')
                            except ldError.LangDetectException:
                                pass
                    match_words.append(word)
    return(matches)

#5 remove stopwords & clean text
from sklearn.feature_extraction.text import CountVectorizer
import pandas as pd
import numpy as np
import nltk
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer

nltk.download('stopwords')
nltk.download('punkt')
from nltk.tokenize import word_tokenize
import re
remove_non_alpha = re.compile('[^a-zA-Z \-]')

def clean_text(db_name,remove_w=[],join_w=[],stem=False):
    db = client[db_name]
    phrases = db.phrases
    clean_phrases = []
    cursor = phrases.find({})
    rm_w_count =0
    join_w_count =0
    for n,phrase in enumerate(cursor):
        #get phrase text (sentence)
        sentence = phrase["phrase"]

        #remove words from a list.
        for rm_word in remove_w:
            while (re.search(r"\b" + re.escape(rm_word) + r"\b", sentence)):
                rm_w_count = rm_w_count+1
                sentence = re.sub(r"\b" + re.escape(rm_word) + r"\b","", sentence)

        #join words from a list.
        for join_word in join_w:
            while (re.search(r"\b" + re.escape(join_word) + r"\b", sentence)):
                join_w_count = join_w_count+1
                hyphen_word = join_word.replace(" ", "-")
                if(join_word == hyphen_word):
                    break
                sentence = re.sub(r"\b" + re.escape(join_word) + r"\b",hyphen_word, sentence)

        #remove non_alphanumeric characters
        sentence = remove_non_alpha.sub('',sentence.lower())

        #tokenize sentence
        tokens = word_tokenize(sentence)

        #remove Punctuation (already done)
        #words = [word for word in tokens if word.isalpha()]

        #remove stopwords
        stop_words = set(stopwords.words('english'))
        words = [w for w in tokens if not w in stop_words]

        #stem words
        if(stem == True):
            from nltk.stem.porter import PorterStemmer
            porter = PorterStemmer()
            words = [porter.stem(word) for word in tokens]

        #un-tokenize string
        cleant = " ".join(list(words))

        #append clean string to list
        clean_phrases.append(cleant)
        print(str(n)+"-"+str(len(tokens)),end='\r')
    print("{} palavras removidas".format(rm_w_count))
    print("{} palavras unidas".format(join_w_count))
    return(clean_phrases)

def clean_all_text(db_name,remove_w=[],join_w=[],stem=False):
    db = client[db_name]
    phrases = db.all_phrases
    clean_phrases = []
    cursor = phrases.find({})
    rm_w_count =0
    join_w_count =0
    for n,phrase in enumerate(cursor):
        remove_non_alpha = re.compile('[^a-zA-Z \-]')
        ratio =  len(remove_non_alpha.sub('',phrase["phrase"].lower()))/len(phrase["phrase"].lower())
        if (ratio>0.8):
            #get phrase text (sentence)
            sentence = phrase["phrase"]

            #remove words from a list.
            for rm_word in remove_w:
                while (re.search(r"\b" + re.escape(rm_word) + r"\b", sentence)):
                    rm_w_count = rm_w_count+1
                    sentence = re.sub(r"\b" + re.escape(rm_word) + r"\b","", sentence)

            #join words from a list.
            for join_word in join_w:
                while (re.search(r"\b" + re.escape(join_word) + r"\b", sentence)):
                    join_w_count = join_w_count+1
                    hyphen_word = join_word.replace(" ", "-")
                    if(join_word == hyphen_word):
                        break
                    sentence = re.sub(r"\b" + re.escape(join_word) + r"\b",hyphen_word, sentence)

            #remove non_alphanumeric characters
            sentence = remove_non_alpha.sub('',sentence.lower())

            #tokenize sentence
            tokens = word_tokenize(sentence)

            #remove Punctuation (already done)
            #words = [word for word in tokens if word.isalpha()]

            #remove stopwords
            stop_words = set(stopwords.words('english'))
            words = [w for w in tokens if not w in stop_words]

            #stem words
            if(stem == True):
                from nltk.stem.porter import PorterStemmer
                porter = PorterStemmer()
                words = [porter.stem(word) for word in tokens]

            #un-tokenize string
            cleant = " ".join(list(words))

            #append clean string to list
            clean_phrases.append(cleant)
            print(str(n)+"-"+str(len(tokens)),end='\r')
    print("")
    print("{} palavras removidas".format(rm_w_count))
    print("{} palavras unidas".format(join_w_count))
    return(clean_phrases)

#5 create n_gram and vocab
from sklearn.feature_extraction.text import CountVectorizer
def create_n_gram(phrases):
    doc = pd.DataFrame(phrases)
    doc.columns = ['text']
    vect = CountVectorizer()
    vects = vect.fit_transform(doc.text)
    return(vect,vects)

#clean n_gram (optional, DOES NOT WORK with porter stemmer)
def clean_n_gram():
    with open('./american-english-huge','r') as f:
            eng_words_huge=set()
            for line in f:
                    strip_lines=line.strip()
                    listli=strip_lines.split()
                    eng_words_huge.add(listli[0])

#6 run LDA
import lda

def run_lda(vect,vects,topic_number,db_name):
    db = client[db_name]
    phrases = db.phrases
    topics = db.topics
    X = vects.toarray()
    model = lda.LDA(n_topics=topic_number, n_iter=2500, random_state=1,refresh=10)
    model.fit(X)  # model.fit_transform(X) is also available
    #lista de palavras e probabilidades de cada tópico:
    n_words_on_topic = (model.nzw_ > 0).sum(axis=1)
    top_word_indices_of_topics = [(-model.nzw_[a]).argsort()[:n_words_on_topic[a]] for a in range(len(n_words_on_topic))]
    #save topics to the Db
    topics_on_db = topics.count_documents({})
    for i,top_word_indices in enumerate(top_word_indices_of_topics):
        w_p = [{"word":(vect.get_feature_names())[word],"prob":np.around(model.topic_word_,5)[i][word]} for word in top_word_indices]
        this_topic = {"_id":i,"word_probabilities":w_p}
        if (topics_on_db == 0):
            topics.insert_one(this_topic)
    #add topics to the phrases in the Db
    n_topics_on_document = (model.ndz_ > 0).sum(axis=1)
    top_topic_indices_of_documents = [(-model.ndz_[a]).argsort()[:n_topics_on_document[a]] for a in range(len(n_topics_on_document))]
    for i,top_topic_indices in enumerate(top_topic_indices_of_documents):
        doc_topics = [{"topic": int(topic),"prob":float(np.around(model.doc_topic_,5)[i][topic])} for topic in top_topic_indices]
        phrases.update_one({"_id" :int(i)},{"$set" : {"topics":doc_topics}})
    return(model)

#populate_DB
import datetime
import ast
import uuid
import timeit

def run_analysis():
    #setup mongo client
    client = MongoClient('localhost', 27017)
    #create/access history of
    hist_db = client.lda_arxiv_log
    history = hist_db.history
    entries = history.count_documents({})
    latest = history.find_one({"$query": {}, "$orderby": {"$natural" : -1}})
    if(latest == None):
        print("no Analysis was found in the database... \n we will have to create yours from scratch!")
        this_analysis = input_analysis()
    else:
        valid_ans = bool(False)
        while (not valid_ans):
            redo = input("Do you want to redo an analysis? Y-n")
            if(redo in {"Y","y"}):
                analysis_hist = [hi for hi in history.find({"$query": {}, "$orderby": {"$natural" : -1}})]
                nicknames_list = [hi["nick_name"] for hi in analysis_hist]
                print("these are all the analysis you made:")
                for hi in analysis_hist:
                    print(hi)
                valid_ans = bool(False)
                while (not valid_ans):
                    analysis_old_name = input("please give us the nickname of the analysis you want to redo:")
                    this_analysis = next((hi for hi in analysis_hist if hi["nick_name"] == analysis_old_name), None)
                    if(this_analysis != None):
                        valid_ans = True
                    else:
                        print("please write a valid nickname")
                valid_ans =True
            elif(redo in {"N","n",}):
                this_analysis = input_analysis()
                valid_ans =True
            else:
                print("Please answer with Y,y(Yes) or N,n(No)...")

    glob_paths_ex = ["./pdf/*/"]
    query_words_ex = {"artificial intelligence", "machine learning", "m.l.","a.i."}
    query_sections_ex = {"abstract","text"}
    all_phrases_db_name = "ALL_PHRASES_ARXIV"

    #CHECK IF ALL_PHRASES EXISTS
    phr_db = client[all_phrases_db_name]
    db_all_phrases = phr_db.all_phrases
    all_phrases_on_db = db_all_phrases.count_documents({})
    if(all_phrases_on_db == 0):
            print("###list_xmls###")
            files = list_xmls(this_analysis["docs_path_list"])
            print("###create_texts###")
            test_text = create_texts(files)
            print("###create_phrses###")
            all_phrases = create_phrses(test_text,all_phrases_db_name)
    else:
        cursor = db_all_phrases.find({})
        all_phrases = [phrase['_id'] for phrase in cursor]
        print("###{} phrses on DB###".format(len(all_phrases)))

    print("###search_texts###")
    test_phrases = search_texts(this_analysis["query"]["words"],this_analysis["query"]["sections"],this_analysis["db_name"],all_phrases_db_name)
    print("###clean_text###")
    test_clean = clean_text(this_analysis["db_name"],join_w=this_analysis["query"]["words"])
    print("###create_n_gram###")
    (vect,vects) = create_n_gram(test_clean)
    starttime = timeit.default_timer()
    print(">>> start time is :",starttime)
    model = run_lda(vect,vects,this_analysis["n_topics"],this_analysis["db_name"])
    print("<<< time difference is :", timeit.default_timer() - starttime)
    this_analysis["date"]= datetime.datetime.utcnow()
    this_analysis["_id"]= str(uuid.uuid4())
    history.insert_one(this_analysis)
    print("thanks, your analysis parameters are stored with the nickname: {}.".format(this_analysis["nick_name"]))

def input_analysis():
        hist_db = client.lda_arxiv_log
        history = hist_db.history
        this_analysis ={}
        print("The databadeses in your machine are:")
        dbs_in_machine = []
        for db in client.list_databases():
            print(db)
            dbs_in_machine.append(db["name"])
        database_name = "arxiv_LDA_{}".format(input("Write the name of the new (or old) database: arxiv_LDA_..."))
        this_analysis["db_name"] = database_name
        #get docs_path_list
        valid_ans = bool(False)
        while (not valid_ans):
            try:
                this_analysis["docs_path_list"] = ast.literal_eval(input('Write a list of paths where the documents are found: ["./in", "./this/way", "./pdf/*/"]') or '["./pdf/*/"]' )
                valid_ans = (type(this_analysis["docs_path_list"])==list)
                #check if paths exist...
            except:
                valid_ans = False
            if (valid_ans == False):print("please write your LIST of paths in pythonic syntax")

        this_analysis["query"] = {}
        valid_ans = bool(False)
        while (not valid_ans):
            try:
                this_analysis["query"]["words"] = ast.literal_eval(input('Write your set of query words: [ "in", "this way"]') or '["artificial intelligence", "machine learning", "m.l.","a.i."]')
                valid_ans = (type(this_analysis["query"]["words"])==list)
            except:
                valid_ans = False
            if (valid_ans == False):print("please write your LIST of words in pythonic syntax")

        valid_ans = bool(False)
        while (not valid_ans):
            try:
                this_analysis["query"]["sections"] = ast.literal_eval(input('Write your set of sections that you want to query: [ "in", "this way"]') or '["abstract","text"]')
                valid_ans = (type(this_analysis["query"]["sections"])==list)
            except:
                valid_ans = False
            if (valid_ans == False):print("please write your LIST of sections in pythonic syntax")

        valid_ans = bool(False)
        while (not valid_ans):
            try:
                this_analysis["n_topics"] = int(input("How many topics do you wish?") or 20)
                valid_ans = True
            except:
                valid_ans = False
            if (valid_ans == False):print("Number of topics must be an Integer")

        valid_ans = bool(False)
        nicknames_list = [hi["nick_name"] for hi in history.find({"$query": {}, "$orderby": {"$natural" : -1}})]
        while (not valid_ans):
            this_analysis["nick_name"] = input("we are almost there... Just give your analysis an nickname:")
            if (this_analysis["nick_name"] not in nicknames_list):
                valid_ans = True
            else:
                print("this nickname already exists please give it an Unique one!")
        return(this_analysis)


from extensions.module_result.models.primitives import OARS
from extensions.tests.module_result.IntegrationTests.test_samples import common_fields
from extensions.tests.module_result.UnitTests.primitives_tests import (
    PrimitiveTestCase,
    COMMON_FIELDS,
)

def oars_primitive():

    primitive_dict = {
        **common_fields(),
        OARS.OARS_SCORE: 50.7,
        OARS.PAIN_SCORE: 20.3,
        OARS.SLEEP_SCORE: 25.5,
        OARS.NAUSEA_SCORE: 67.0,
        OARS.MOBILITY_SCORE: 54.3,
    }

    primitive_dict.update(COMMON_FIELDS)

    return primitive_dict

class OARSPrimitiveTestCase(PrimitiveTestCase):
    def test_success_create(self):
        primitive_dict = oars_primitive()
        try:
            primitive = OARS.create_from_dict(oars_primitive(), name="OARS")
        except Exception:
            self.fail()
        self.assertIsNotNone(primitive)
        self.assertEqual(primitive.oarsScore, primitive_dict[OARS.OARS_SCORE])
        self.assertEqual(primitive.sleepScore, primitive_dict[OARS.SLEEP_SCORE])
        self.assertEqual(primitive.mobilityScore, primitive_dict[OARS.MOBILITY_SCORE])
        self.assertEqual(primitive.painScore, primitive_dict[OARS.PAIN_SCORE])
        self.assertEqual(primitive.nauseaScore, primitive_dict[OARS.NAUSEA_SCORE])


# database and gamecontroller imports

import smtplib
from time import sleep

from validate_email import validate_email

import ServerComponents.Suppurt.support as supp
import ChessAI.GameController.game_controller as game_controller
import Vector2d.Vector2d as vec

# server connection
import eventlet

from ChessBoard.chess_board import Board
from ChessBoard.chess_figure import Side
from ServerComponents.Suppurt.server import execute_no_res_async, execute_one_res_async, execute_all_res_async
from message_sender import send_email

eventlet.monkey_patch()
from flask import Flask, request
from flask_socketio import SocketIO

app = Flask(__name__)
socketio = SocketIO(app, async_mode='eventlet')
usersAddr = {}

clients = {}
user_client_map = {}
messages = {}

# email data
# server.starttls()
# server.login("chess.classic.official@gmail.com", "ChhessClassicc1488")

class Server:

    def __init__(self, port):
        self.port = port

    def run(self):
        execute_no_res_async("call chess.start_server()")
        socketio.start_background_task(self.message_queue_processing)
        socketio.run(app, port=self.port)

    def message_queue_processing(self):
        while True:
            rec_id = execute_one_res_async("select chess.get_request()")[0]
            records = execute_all_res_async("select * from chess.get_messages({})".format(rec_id))
            for rec in records:
                user_id = int(rec[0])
                action_to_params = str(rec[1]).split('?')
                action = action_to_params[0]
                params = action_to_params[1]
                user_sid = user_client_map.get(user_id, None)

                # fill none messages
                if user_sid is not None:
                    print("user id is: " + str(user_id))
                    print("user SID is: " + str(user_sid))
                    print("action is " + str(action))
                    print("params is " + str(params))
                    socketio.emit(action, params, room=user_sid)

            execute_no_res_async('call chess.run_jobs()')

@socketio.on('connect')
def on_connect():
    print("%s connected" % (request.sid))
    if request.sid in clients and clients[request.sid] is not None:
        user_client_map[clients[request.sid]] = None
    clients[request.sid] = None

@socketio.on('disconnect')
def on_disconnect():
    print("%s disconnected" % (request.sid))
    if request.sid in clients and clients[request.sid] is not None:
        query = "call chess.on_disconnect({0})".format(clients[request.sid])
        execute_no_res_async(query)

@socketio.on('verify_message')
def on_verify_message(data):
    # print("Message recieved: " + str(data) + "from client " + str(clients[request.sid]))

    paramsDict = supp.getParamsValMap(data)
    if request.sid in clients and clients[request.sid] is not None:
        query = "call chess.verify_message({0}, {1})".format(paramsDict['request_id'], clients[request.sid])
        execute_no_res_async(query)

@socketio.on('confirm_auth')
def on_confirm_auth(data):
    print("Message recieved: " + str(data))

    paramsDict = supp.getParamsValMap(data)

    query = "select chess.confirm_auth('{0}', '{1}')".format(paramsDict['email'],
                                                           paramsDict['auth_code'])
    res = int(execute_one_res_async(query)[0])
    if res == 0:
        # send error message
        socketio.emit('error', 'message=wrong auth code or email', room=request.sid)

@socketio.on('auth')
def on_auth(data):
    print("Message recieved: " + str(data))
    paramsDict = supp.getParamsValMap(data)
    print("Parsed params: " + str(paramsDict))

    if not validate_email(paramsDict['email']):
        return

    query = "select chess.registrate('{0}', '{1}', '{2}')".format(paramsDict['login'],
                                                                  paramsDict['password'],
                                                                  paramsDict['email'])

    res = execute_one_res_async(query)[0]
    if res == "":
        # send error message
        socketio.emit('error', 'message=Wrong registration params', room=request.sid)

@socketio.on('login')
def on_login(data):
    print("Message recieved: " + str(data))

    paramsDict = supp.getParamsValMap(data)

    query = "select chess.login('{0}', '{1}')".format(paramsDict['login'], paramsDict['password'])
    # set user_id for session
    print("login SID is " + str(request.sid))
    user_id = int(execute_one_res_async(query)[0])
    if user_id < 0:
        print('Invalid user!')
        # send error message
        socketio.emit('error', 'message=Unknown login or password', room=request.sid)
        return
    clients[request.sid] = user_id
    user_client_map[user_id] = request.sid
    print("login ID is " + str(clients[request.sid]))

@socketio.on('find_pair')
def on_find_pair(data):
    print("Message recieved: " + str(data))
    paramsDict = supp.getParamsValMap(data)
    if request.sid in clients and clients[request.sid] is not None:
        query = "call chess.find_pair({0}, {1}, {2}, {3}," \
                " p_game_time := TIME '00:{4}:00')".format(clients[request.sid],
                                                           int(paramsDict['low_rate']),
                                                           int(paramsDict['hight_rate']),
                                                           int(paramsDict['move_time']),
                                                           str(paramsDict['game_time']).rjust(2, '0'))
        execute_no_res_async(query)

@socketio.on('find_pair_list')
def on_find_pair_list(data):
    print("Message recieved: " + str(data))
    if request.sid in clients and clients[request.sid] is not None:
        query = "call chess.add_pairings_list({0})".format(clients[request.sid])
        execute_no_res_async(query)

@socketio.on('start_game_by_pairing')
def on_find_pair_list(data):
    paramsDict = supp.getParamsValMap(data)
    print("Message recieved: " + str(data))
    if request.sid in clients and clients[request.sid] is not None:
        query = "call chess.start_game_by_pairing_id({0}, {1})".format(clients[request.sid], paramsDict['pairing_id'])
        execute_no_res_async(query)

@socketio.on('surrender')
def on_surrender(data):
    print("Message recieved: " + str(data))
    if request.sid in clients and clients[request.sid] is not None:
        query = "call chess.surrender({0})".format(clients[request.sid])
        execute_no_res_async(query)

@socketio.on('update_board')
def on_update_board(data):
    print("Message recieved: " + str(data))
    paramsDict = supp.getParamsValMap(data)

    # get board from database
    if clients[request.sid] is not None:
        query = "select * from chess.get_current_game_board_state({0})".format(clients[request.sid])
    else:
        return
    try:
        rec = execute_one_res_async(query)
        print("Game state is " + str(rec))
        board = rec[0]
        side = int(rec[1])
    except:
        print("Game doesn't exists")
        return
    # print("server_board is " + str(rec))
    # convert to game_controller
    print("Board is " + str(board))
    print("Side is " + str(side))
    if board is not None:
        cur_game_controller = game_controller.GameController(None, str(board))
    else:
        cur_game_controller = game_controller.GameController(Board())

    move = vec.Move(vec.Vector2d(int(paramsDict['p1']), int(paramsDict['p2'])),
                    vec.Vector2d(int(paramsDict['p3']), int(paramsDict['p4'])))

    res = cur_game_controller.check_move(move, Side(side))

    if res == game_controller.MoveResult.INCORRECT:
        print("Wrong move send")
        return

    is_playing = 1
    game_result = None
    pawn_swaped_figure = paramsDict['swapped_figure']

    print('Swaped figure is ' + str(pawn_swaped_figure))

    cur_game_controller.update(move, Side(side))

    if pawn_swaped_figure is not None:
        cur_game_controller.swap_pawn(move.point_to, pawn_swaped_figure)
        res = cur_game_controller.check_board_res(Side(side))

    if res == game_controller.MoveResult.STALEMATE:
        is_playing = 0
    elif res == game_controller.MoveResult.MATE:
        is_playing = 0
        game_result = 0 if Side(side) is Side.WHITE else 1

    if game_result is None:
        execute_no_res_async("call chess.update_game_state({0}, '{1}', "
                             "{2}::bit, NULL)".format(clients[request.sid],
                                                      cur_game_controller.serialize_to_str(),
                                                      is_playing))
    else:
        execute_no_res_async("call chess.update_game_state({0}, '{1}', "
                             "{2}::bit, {3}::bit)".format(clients[request.sid],
                                                          cur_game_controller.serialize_to_str(),
                                                          is_playing,
                                                          game_result))

@socketio.on('update_pack')
def on_update_pack(data):
    print("Message recieved: " + str(data))
    params_dict = supp.getParamsValMap(data)
    if request.sid in clients and clients[request.sid] is not None:
        try:
            query = "call chess.update_pack({0}, '{1}')" \
                    "".format(clients[request.sid], params_dict['pack_name'])
            print('query is ' + query)
            execute_no_res_async(query)
        except:
            print("Error while update pack")

@socketio.on('message')
def on_message(data):
    print("Message recieved: " + str(data))
    socketio.emit('message', 'Response from server')

server = Server(8000)
server.run()


import os
import ffmpeg
import argparse
import json
import shutil

def to_frame(name, config):
    if os.path.exists(f'{name}/frames'):
        shutil.rmtree(f'{name}/frames')
    os.mkdir(f'{name}/frames')
    (
        ffmpeg.input("badapple.mp4")
        .filter("fps", fps=config["fps"])
        .filter("scale", width=config["width"], height=config["height"])
        .output(f"{name}/frames/frame%04d.png")
        .overwrite_output()
        .run()
    )

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        prog="Bad Apple to frame",
        description="Convert Bad Apple to frame folder using config file",
    )
    parser.add_argument("name")
    name = parser.parse_args().name
    with open(f"{name}/config.json", "r") as f:
        to_frame(name, json.load(f))


from fastapi import APIRouter, HTTPException, Depends
from config import SessionLocal
from sqlalchemy.orm import Session
from schemas import RequestAccount, Response
import accounts.crud_accounts as crud_accounts

router_accounts = APIRouter(
    prefix="/accounts",
    tags=["accounts"]
)

def get_db():
    db = SessionLocal()
    try:
        yield db
    finally:
        db.close()

@router_accounts.post('/create')
async def create(request: RequestAccount, db: Session = Depends(get_db)):
    crud_accounts.create_account(db, request.parameter)
    return Response(code=200, status="Ok", message="Account created successfully").dict(exclude_none=True)

@router_accounts.get("/")
async def get(db: Session = Depends(get_db)):
    _accounts = crud_accounts.get_accounts(db, 0, 100)
    return Response(code=200, status="ok", message="Success fetch all data", result=_accounts).dict(exclude_none=True)

@router_accounts.get("/{account_login}")
async def get_account_by_login(account_login: str, db: Session = Depends(get_db)):
    _account = crud_accounts.get_account_by_login(
        db, account_login)
    return Response(code=200, status="ok", message="Success get data by project_id", result=_account).dict(exclude_none=True)

@router_accounts.delete("/remove_account_by_login")
async def remove_account_by_login(login: str, db: Session = Depends(get_db)):
    crud_accounts.remove_account_by_login(db, login)
    return Response(code=200, status="Ok", message=f"Account with login: {login} successfuly deleted")


from django.urls import path
from .import views

urlpatterns = [
    path("",views.Index,name="Index"),
    path("SignUp",views.SignUp,name="SignUp"),
    path("SignIn",views.SignIn,name="SignIn"),
    path("SignOut",views.SignOut,name="SignOut"),
    path("MerchantIndex",views.MerchantIndex,name="MerchantIndex"),
    path("Inventory",views.Inventory,name="Inventory"),
    path("Profile",views.Profile,name="Profile"),
    path("AdminIndex",views.AdminIndex,name="AdminIndex"),

]


from app.repo.create_cursor import connect_db
from flask import Flask, request, jsonify
from app.repo.create_tables import * # Assuming Coordinates is a model
from sqlalchemy.exc import SQLAlchemyError  # Import the exception for SQLAlchemy errors

def cord_insert(route_id, route_name,ilat, ilng, dlat, dlng, cord_list):
    try:
        cord = []  # List of all the intermediate coordinates
        cord.append([ilat, ilng])

        for i in cord_list:
            temp = i.split(',')
            x = [float(temp[0]), float(temp[1])]
            cord.append(x)

        cord.append([dlat, dlng])

        db = connect_db()  # Assuming this establishes a database connection
        existing_route = route.query.filter_by(id=route_id).first()

        if not existing_route:
            # Create a new route if it doesn't exist
            new_route = route(id=route_id, name=route_name)
            db.session.add(new_route)
        # Delete previous data for the given route_id
        db.session.query(coordinates).filter_by(route_id=route_id).delete()

        for i in range(len(cord)):
            print(cord[i])
            coordinate = coordinates(name="coord" + str(i), longitude=cord[i][1], latitude=cord[i][0], order=i, route_id=route_id)
            db.session.add(coordinate)

        db.session.commit()

        return True
    except SQLAlchemyError as e:
        db.session.rollback()  # Roll back the transaction in case of an error
        return str(e)  # You can return the exception message or handle it as needed


from __future__ import division

# LIBTBX_SET_DISPATCHER_NAME diffBragg.shoebx

from pylab import *

from argparse import ArgumentParser
parser = ArgumentParser()
parser.add_argument("modeler_file", type=str, help="path to a diffBragg modeler file (output from hopper, see the imgs folder in the outdir)")
parser.add_argument("--stacked", action="store_true")
parser.add_argument("--model_clim", nargs=2, default=[0,None], type=float)
parser.add_argument("--data_clim", nargs=2, default=[0,None], type=float)
args = parser.parse_args()

FIG,(ax0,ax1,ax2) = subplots(nrows=1,ncols=3)
FIG.set_size_inches((5,2))

def press(event):
    if event.key == 'right':
        FIG.loop_counter += 1
    elif event.key=="left":
        FIG.loop_counter = FIG.loop_counter -1
    FIG.loop_counter = max(FIG.loop_counter,0)
    FIG.loop_counter = min(FIG.loop_counter, FIG.nspots-1)

    if event.key=="escape":
        FIG.loop_counter = FIG.nspots

def centroid_poly(x,y):
    y1 = y[:-1]
    y2 = y[1:]

    x1 = x[:-1]
    x2 = x[1:]
    xy = (x1*y2 - x2*y1)
    Cx = np.sum((x1+x2)*xy)/6.
    Cy = np.sum((y1+y2)*xy)/6.
    A = 0.5*np.sum(xy)
    return Cx/A, Cy/A

M = np.load(args.modeler_file, allow_pickle=True)[()]
num_spots = len(M.pids)

FIG.loop_counter = 0
FIG.nspots = num_spots
FIG.canvas.mpl_connect('key_press_event', press)

assert len(set(M.roi_id)) == max(M.roi_id)+1
sigma_rdout = M.params.refiner.sigma_r / M.params.refiner.adu_per_photon
#M.hi = (0,0,0)]
#hkls = M.Hi
cmap = 'gray_r'

from cctbx import uctbx
# TODO put unit cell manager in the modeler file
#PSII
uc = uctbx.unit_cell((116.92, 221.635, 307.834, 90, 90, 90))
#lysozyme?
uc = uctbx.unit_cell((79.1,79.1,38.4,90,90,90))
diffs = []
d_max = 0
d_min = 9999

#F2 = plt.figure()
#AX2 = F2.gca()
#AX2.add_patch(Rectangle(xy=(0,0), width=4000, height=4000, fc='none', ec='b'))
#AX2.set_xlim(-10,4010)
#AX2.set_ylim(-10,4010)

if args.stacked:

    if isinstance(M.all_sigma_rdout, np.ndarray):
        data_subimg, model_subimg, trusted_subimg, bragg_subimg, sigma_rdout_subimg = M.get_data_model_pairs()
    else:
        data_subimg, model_subimg, trusted_subimg, bragg_subimg = M.get_data_model_pairs()
        sigma_rdout_subimg = None

    sub_sh = tuple(np.max([im.shape for im in model_subimg], axis=0))
    size_edg = int(np.sqrt(len(data_subimg))) + 1
    full_im = np.zeros((size_edg * sub_sh[0], size_edg * sub_sh[1]))
    full_dat_im = np.zeros((size_edg * sub_sh[0], size_edg * sub_sh[1]))
    for j in range(size_edg):
        for i in range(size_edg):
            mod_idx = j * size_edg + i
            if mod_idx >= len(model_subimg):
                continue
            im = model_subimg[mod_idx].copy()
            dat_im = data_subimg[mod_idx].copy()
            ydim, xdim = im.shape
            if im.shape != sub_sh:
                im = np.pad(im, ((0, sub_sh[0] - ydim), (0, sub_sh[1] - xdim)), mode='constant', constant_values=np.nan)
                dat_im = np.pad(dat_im, ((0, sub_sh[0] - ydim), (0, sub_sh[1] - xdim)), mode='constant',
                                constant_values=np.nan)
            Ysl = slice(j * sub_sh[0], (j + 1) * sub_sh[0], 1)
            Xsl = slice(i * sub_sh[1], (i + 1) * sub_sh[1], 1)
            full_im[Ysl, Xsl] = im
            full_dat_im[Ysl, Xsl] = dat_im
    subplot(121)
    gca().set_title("DATA", fontsize=18)
    imshow(full_dat_im, vmin=args.data_clim[0], vmax=args.data_clim[1], cmap='gray_r')
    gca().set_xticks(np.arange(sub_sh[1], size_edg*sub_sh[1], sub_sh[1]))
    gca().set_yticks(np.arange(sub_sh[0], size_edg*sub_sh[0], sub_sh[0]))
    gca().grid(1, color='k', ls='--')
    gca().set_xticklabels([])
    gca().set_yticklabels([])
    gca().tick_params(length=0)
    #gca().set_xlim(0, size_edg*sub_sh[1])
    #gca().set_ylim(0, size_edg*sub_sh[0])
    #gca().set_yticks([])
    subplot(122)
    gca().set_title("MODEL", fontsize=18)
    imshow(full_im, vmin=args.model_clim[0], vmax=args.model_clim[1], cmap='gnuplot')
    gca().set_xticks(np.arange(sub_sh[1], size_edg*sub_sh[1], sub_sh[1]))
    gca().set_yticks(np.arange(sub_sh[0], size_edg*sub_sh[0], sub_sh[0]))
    gca().grid(1, color='w', ls='--')
    gca().set_xticklabels([])
    gca().set_yticklabels([])
    gca().tick_params(length=0)
    #gca().set_xlim(0, size_edg*sub_sh[1])
    #gca().set_ylim(0, size_edg*sub_sh[0])
    #gca().set_xticks([])
    #gca().set_yticks([])
    subplots_adjust(wspace=0, hspace=0, bottom=0.02, top=0.95, left=0.01, right=0.99)
    show()
    exit()

a = b = None
while FIG.loop_counter < num_spots:
    i_h = FIG.loop_counter
    h = k = l = np.nan
    d = -1 #uc.d((h,k,l))
    #if d > d_max: d_max = d
    #if d < d_min: d_min = d

    # you could put a filter here to choose only those refls in a certain resolution
    sel = M.roi_id == i_h
    x1, x2, y1, y2 = M.rois[i_h]
    sh = y2 - y1, x2 - x1
    data = M.all_data[sel].reshape(sh)
    trusted = M.all_trusted[sel].reshape(sh)
    any_trusted = np.any(trusted)

    bg = M.all_background[sel].reshape(sh)
    bragg = M.best_model[sel].reshape(sh)
    model = bragg + bg
    diff = data.sum() - model.sum()

    Z = (model - data) / np.sqrt(model + sigma_rdout ** 2)
    diffs.extend(Z.ravel())

    vals = Z[trusted]
    data_trust = data[trusted]
    if not any_trusted:
        data_thresh = 0
    else:
        data_thresh = np.percentile(data_trust,97)
    y,x = np.indices(data.shape)

    ycent = (bragg.ravel()*y.ravel()).sum() /bragg.sum()
    xcent = (bragg.ravel()*x.ravel()).sum() / bragg.sum()

    if not any_trusted:
        sigZ_val = np.nan
    else:
        sigZ_val = Z[trusted].std()

    for ax in ax0,ax1,ax2:
        ax.clear()
    im = ax0.imshow(data, cmap=cmap)
    vmin, vmax = im.get_clim()
    ax1.imshow(model,cmap=cmap, vmin=vmin, vmax=vmax)

    ax1.contour(bragg, levels=[1e-4,1e-3,1e-2,1e-1,1,10], cmap='jet')
    C = ax0.contour(bragg, levels=[1e-4,1e-3,1e-2,1e-1,1,10], cmap='jet')
    ax1.plot( xcent, ycent, 'rx')
    #C = ax0.contour(bragg, levels=[0.1*data_thresh, 0.5*data_thresh,data_thresh], cmap='jet')
    #from IPython import embed;embed()

    #try:
    #    segs = C.allsegs[-1][0]
    #    xcent, ycent = centroid_poly(segs[:,0], segs[:,1])
    #    ax0.plot( [xcent], [ycent], 'o', mec='b', ms=10, alpha=0.5)
    #    ax0.plot( segs[:,0], segs[:,1], '.', ms=3, color='k')
    #except IndexError:
    #    pass

    v = np.max(np.abs(Z))
    v = 10
    ax2.imshow(Z, cmap='RdYlBu', vmin=-v, vmax=v)

    for AX in [ax0, ax1, ax2]:
        AX.grid(1, color='#777777', ls='--', lw=0.4)
        AX.set_xticklabels([])
        AX.set_yticklabels([])
    ax0.set_title("data %d,%d,%d,%d"% (x1,x2,y1,y2), pad=0, fontsize=8)
    ax1.set_title("model", pad=0, fontsize=8)
    ax2.set_title("Z (sigZ=%f)" % sigZ_val, pad=0, fontsize=8)

    FIG.suptitle("spot %d" % i_h)
    if a is not None:
        a.remove()
    if b is not None:
        b.remove()
    a = FIG.add_axes([.91, .25, 0.02, .5])
    FIG.colorbar(ax2.images[0], cax=a)
    a.tick_params(length=0, labelsize=8, pad=1)
    a.set_title("Z", pad=0, fontsize=8)

    b = FIG.add_axes([.2, .1, .33, .02])
    FIG.colorbar(ax0.images[0], cax=b, orientation='horizontal')
    b.yaxis.set_ticks_position('left')
    b.tick_params(length=0, labelsize=8, pad=5)
    b.set_title("counts", pad=0, fontsize=8)
    draw()
    waitforbuttonpress()

    i_h += 1

plt.close()


from lib2to3.pgen2 import token
from statistics import mode
from keras.preprocessing.text import Tokenizer
import numpy as np

#데이터
docs = ['나는 형권이가 정말 재미없다 너무 정말']

# 긍정 1 , 부정 0
labels = np.array([1])

token = Tokenizer()
token.fit_on_texts(docs)
print(token.word_index)

{'정말': 1, '나는': 2, '형권이가': 3, '재미없다': 4, '너무': 5}

x = token.texts_to_sequences(docs) #pad 로 앞부분 부터 0을 채우는게 좋음
print(x)

#[[2, 3, 1, 4, 5, 1]]

from keras.preprocessing.sequence import pad_sequences
pad_x = pad_sequences(x,padding='pre',maxlen=4,) #truncating  # truncating=
print(pad_x)
print(pad_x.shape)  #((1, 4)) #rehape 해서 3~4차원으로 LSTM 가능

word_size = len(token.word_index)
print('word_size:',word_size)    #word_size:5 #단어사전의 갯수 : 5#
print(np.unique(pad_x, return_counts=True)) #(array([1, 3, 4, 5]), array([2, 1, 1, 1], dtype=int64))

#.모델
from tensorflow.python.keras.models import Sequential
from tensorflow.python.keras.layers import Dense,LSTM,Embedding,Flatten,Conv1D #Embedding 통상 레이어스를 많이씀

model = Sequential()
                  #input dim은 단어 사전에 겟수 output_dim
# model.add(LSTM())
model.add(Embedding(input_dim=5, output_dim=10, input_length=5)) #길이  input_length=5
# model.add(Embedding(input_dim=31, output_dim=10)) #길이  input_length=5(명시를 안 해도 알아서 잡아줌)
# model.add(Embedding(31, 10)) #(단어 사전의 갯수 31 아웃풋 10)
# model.add(Embedding(31, 10, 5))
# model.add(Embedding(31, 10, input_length=5)) # 10개짜리가 5개씩 묶여있음 (인풋랭스는 정확하게 명시 해야함)
# model.add(LSTM(32)) #(2차원)
model.add(Conv1D(32, 2))
model.add(Flatten())
model.add(Dense(1, activation='sigmoid'))
model.summary()

#3.컴파일훈련
model.compile(loss='binary_crossentropy',optimizer='adam', metrics=['acc'])
model.fit(pad_x, labels ,epochs=16, batch_size=16)

#4. 평가 ,예측
acc = model.evaluate(pad_x,labels)[1]  #<<<<<< [0]을 넣으면 loss [1]을 넣으면 accrcy
print('acc:',acc)

# y_predict = ('나는 형권이가 정말 재미없다 너무 정말')

# y_predict=model.predict('나는 형권이가 정말 재미없다 너무 정말')

# # from sklearn.metrics import r2_score
# # r2 = r2_score( y_predict)
# # print('loss : ' , loss)
# # print('r2스코어 : ', r2)

# if 	y_predict <= 0.6 :
#     print('재미있다') # 출력값:
# else :
#     print('재미없다') # 출력값:

# ###############실습#######################
# x_predict =  '나는 형권이가 정말 재미없다 너무 정말'
# '''


import pickle

class MemberRepo:
    """Gets and updates file for info on members.

    """
    def __init__(self):
        self.default_save_file = "SportStudSave01.p"

    def save(id_map, name_map, phone_map, email_map):
        pickle.dump(id_map, open("saves/id_map.p", "wb"))
        pickle.dump(name_map, open("saves/name_map.p", "wb"))
        pickle.dump(phone_map, open("saves/phone_map.p", "wb"))
        pickle.dump(email_map, open("saves/email_map.p", "wb"))

    def load():
        try:
            id_map = pickle.load(open("saves/id_map.p", "rb"))
            name_map = pickle.load(open("saves/name_map.p", "rb"))
            phone_map = pickle.load(open("saves/phone_map.p", "rb"))
            email_map = pickle.load(open("saves/email_map.p", "rb"))
            #nota close eða with
            return id_map, name_map, phone_map, email_map
        except FileNotFoundError or TypeError:
            print('Unfortunately we could not find your save file. No data was loaded.')

    def save_un_id(unique_id):
        pickle.dump(unique_id, open("saves/unique_id.p", "wb"))

    def load_un_id():
        try:
            unique_id = pickle.load(open("saves/unique_id.p", "rb"))
            return int(unique_id)
        except FileNotFoundError:
            print("Unfortunately we could not find your save file. Not all data was loaded.")
            return 1

class SportsRepo:
    """Gets and updates file for info on sports and groups"""

    def save(sp_map):
        pickle.dump(sp_map, open("saves/sp_map.p", "wb"))

    def load():
        try:
            sp_map = pickle.load(open("saves/sp_map.p", "rb"))
            return sp_map
            #nota close eða with
        except FileNotFoundError or TypeError:
            print('Unfortunately we could not find your save file. No data was loaded.')

# class GroupRepo:
#     """Gets and updates file for info on sports and groups"""

#     def save(gr_map):
#         pickle.dump(gr_map, open("saves/gr_map.p", "wb"))

#     def load():
#         try:
#             gr_map = pickle.load(open("saves/gr_map.p", "rb"))
#             return gr_map
#             #nota close eða with
#         except FileNotFoundError or TypeError:
#             print('Unfortunately we could not find your save file. No data was loaded.')


from pysubparser import parser

subtitles = parser.parse('./original_subtitle.srt')
f = open("new_subtitle.srt", "w")

prevStart = None
i = 1
for subtitle in subtitles:
    if prevStart and prevStart == subtitle.start:
        f.write(subtitle.text + '\n')
    else:
        f.write('\n')
        f.write(str(i) + '\n')
        i += 1
        f.write(
            str(subtitle.start.strftime("%H:%M:%S,%f")[:-3]) + " --> " +
            str(subtitle.end.strftime("%H:%M:%S,%f")[:-3]) + '\n')
        f.write(subtitle.text + '\n')

        prevStart = subtitle.start

f.close()
print("Finished")


# 将一个 二叉搜索树 就地转化为一个 已排序的双向循环链表 。

# 对于双向循环列表，你可以将左右孩子指针作为双向循环链表的前驱和后继指针，第一个节点的前驱是最后一个节点，最后一个节点的后继是第一个节点。

# 特别地，我们希望可以 就地 完成转换操作。当转化完成以后，树中节点的左指针需要指向前驱，树中节点的右指针需要指向后继。还需要返回链表中最小元素的指针。

#  示例 1：
# 输入：root = [4,2,5,1,3]
# 输出：[1,2,3,4,5]

# 示例 2：
# 输入：root = [2,1,3]
# 输出：[1,2,3]

# 示例 3：
# 输入：root = []
# 输出：[]
# 解释：输入是空树，所以输出也是空链表。

# 示例 4：
# 输入：root = [1]
# 输出：[1]

# 来源：力扣（LeetCode）
# 链接：https://leetcode.cn/problems/convert-binary-search-tree-to-sorted-doubly-linked-list
# 著作权归领扣网络所有。商业转载请联系官方授权，非商业转载请注明出处。

# 思路和心得：

# 1.BST的LNR就是有序的。一般就用中序LNR

# 2.全局指针

# 3.dfs的过程其实就是思路

# 作者：HanXin_HanXin
# 链接：https://leetcode.cn/problems/convert-binary-search-tree-to-sorted-doubly-linked-list/solution/c-python3-zhong-xu-huan-xu-zhong-xu-jie-y04fh/
# 来源：力扣（LeetCode）
# 著作权归作者所有。商业转载请联系作者获得授权，非商业转载请注明出处。

"""
# Definition for a Node.
class Node:
    def __init__(self, val, left=None, right=None):
        self.val = val
        self.left = left
        self.right = right
"""

from tables import Node

class Solution:
    def dfs_LNR(self, rt: 'Node') -> None:
        if rt:
            self.dfs_LNR(rt.left)

            if self.last:           #左子不空，从全局的中序遍历，已经遍历完last了，当前rt要和last链接起来
                self.last.right = rt
                rt.left = self.last
            else:                   #左子为空
                self.first = rt

            self.last = rt          #rt也在全局范围内统计了，成为统计好的最后一个

            self.dfs_LNR(rt.right)

    def treeToDoublyList(self, root: 'Node') -> 'Node':
        if not root:
            return None

        self.first = None
        self.last = None

        self.dfs_LNR(root)

        self.first.left = self.last     #首尾链接起来
        self.last.right = self.first

        return self.first               #返回头

# 将一个 二叉搜索树 就地转化为一个 已排序的双向循环链表 。

# 对于双向循环列表，你可以将左右孩子指针作为双向循环链表的前驱和后继指针，第一个节点的前驱是最后一个节点，最后一个节点的后继是第一个节点。

# 特别地，我们希望可以 就地 完成转换操作。当转化完成以后，树中节点的左指针需要指向前驱，树中节点的右指针需要指向后继。还需要返回链表中最小元素的指针。

# Python3 中序遍历
class Solution:
    def treeToDoublyList(self, root: 'Optional[Node]') -> 'Optional[Node]':
        if not root:
            return None
        head = Node(-1)
        pre = head
        def inorder(root):
            nonlocal pre
            if not root:
                return
            inorder(root.left)
            pre.right = root
            root.left = pre
            pre = pre.right
            inorder(root.right)
        inorder(root)
        head.right.left = pre
        pre.right = head.right
        return head.right

# 输入：root = [2,1,3]
# 输出：[1,2,3]

企业真题/蚂蚁/426.二叉树转双向链表_蚂蚁AI算法一面0530.py

426.二叉树转双向链表_蚂蚁AI算法一面0530.py


import common
from codegen import TabsCodeHandler

class LispCodeGenerator:
#wxNotebook(parent, id, pos, size, style, name)
    new_signature = [
        '$parent', '$id', '$pos', '$size', '$style', '$name'
    ]

    def get_code(self, window):
        plgen = common.code_writers['lisp']
        prop = window.properties
        id_name, id = plgen.generate_code_id(window)

        layout_props = []
        tabs = prop.get('tabs', [])
        for label, tab_win in tabs:
            tab_win = tab_win.replace('_','-')
            layout_props.append('(wxNotebook_AddPage (slot-%s obj) (slot-%s obj) %s 1 -1)\n' % \
                                (window.name, tab_win, plgen.quote_str(label)))

        if not window.parent.is_toplevel:
            parent = '(slot-%s obj)' % window.parent.name
        else:
            parent = '(slot-top-window obj)'

        if window.is_toplevel:
            l = []
            if id_name: l.append(id_name)
            l.append('(setf (slot-%s obj) (wxNotebook_Create %s %s -1 -1 -1 -1 wxNB_TOP))\n' %
                (window.name, parent,id))
            return l, [], []
        style = prop.get("style")
        if style:
            style = style.strip().replace('|',' ')
            if style.find(' ') != -1:
                style = '(logior %s)' % style
        else:
            style = 'wxNB_TOP'

        init = []
        if id_name: init.append(id_name)
        init.append('(setf (slot-%s obj) (wxNotebook_Create %s %s -1 -1 -1 -1 %s))\n'
                    % (window.name, parent, id, style))

        props_buf = plgen.generate_common_properties(window)
        return init, props_buf, layout_props

    def get_properties_code(self, obj):
        prop = obj.properties
        plgen = common.code_writers['lisp']
        props_buf = []
        tabs = prop.get('tabs', [])
        for label, window in tabs:
            props_buf.append('(wxNotebook_AddPage (slot-%s obj) page %s 1 -1);\n' % \
                             (window, plgen.quote_str(label)))
        props_buf.extend(plgen.generate_common_properties(obj))
        return props_buf

# end of class LispCodeGenerator

def initialize():
    common.class_names['EditNotebook'] = 'wxNotebook'
    common.class_names['NotebookPane'] = 'wxPanel'
    common.toplevels['EditNotebook'] = 1
    common.toplevels['NotebookPane'] = 1

    plgen = common.code_writers.get('lisp')
    if plgen:
        plgen.add_widget_handler('wxNotebook', LispCodeGenerator())
        plgen.add_property_handler('tabs', TabsCodeHandler, 'wxNotebook')


# 리스트 내포(comprehension)
# 실행 순서 : for -> if -> 표현식 -> 리스트

# for 사용
nums = [i for i in range(5)]
print(nums)

nums2 = [100, 200, 300]
double_nums = [i*2 for i in nums2]
print(double_nums)

# if 사용
nums3 = [i for i in range(10) if i%2 == 0]
print(nums3)

nums4 = [100, 200, 300, 400, 500, 600]
double_nums2 = [i*2 for i in nums4 if i >= 300]
print(double_nums2)


def bubbleSort(arr):
    n = len(arr)
    for i in range(n-1):
        for j in range(0, n-i-1):
            if arr[j] > arr[j+1] :
                arr[j], arr[j+1] = arr[j+1], arr[j]

arr = []

print("Please type the arrays:")
num = int(input("< "))
print("Please type the array:")
for i in range(num):
    s = int(input("< "))
    arr.append(s)

bubbleSort(arr)

print ("Sorted array is:")
for i in range(len(arr)):
    print ("%d" %arr[i]),


"""
@brief      test tree node (time=20s)
"""
import os
import unittest
import time
from pyquickhelper.loghelper.flog import fLOG
from pyquickhelper.pycode import get_temp_folder, ExtTestCase, skipif_travis
from pyquickhelper.loghelper.time_helper import repeat_execution, repeat_script_execution

class TestTimeHelper(ExtTestCase):

    def test_repeat_execution(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        ct = [0]

        def fct_count():
            ct[0] += 1
            return ct[0]

        res = repeat_execution(fct_count, verbose=1, fLOG=fLOG)
        self.assertEqual(res, [1, 2, 3, 4, 5])

    def test_repeat_execution_exc(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        ct = [0]

        def fct_count():
            raise RuntimeError("issue")

        res = repeat_execution(fct_count, verbose=1, fLOG=fLOG, exc=False,
                               stop_after_second=2)
        self.assertEmpty(res)

    def test_repeat_execution_sleep(self):
        ct = [0]

        def fct_count():
            ct[0] += 1
            time.sleep(1.5)
            return ct[0]

        res = repeat_execution(fct_count)
        self.assertLess(len(res), 4)

    @skipif_travis("stuck")
    def test_repeat_execution_script(self):
        temp = get_temp_folder(__file__, "temp_repeat_execution_script")
        outfile = os.path.join(temp, "out.txt")
        errfile = os.path.join(temp, "err.txt")
        script = os.path.join(temp, "script.py")
        with open(script, "w") as f:
            f.write("print('machine')\n")

        res = repeat_script_execution(script, outfile=outfile, errfile=errfile,
                                      stop_after_second=3)

        with open(outfile, "r") as f:
            out = f.read()
        self.assertIn("machine\n", res)
        self.assertIn("[repeat_script_execution]", out)
        self.assertIn("iter=1\n", out)
        self.assertIn("machine\n", out)

if __name__ == "__main__":
    unittest.main()


"""
Holds the class SegmentedDmCommand that is used to create a dict that will be sent to the
segmented DM hardware as a command.

This module can be used to create a command in the following way:
    command = SegmentedDmCommand(dm_config_id, apply_flat_map=True,
                                 filename_flat='repo-path/hardware/iris-ao/CustomFLAT.ini')
    command.read_initial_values(segment_ptt_values)
    command.update_one_segment(segment_index, segment_ptt_values)
    command.display()
    dm_object.apply_shape(command)

Note that a command can be initialized from zeros, None, a list of tuples giving (P,T,T) for each segment,
an .ini file, or a .PTT### file. The PoppySegmentedDmCommand class can be used to create a command from
input Zernike coefficients.
Additionally, a single segment can be changed using the SegmentedDmCommand.update_one_segment()
method.

"""
from configparser import NoOptionError
import json
import os

import astropy.units as u
import matplotlib.pyplot as plt
import numpy as np
import poppy

from catkit.catkit_types import MetaDataEntry
from catkit.config import CONFIG_INI
from catkit.hardware.iris_ao import util

class SegmentedAperture(poppy.dms.HexSegmentedDeformableMirror):
    """
    Create a segmented aperture with Poppy using the parameters for the testbed
    and segmented DM from the config.ini file.

    To create an aperture, with the config.ini file loaded, run:
        segmented_aperture_obj = SegmentedAperture()

    :param dm_config_id: str, name of the section in the config_ini file where information
                         regarding the segmented DM can be found.
    :param rotation: float, rotation angle of the hex segmented DM in deg
    :attribute outer_ring_corners: bool, whether or not the segmented aperture includes
                                   the corner segments on the outer-most ring. If True,
                                   corner segments are included. If False, they are not
    :attribute center_segment: bool, whether of not the segmented aperture includes the
                               center segment. If True, the center segment is included.
                               If False, it is not.
    :attribute flat_to_flat: float, physical distance from flat to flat side of a DM segment
                             in units of mm
    :attribute gap: float, physical distance between DM segments in units of microns
    :attribute number_segments_in_pupil: int, number of segments included in the aperture for active
                                         control as specified by the "active_number_of_segments"
                                         parameter in the config file
    """

    def __init__(self, dm_config_id, rotation=0):
        # Set config sections
        self.dm_config_id = dm_config_id

        # Parameters specific to testbed setup being used
        self.rotation = rotation

        # Parameters specifc to the aperture and segmented DM being used
        self.outer_ring_corners = CONFIG_INI.getboolean(self.dm_config_id, 'include_outer_ring_corners')
        self.center_segment = CONFIG_INI.getboolean(self.dm_config_id, 'include_center_segment')
        self.flat_to_flat = CONFIG_INI.getfloat(self.dm_config_id, 'flat_to_flat_mm') * u.mm
        self.gap = CONFIG_INI.getfloat(self.dm_config_id, 'gap_um') * u.micron
        self.number_segments_in_pupil = CONFIG_INI.getint(self.dm_config_id, 'active_number_of_segments')

        # Get the specific segments
        self._num_rings = self.get_number_of_rings_in_pupil()
        self._segment_list = self.get_segment_list()

        super().__init__(name='Segmented DM',
                         rings=self._num_rings,
                         flattoflat=self.flat_to_flat,
                         gap=self.gap,
                         segmentlist=self._segment_list,
                         rotation=self.rotation)

    def get_number_of_rings_in_pupil(self):
        """
        Get the number of rings based on the number of active segments specified in the
        config.ini file.

        :return: num_rings: int, the number of full rings of hexagonal segments in the aperture
        """
        active_segs_per_ring = self.get_active_number_of_segments_per_ring()

        try:
            num_rings = active_segs_per_ring.index(self.number_segments_in_pupil)
        except ValueError as error:
            raise ValueError("Invalid number of segments. Please check your config.ini file.") from error

        return num_rings

    def get_active_number_of_segments_per_ring(self, max_number_of_rings=7):
        """
        Given pupil specifics, returns a list of length equal to the maximum
        number of rings where each element is the total number of segments in a
        pupil with that number of rings.

        Given the use of the center segment or outer ring corner segments,
        return a list of the number of total active segments in a pupil, of a variety
        of number of rings that is indicated by its index in the list. For example,
        the first element in the list, with index of 1, will give the total number of
        active segments in a pupil with 1 ring which include the center segment if that
        segment is active. Note: The maximum number of rings for the PTT489 from IrisAO is 7

        return: list of total number of active segments in a pupil for pupils with
                a number of rings as indicated by the index of in the list
        """
        max_number_segments_in_pupil_per_ring = self.get_max_number_segments_in_pupil_per_ring(max_number_of_rings)

        # If no outer corners, you will have 6 fewer segments in that outer ring
        if not self.outer_ring_corners:
            active_segments_in_pupil_per_ring = [num-6 if num > 6 else num for num in max_number_segments_in_pupil_per_ring]
        # If no center segment, you will have 1 fewer segment overall
        if not self.center_segment:
            active_segments_in_pupil_per_ring = [num-1 for num in max_number_segments_in_pupil_per_ring]

        if self.outer_ring_corners and self.center_segment:
            active_segments_in_pupil_per_ring = max_number_segments_in_pupil_per_ring

        return active_segments_in_pupil_per_ring

    def get_max_number_segments_in_pupil_per_ring(self, number_of_rings=7):
        """Returns a list of length equal to the maximum number of rings in a pupil
        with hexagonal segments where each element is the total number of segments in a
        pupil with that number of rings.

        :return: list, list of length equal to the total number of rings where each element
        is the total number of segments in a pupil with that number of rings.
        """
        max_number_segments_in_pupil_per_ring = [1,] # number of segments in the pupil per ring
        for i in np.arange(number_of_rings)+1:
            max_number_segments_in_pupil_per_ring.append(max_number_segments_in_pupil_per_ring[i-1]+i*6)

        return max_number_segments_in_pupil_per_ring

    def get_segment_list(self):
        """
        Grab the list of segments to be used in your pupil taking into account if your
        aperture has a center segment and/or the segments in the corners of the outer ring.
        This list is passed to Poppy to help create the aperture.

        :param num_rings: int, The number of rings in your pupil
        :return: list, the list of segments as passed into poppy
        """
        num_segs = self.total_number_segments_in_aperture(self._num_rings)
        seglist = np.arange(num_segs)

        # If no outer corners, you will have 6 fewer segments in that outer ring
        if not self.outer_ring_corners:
            inner_segs = seglist[:(num_segs-6*self._num_rings)]
            outer_segs = seglist[(num_segs-6*self._num_rings):]
            outer_segs = np.delete(outer_segs, np.arange(0, outer_segs.size,
                                                         self._num_rings)) # delete corner segs
            seglist = np.concatenate((inner_segs, outer_segs))

        if not self.center_segment:
            seglist = seglist[1:]
        return seglist

    def total_number_segments_in_aperture(self, number_of_rings=7):
        """
        For a segmented aperture of rings = number_of_rings, give the total number of
        segments in the aperture given an aperture of 1, 2, 3, etc. rings. Will return
        a list of the total number of segments in the *aperture* per ring where the ring
        is indicated by the index in the list.

        param: number_of_rings, int, the number of rings in the aperture
        return: a list of segments where the index of the value corresponds to the
                 number of rings. For example: index 0 corresponds with the center
                 where there is one segment.
        """
        number_segments_in_pupil_per_ring = self.get_max_number_segments_in_pupil_per_ring(number_of_rings)

        return number_segments_in_pupil_per_ring[number_of_rings]

class SegmentedDmCommand(object):
    """
    Handle segmented DM specific commands in terms of piston, tip and tilt (PTT) for
    each segment. Creates a dictionary of the form {seg: (piston, tip, tilt)} that can
    be loaded onto the hardware.

    Units of the loaded command are defined with the atrribute "dm_command_units" and are
    usually in um (for piston) and mrad (for tip and tilt).

    This class does NOT interact with hardware directly.

    :param dm_config_id: str, name of the section in the config_ini file where information
                         regarding the segmented DM can be found.
    :param apply_flat_map: If true, add the custom flat map correction to the data before creating the command
    :param filename_flat: string, full path to custom flat map, only needed if apply_flat_map=True
    :param rotation: float, rotation angle of the hex segmented DM in deg
    :attribute data: list of tuples, input data that can then be updated. This attribute never
                     never includes the custom flat map values; units are determined with the attribute dm_command_units
    :attribute apply_flat_map: bool, whether or not to apply the custom flat map
    :attribute source_pupil_numbering: list, numbering native to data
    :attribute command: dict, final command with flat if apply_flat_map = True; units are determined with the attribute dm_command_units
    :attribute filename_flat: str, full path to custom flat, only needed if apply_flat_map=True
    :attribute total_number_segments: int, total number of segments in DM, includes dead segments
    :attribute active_segment_list: int, number of active segments in the DM
    :attribute dm_command_units: tuple of floats, the units of the piston, tip, tilt
                                 values on the hardware
    :attribute aperture: poppy.dms.HexSegmentedDeformableMirror object, the aperture
                         that you are defining
    """

    def __init__(self, dm_config_id, apply_flat_map=False, filename_flat=None, rotation=0):
        self.dm_config_id = dm_config_id

        # Initialize class used to model the segmented aperture geometry
        self.aperture = SegmentedAperture(dm_config_id, rotation=rotation)

        # Determine if the custom flat map will be applied
        self.apply_flat_map = apply_flat_map
        if self.apply_flat_map:
            if filename_flat is None:
                raise ValueError("Must provide a filename for the segmented DM flat file, not None.")
            self.filename_flat = filename_flat
            # Check that the file exists
            if not os.path.isfile(self.filename_flat):
                raise FileNotFoundError(f"{self.filename_flat} either does not exists or is not currently accessible")

        # Establish segment information
        try:
            self.segments_in_pupil = json.loads(CONFIG_INI.get(self.dm_config_id, 'active_segment_list'))
            if len(self.segments_in_pupil) != self.aperture.number_segments_in_pupil:
                raise ValueError("The length of active_segment_list does not match the active_number_of_segments in the config.ini. Please update your config.ini.")
        except NoOptionError:
            self.segments_in_pupil = util.iris_pupil_naming(self.dm_config_id)

        # Set units for piston, tip, tilt
        dm_command_units = CONFIG_INI.get(self.dm_config_id, 'dm_ptt_units').split(',')
        self.dm_command_units = [u.Unit(dm_command_units[0]), u.Unit(dm_command_units[1]),
                                 u.Unit(dm_command_units[2])]

        # Initalize command
        self.data = util.create_zero_list(self.aperture.number_segments_in_pupil)
        self.input_data = None
        self.command = None

    def read_initial_command(self, segment_values):
        """
        Read a new command and assign to the attribute 'data'

        :param segment_values: str, list. Can be .PTT111, .ini files or a list with piston, tip,
                               tilt values in a tuple for each segment. See the load_command doc
                               string for more information.
        """
        self.input_data = self.data = self._read_command(segment_values)

    def _read_command(self, segment_values):
        """
        Read and return command from one of the allowed formats

        :param segment_values: str, list. Can be .PTT111, .ini files or a list with piston, tip,
                               tilt values in a tuple for each segment. See the load_command doc
                               string for more information.
        :return: list of tuples for each commanded segment
        """
        ptt_list, segment_names = util.read_segment_values(segment_values, self.dm_config_id)
        if segment_names is not None:
            command_list = []
            for seg_name in self.segments_in_pupil:  # Pull out only segments in the pupil
                ind = np.where(np.asarray(segment_names) == seg_name)[0][0]
                command_list.append(ptt_list[ind])
        else:
            command_list = ptt_list
        return command_list

    def get_data(self):
        """ Grab the current shape to be applied to the DM (does NOT include the custom flat map)
        """
        return self.data

    def to_command(self):
        """ Output command suitable (in the form a dictionary with an entry for each segment)
        for sending to the hardware driver. The custom flat map will be added only at this stage.
        """
        if self.apply_flat_map:
            command_data = self.add_map(self.filename_flat, return_new_map=True)
        else:
            command_data = self.data
        command_dict = dict(zip(self.segments_in_pupil, command_data))

        return command_dict

    def update_one_segment(self, segment_ind, ptt_tuple, add_to_current=True):
        """ Update the value of one segment by supplying the new command that will be added
        to or will replace the current PTT tuple on this segment only. To identify the segment to be changed, give
        the *command index* (not its name like in the GUI) of that segment in the active segment list. This will be added
        to the current value only if the add_to_current flag is set to True, otherwise, value
        given will replace the current value.

        :param segment_ind: int, for the segment in the pupil that is to be updated,
                            provide the index of it's location in the active segment list
        :param ptt_tuple: tuple with three values for piston, tip, and tilt in um and mrad unless specified otherwise
                          with self.dm_command_units
        """
        if add_to_current:
            command_list = util.create_zero_list(self.aperture.number_segments_in_pupil)
            command_list[segment_ind] = ptt_tuple
            self.add_map(command_list)
        else:
            self.data[segment_ind] = ptt_tuple

    def add_map(self, segment_values_to_add, return_new_map=False):
        """
        Add a command to the one already loaded.

        :param new_command: str or list (.PTT111 or .ini file, or list, for example, from POPPY)
        """
        original_data = self.get_data()
        data_to_add = self._read_command(segment_values_to_add)

        new_map = [tuple(map(sum, zip(orig, addition))) for orig,
                   addition in zip(original_data, data_to_add)]

        if return_new_map:
            return new_map
        else:
            self.data = new_map

    def to_ini(self, filename, out_dir=''):
        """
        Write the command to a .ini file. Note: This will NOT include the applied custom flat
        map values
        :param filename: str, name of ini file to be written out
        :param out_dir: str, name of directory where ini file will be saved
        """
        if not filename.endswith(".ini"):
            filename += ".ini"

        data = self.get_data()
        command = dict(zip(self.segments_in_pupil, data))
        path = os.path.join(out_dir, filename)
        util.write_ini(command, path, dm_config_id=self.dm_config_id)

    def get_extra_meta_data(self):
        """
        Create meta data to be saved with fits files that gives the ptt values/segment
        and if the flat map was applied.The values saved will NOT include custom flat map PTT values.
        """
        metadata = []
        if self.apply_flat_map:
            metadata.append(MetaDataEntry("Flat Name", "FLATMAP", self.filename_flat,
                                          "Flat map name/file if applied"))
        rounded_ptt_list = round_ptt_list(self.data, decimals=4)    # since self.data is usually in units of (um, mrad, mrad), it is ok to round to 4 digits here)
        for seg, ptt in zip(self.segments_in_pupil, rounded_ptt_list):
            metadata.append(MetaDataEntry(f"Segment {seg}", f"SEG{seg}", str(ptt),
                                          f"Piston/GradX/GradY applied for segment {seg}"))
        return metadata

    def display(self, display_wavefront=True, display_psf=True, psf_rotation_angle=0., vmax_opd=0.5e-6*u.meter, vmin_psf=1e-8, vmax_psf=1e-2,
                save_figures=True, figure_name_prefix='', out_dir=''):
        """
        Display either the deployed mirror state ("wavefront") and/or the PSF created
        by this mirror state.

        :param display_wavefront: bool, If true, display the deployed mirror state
        :param display_psf: bool, If true, display the simulated PSF created by the
            mirror state
        :param psf_rotation_angle: int/float, Degree value by which to rotate the simulated
            PSF in order to match your output
        :param figure_name_prefix: str, String to be added to filenames of output figures
        :param out_dir: str, name of output directory
        :param save_figures: bool, If true, save out the figures in the directory specified
                             by out_dir
        """
        # update opd on mirror
        self.apply_current_wavefront()

        # make figure
        if figure_name_prefix:
            figure_name_prefix = f'{figure_name_prefix}_'
        if display_wavefront:
            self.plot_wavefront(figure_name_prefix, out_dir, vmax=vmax_opd, save_figure=save_figures)
        if display_psf:
            self.plot_psf(rotation_angle=psf_rotation_angle, vmin=vmin_psf, vmax=vmax_psf, figure_name_prefix=figure_name_prefix, out_dir=out_dir,
                          save_figure=save_figures)

    def plot_wavefront(self, figure_name_prefix, out_dir, vmax=0.5e-6*u.meter, save_figure=True):
        """
        Plot the deployed mirror state (wavefront error)

        :param figure_name_prefix: str, String to be added to filenames of output figures
        :param out_dir: str, name of output directory
        :param vmax: astropy unit quantity, the maximum value to display in the plot (as expected
                     by Poppy)
        :param save_figure: bool, If true, save out the figures in the directory specified by
                            out_dir
        """
        plt.figure()
        self.aperture.display(what='opd', title='Wavefront error applied to the active segments',
                              opd_vmax=vmax)
        if save_figure:
            plt.savefig(os.path.join(out_dir, f'{figure_name_prefix}wfe_on_dm.png'))
            plt.close()
        else:
            plt.show()

    def apply_current_wavefront(self):
        """
        Get the deployed mirror state (wavefront error)
        To make plot:
        total_iris = iris_ao.HicatSegmentedDmCommand()
        total_iris.read_initial_command([tuple(row) for row in command_to_plot])
        total_iris.apply_current_wavefront()
        command = total_iris.aperture.sample(what='opd')
        image = ax.matshow(command)
        """
        display_data = set_to_dm_limits(self.data)
        # Convert the PTT list from DM to Poppy units
        converted_list = convert_ptt_units(display_data, tip_factor=1, tilt_factor=-1,
                                           starting_units=self.dm_command_units,
                                           ending_units=(u.m, u.rad, u.rad))
        # We want to round to four significant digits when in DM units (um, mrad, mrad).
        # Here, we are in SI units (m, rad, rad), so we round to the equivalent, 10 decimals.
        rounded_list = round_ptt_list(converted_list, decimals=10)
        for seg, values in zip(self.aperture.segmentlist, rounded_list):
            self.aperture.set_actuator(seg, values[0], values[1], values[2])

    @poppy.utils.quantity_input(display_wavelength=u.nm)   # decorator provides a check on input units
    def plot_psf(self, wavelength=640*u.nm, figure_name_prefix=None, rotation_angle=0,  out_dir=None, vmin=1e-8,
                 pixelscale=0.010, instrument_fov=1.0,
                 vmax=1e-2, save_figure=True):
        """
        Plot the simulated PSF based on the mirror state. Optionally save figure to a file as well

        :param wavelength: Wavelength to calculate and display a PSF for
        :param pixelscale: pixel scale to calculate and display the PSF on
        :param instrument_fov: instrument field of view to calculate and display the PSF on
        :param rotation_angle: float, the rotation to apply to the PSF image in order to match
                               our testbed. Note that this value is specific to each testbed
        :param save_figure: bool, If true, save out the figures in the directory specified by
                            out_dir
        :param figure_name_prefix: str, String to be added to filenames of output figures
        :param out_dir: str, name of output directory
        :param vmin: float, the minimum value to display in the plot
        :param vmax: float, the maximum value to display in the plot
       """
        plt.figure()
        osys = poppy.OpticalSystem()
        osys.add_pupil(self.aperture)
        osys.add_detector(pixelscale=pixelscale, fov_arcsec=instrument_fov)
        osys.add_rotation(angle=rotation_angle)

        psf = osys.calc_psf(wavelength=wavelength)
        poppy.display_psf(psf, vmin=vmin, vmax=vmax,
                          title='PSF created by the shape put on the active segments')
        if save_figure:
            if out_dir is None or figure_name_prefix is None:
                raise ValueError("Must specify out_dir and figure_name_prefix if save_figure is True")
            plt.savefig(os.path.join(out_dir, f'{figure_name_prefix}simulated_psf.png'))
            plt.close()

def load_command(segment_values, dm_config_id,
                 apply_flat_map=True, filename_flat=None):
    """
    Loads the segment_values from a file or list and returns a SegmentedDmCommand object.

    There are only two allowed segment mappings for the input formats, Native and Centered Pupil.
    See the README for the Iris AO for more details.

    :param segment_values: str, list. Can be .PTT111, .ini files or a list with piston, tip,
                           tilt values in a tuple for each segment. For the list, the first
                           element is the center or top of the innermost ring of the pupil,
                           and subsequent elements continue up and/or clockwise around the
                           pupil (see README for more information)
    :param dm_config_id: str, name of the section in the config_ini file where information
                         regarding the segmented DM can be found.
    :param apply_flat_map: bool, whether to apply a flat map in addition to the data when sending command to hardware
    :param filename_flat: string, full path to the custom flat map, only needed if apply_flat_map=True
    :return: SegmentedDmCommand object representing the command dictionary.
    """
    dm_command_obj = SegmentedDmCommand(dm_config_id=dm_config_id,
                                        apply_flat_map=apply_flat_map,
                                        filename_flat=filename_flat)
    dm_command_obj.read_initial_command(segment_values)

    return dm_command_obj

## POPPY
def round_ptt_list(ptt_list, decimals=3):
    """
    Make sure that the PTT coefficients are rounded to a reasonable number of decimals

    :param ptt_list: list, of tuples existing of piston, tip, tilt, values for each
                     segment in a pupil
    :return: list of coefficients for piston, tip, and tilt, for your pupil rounded to
             the specified number of decimal places
    """
    return [(np.round(ptt[0], decimals), np.round(ptt[1], decimals),
             np.round(ptt[2], decimals)) for ptt in ptt_list]

def convert_ptt_units(ptt_list, tip_factor, tilt_factor, starting_units, ending_units):
    # TODO: this should follow the logic from here: https://github.com/spacetelescope/catkit/blob/8c1e22e8f96d51f4453bba27c4710d27da5d37c5/catkit/emulators/iris_ao_controller.py#L50
    # This is used in display, where it's not super important, and it's used in the global Zernike commands, so those
    # are not super reliable at this point.
    """
    Convert the PTT list to or from Poppy units and the segmented DM units.

    Note that this has been created for the IrisAO segmented DMs, therefore
    the tip and tilt values are swapped with what Poppy desginates.

    Poppy to segmented DM:
    - tip_factor = -1
    - tilt_facotr = 1
    - starting_units = (u.m, u.rad, u.rad)
    - ending_units = dm_ptt_units from the config.ini file

    Segmented DM to Poppy
    - tip_factor = 1
    - tilt_factor = -1
    - starting_units = dm_ptt_units from the config.ini file
    - ending_units = (u.m, u.rad, u.rad)

    :param ppt_list: list, of tuples existing of piston, tip, tilt, values for each
                     segment in a pupil, in the respective starting_units
    :param tip_factor: int, either -1 or 1 based on the information above
    :param tilt_factor: int, either -1 or 1 based on the information above
    :param starting_units: tuple or list of the units associated with the piston, tip,
                           tilt values respectively of the input ptt_list
    :param ending_units: tuple_or_list of the units associated with the piston, tip,
                           tilt values respectively of the expected output
    :return: list of tuples of the piston, tip, tilt values for each segment listed,
             in the respective ending_units
    """
    converted = [(ptt[0]*(starting_units[0]).to(ending_units[0]),
                  tip_factor*ptt[2]*(starting_units[2]).to(ending_units[2]),
                  tilt_factor*ptt[1]*(starting_units[1]).to(ending_units[1])) for ptt in ptt_list]
    return converted

def set_to_dm_limits(ptt_list, limit=5.):
    """
    Check that the values for piston, tip, and tilt are not exceeding the hardware
    limit and reset to limit if limit is exceeded. These limits are the same as what
    the IrisAO GUI has set.

    :param ppt_list: list, of tuples existing of piston, tip, tilt, values for each
                     segment in a pupil, in DM units
    :param limit: float, in DM units. Default = 5.
    :return: list of tuples of the piston, tip, tilt values in DM units for each segment listed
             such that none of the values exceed the limit
    """
    updated = [tuple(min(i, limit) for i in ptt) for ptt in ptt_list]

    return updated

def get_wavefront_from_coeffs(coeff_list, basis):
    """
    Get the wavefront from the coefficients created by the basis given. This gives
    the per-segment wavefront based on the global coefficients given and the basis
    created.

    :params basis: POPPY Segment_PTT_Basis object, basis created that represents pupil
    :params coeff_list: list, per-segment list of piston, tip, tilt values for
                         the pupil described by basis
    :return wavefront, POPPY wavefront
    """
    wavefront = poppy.zernike.opd_from_zernikes(coeff_list, basis=basis)
    return wavefront

class PoppySegmentedDmCommand(SegmentedDmCommand):
    """ Create segmented DM command based on provided wavefront parameters

    This is currently limited to global shapes defined by Zernike coefficients

    The output is
    a SegmentedDmCommand containing piston, tip, tilt values with DM units,
    usually (um, mrad, mrad), respectively, for each segment.

    This class inherits the SegmentedAperture class.

    To use to get command for the segmented DM:
      command = PoppySegmentedDmCommand(coefficients, dm_config_id)
      dm_object.apply_shape(command)

    The method .to_dm_list() will place the command in the correct units to be sent to the hardware;
    this happens automatically in __init__.

    :param global_coefficients: list of global zernike coefficients in the form
                                [piston, tip, tilt, defocus, ...] (Noll convention)
                                in meters of optical path difference (not waves)
    :param dm_config_id: str, name of the section in the config_ini file where information
                         regarding the segmented DM can be found.
    :param display_wavelength: float, wavelength in nm of the poppy optical system used for
                        (extremely oversimplified) focal plane simulations
    :param rotation: float, rotation angle of the hex segmented DM in deg
    :attribute radius: float, half of the flat-to-flat distance of each segment
    :attribute num_terms: int, total number of PTT values on all segments (= 3 x number of segments)
    :attribute dm_command_units: list, the units of the piston, tip, tilt (respecitvely)
                                 values when coming from the DM or DM command
    :attribute global_coefficients: list of global zernike coefficients in the form
                                [piston, tip, tilt, defocus, ...] (Noll convention)
                                in meters of optical path difference (not waves)
    :attribute basis: poppy.zernike.Segment_PTT_Basis object, basis based on the characteristics
                      of the segmented DM being used
    :attribute list of coefficients: list of piston, tip, tilt coefficients in units of m, rad, rad
    """
    @poppy.utils.quantity_input(display_wavelength=u.nm)   # decorator provides a check on input units
    def __init__(self, global_coefficients, dm_config_id, rotation=0, **kwargs):
        # Initialize parent class
        super().__init__(dm_config_id,rotation=rotation, **kwargs)

        self.radius = (self.aperture.flat_to_flat/2).to(u.m)
        self.num_terms = (self.aperture.number_segments_in_pupil) * 3

        # Grab the units of the DM for the piston, tip, tilt values to convert to
        dm_command_units = CONFIG_INI.get(self.dm_config_id, 'dm_ptt_units').split(',')
        self.dm_command_units = [u.Unit(dm_command_units[0]), u.Unit(dm_command_units[1]),
                                 u.Unit(dm_command_units[2])]

        self.global_coefficients = global_coefficients

        # Create the specific basis for this pupil
        self.basis = self.create_ptt_basis()

        # Create list of segment coefficients
        self.list_of_coefficients = self.get_list_from_global()

        # Set the command data based on that list of coefficients
        self.read_initial_command(self.to_dm_list())

    def create_ptt_basis(self):
        """
        Create the basis needed for getting the per/segment coeffs back

        :return: Poppy Segment_PTT_Basis object for the specified pupil
        """
        pttbasis = poppy.zernike.Segment_PTT_Basis(rings=self.aperture._num_rings,
                                                   flattoflat=self.aperture.flat_to_flat,
                                                   gap=self.aperture.gap,
                                                   segmentlist=self.aperture._segment_list)
        return pttbasis

    def create_wavefront_from_global(self, global_coefficients, wavelength=640*u.nm):
        """
        Given an list of global coefficients, create wavefront

        :param global_coefficients: list of global zernike coefficients in the form
                                    [piston, tip, tilt, defocus, ...] (Noll convention)
                                    in meters of optical path difference (not waves)
        :param wavelength: Wavelength to create this wavefront at; does not actually affect the
                                    values of the OPD, which is modeled as wavelength-independent.
        :return: Poppy ZernikeWFE object, the global wavefront described by the input coefficients
        """
        wavefront = poppy.ZernikeWFE(radius=self.radius, coefficients=global_coefficients)
        wavefront_out = wavefront.sample(wavelength=wavelength,
                                         grid_size=2*self.radius,
                                         npix=512, what='opd')
        return wavefront_out

    def get_coeffs_from_pttbasis(self, wavefront):
        """
        From a the speficic pttbasis, get back the coeff_list that will be sent as
        a command to the Iris AO

        :param wavefront: Poppy ZernikeWFE object, the global wavefront over the
                          pupil
        :return: list, (piston, tip, tilt) values for each segment in the pupil
                 in units of [m] for piston, and [rad] for tip and tilt
        """
        coeff_list = poppy.zernike.opd_expand_segments(wavefront, nterms=self.num_terms,
                                                        basis=self.basis)
        coeff_list = np.reshape(coeff_list, (self.aperture.number_segments_in_pupil, 3))
        return coeff_list

    def get_list_from_global(self):
        """
        From a global coeff list, get back the per-segment coefficient list

        :return: list of coefficients for piston, tip, and tilt, for your pupil
        """
        wavefront = self.create_wavefront_from_global(self.global_coefficients)
        coeffs_list = self.get_coeffs_from_pttbasis(wavefront)

        return coeffs_list

    def to_dm_list(self):
        """
        Convert the PTT list to DM hardware units so that it can be passed to the
        SegmentDmCommand class

        :return: list of coefficients for piston, tip, and tilt, for your pupil, in DM units,
                 usually um, mrad, mrad unless otherwise specified in the config
        """
        input_list = self.list_of_coefficients
        # Convert from Poppy's m, rad, rad to the DM units
        input_list = convert_ptt_units(input_list, tip_factor=-1, tilt_factor=1,
                                       starting_units=(u.m, u.rad, u.rad),
                                       ending_units=self.dm_command_units)
        coeffs_list = round_ptt_list(input_list)    # since input_list is usually in units of (um, mrad, mrad), it is ok to round to 4 digits here)

        return coeffs_list


#-------------------------------------------------------------------------------
# Copyright (c) 2012 Gael Honorez.
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the GNU Public License v3.0
# which accompanies this distribution, and is available at
# http://www.gnu.org/licenses/gpl.html
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#-------------------------------------------------------------------------------

from PyQt4 import QtCore, QtGui
from PyQt4 import QtWebKit
from stat import *
import util
import urllib
import logging
import os
from fa import maps
from vault import luaparser
import urllib2
import re
import json

logger = logging.getLogger("faf.vault")
logger.setLevel(logging.DEBUG)

class MapVault(QtCore.QObject):
    def __init__(self, client, *args, **kwargs):
        QtCore.QObject.__init__(self, *args, **kwargs)
        self.client = client

        logger.debug("Map Vault tab instantiating")

        self.ui = QtWebKit.QWebView()
        self.ui.page().mainFrame().javaScriptWindowObjectCleared.connect(self.addScript)

        self.client.mapsTab.layout().addWidget(self.ui)

        self.loaded = False
        self.client.showMaps.connect(self.reloadView)
        self.ui.loadFinished.connect(self.ui.show)
        self.reloadView()

    @QtCore.pyqtSlot()
    def reloadView(self):
        if (self.loaded):
            return
        self.loaded = True

        self.ui.setVisible(False)

        #If a local theme CSS exists, skin the WebView with it
        if util.themeurl("vault/style.css"):
            self.ui.settings().setUserStyleSheetUrl(util.themeurl("vault/style.css"))

        self.ui.setUrl(QtCore.QUrl("http://www.faforever.com/faf/vault/maps.php?username={user}&pwdhash={pwdhash}".format(user=self.client.login, pwdhash=self.client.password)))

    @QtCore.pyqtSlot()
    def addScript(self):
        frame  = self.ui.page().mainFrame()
        frame.addToJavaScriptWindowObject("webVault", self)

    def __preparePositions(self, positions, map_size):
        img_size = [256, 256]
        size = [int(map_size['0']), int(map_size['1'])]
        off_x = 0
        off_y = 0

        if (size[1] > size[0]):
            img_size[0] = img_size[0]/2
            off_x = img_size[0]/2
        elif (size[0] > size[1]):
            img_size[1] = img_size[1]/2
            off_y = img_size[1]/2

        cf_x = size[0]/img_size[0]
        cf_y = size[1]/img_size[1]

        regexp = re.compile(" \d+\.\d*| \d+")

        for postype in positions:
            for pos in positions[postype]:
                string = positions[postype][pos]
                values = regexp.findall(positions[postype][pos])
                x = off_x + float(values[0].strip())/cf_x
                y = off_y + float(values[2].strip())/cf_y
                positions[postype][pos] = [int(x), int(y)]

    @QtCore.pyqtSlot()
    def uploadMap(self):

        mapDir = QtGui.QFileDialog.getExistingDirectory (self.client, "Select the map directory to upload", maps.getUserMapsFolder(),  QtGui.QFileDialog.ShowDirsOnly)
        logger.debug("Uploading map from: " + mapDir)
        if mapDir != "" :
            if maps.isMapFolderValid(mapDir) :
                os.chmod(mapDir, S_IWRITE)
                mapName = os.path.basename(mapDir)
                zipName = mapName.lower()+".zip"

                scenariolua = luaparser.luaParser(os.path.join(mapDir,maps.getScenarioFile(mapDir)))
                scenarioInfos = scenariolua.parse({'scenarioinfo>name':'name', 'size':'map_size', 'description':'description', 'count:armies':'max_players','map_version':'version','type':'map_type','teams>0>name':'battle_type'}, {'version':'1'})

                if scenariolua.error:
                    logger.debug("There were " + str(scenariolua.errors) + " errors and " + str(scenariolua.warnings) + " warnings.")
                    logger.debug(scenariolua.errorMsg)
                    QtGui.QMessageBox.critical(self.client, "Lua parsing error", scenariolua.errorMsg + "\nMap uploading cancelled.")
                else:
                    if scenariolua.warning:
                        uploadmap = QtGui.QMessageBox.question(self.client, "Lua parsing warning", scenariolua.errorMsg + "\nDo you want to upload the map?", QtGui.QMessageBox.Yes, QtGui.QMessageBox.No)
                    else:
                        uploadmap = QtGui.QMessageBox.Yes
                    if uploadmap == QtGui.QMessageBox.Yes:
                        savelua = luaparser.luaParser(os.path.join(mapDir,maps.getSaveFile(mapDir)))
                        saveInfos = savelua.parse({'markers>mass*>position':'mass:__parent__', 'markers>hydro*>position':'hydro:__parent__', 'markers>army*>position':'army:__parent__'})
                        if savelua.error or savelua.warning:
                           logger.debug("There were " + str(scenariolua.errors) + " errors and " + str(scenariolua.warnings) + " warnings.")
                           logger.debug(scenariolua.errorMsg)

                        self.__preparePositions(saveInfos, scenarioInfos["map_size"])

                        tmpFile = maps.processMapFolderForUpload(mapDir, saveInfos)
                        if not tmpFile:
                            QtGui.QMessageBox.critical(self.client, "Map uploading error", "Couldn't make previews for " + mapName + "\nMap uploading cancelled.")
                            return None

                        qfile = QtCore.QFile(tmpFile.name)
                        self.client.writeToServer("UPLOAD_MAP", zipName, scenarioInfos, qfile)

                        #removing temporary files
                        qfile.remove()
            else :
                QtGui.QMessageBox.information(self.client,"Map selection",
                        "This folder doesn't contain valid map datas.")

    @QtCore.pyqtSlot(str)
    def downloadMap(self, link):
        link = urllib2.unquote(link)
        name = maps.link2name(link)
        if not maps.isMapAvailable(name):
            maps.downloadMap(name)
            maps.existMaps(True)
        else:
            show = QtGui.QMessageBox.question(self.client, "Already got the Map", "Seems like you already have that map!
Would you like to see it?", QtGui.QMessageBox.Yes, QtGui.QMessageBox.No)
            if show == QtGui.QMessageBox.Yes:
                util.showInExplorer(maps.folderForMap(name))

    @QtCore.pyqtSlot(str)
    def checkMaps(self, data):
        exist_maps = list()
        viewed_maps = json.loads(data)
        for id, name in viewed_maps.iteritems():
            if maps.mapExists(name):
               exist_maps.append(id)
        if len(exist_maps) > 0:
            json_str = json.dumps(exist_maps).replace('"', '\\"')
            self.ui.page().mainFrame().evaluateJavaScript('handle_exist_maps("%s")' % json_str)