"""
The production code for predicting smell events and sending push notifications
(sending push notifications requires the rake script in the smell-pittsburgh-rails repository)
"""

import sys
from util import log, generateLogger, computeMetric, isFileHere
import pandas as pd
from getData import getData
from preprocessData import preprocessData
from computeFeatures import computeFeatures
#from selectFeatures import selectFeatures
from trainModel import trainModel
import joblib
from datetime import timedelta, datetime
import os
import subprocess

# The flag to determine the server type
SERVER = "staging"
#SERVER = "production"

# The flag for enabling the rake call to send push notifications
#ENABLE_RAKE_CALL = False
ENABLE_RAKE_CALL = True

# The path for storing push notification data
DATA_PATH = "data_production/"

def main(argv):
    mode = None
    if len(argv) >= 2:
        mode = argv[1]

    if mode == "train":
        train()
    elif mode == "predict":
        predict()
    else:
        print("Use 'python main.py [mode]'; mode can be 'train' or 'predict'")

def train(f_hr=8, b_hr=3, thr=40, method="HCR"):
    """
    Train the machine learning model for predicting smell events

    Input:
        f_hr: the number of hours to look further and compute responses (Y),
            ...which is the sum of smell ratings (that are larger than 3) over the future f_hr hours
        b_hr: the number of hours to look back and compute features (X),
            ...which are the sensor readings (on ESDR) over the past b_hr hours
        thr: the threshold for binning the smell value into two classes (for classification)
        method: the method used in the "trainModel.py" file
    """
    p = DATA_PATH

    # Set logger
    logger = generateLogger(p+"log.log")
    log("--------------------------------------------------------------------------", logger)
    log("---------------------------------  Train  --------------------------------", logger)

    # Get and preprocess data
    end_dt = datetime.now() - timedelta(hours=24)
    start_dt = end_dt - timedelta(hours=8000)
    log("Get data from " + str(start_dt) + " to " + str(end_dt), logger)
    df_esdr_array_raw, df_smell_raw = getData(start_dt=start_dt, end_dt=end_dt, logger=logger)
    df_esdr, df_smell = preprocessData(df_esdr_array_raw=df_esdr_array_raw, df_smell_raw=df_smell_raw, logger=logger)

    # Compute features
    df_X, df_Y, df_C = computeFeatures(df_esdr=df_esdr, df_smell=df_smell, f_hr=f_hr, b_hr=b_hr, thr=thr, is_regr=False,
        add_inter=False, add_roll=False, add_diff=False, logger=logger, out_p_mean=p+"mean.csv", out_p_std=p+"std.csv")

    # Select features
    # NOTE: currently, the best model uses all the features
    #df_X, df_Y = selectFeatures(df_X, df_Y, logger=logger, out_p=p+"feat_selected.csv")

    # Train, save, and evaluate model
    # NOTE: to know more about the model, see the "HybridCrowdClassifier.py" file
    model = trainModel({"X": df_X, "Y": df_Y, "C": df_C}, method=method, out_p=p+"model.pkl", logger=logger)
    metric = computeMetric(df_Y, model.predict(df_X, df_C), False)
    for m in metric:
        log(metric[m], logger)

def predict(f_hr=8, b_hr=3, thr=40):
    """
    Predict smell events using the trained machine learning model

    For the description of the input arguments, see the docstring in the train() function
    """
    p = DATA_PATH

    # Set logger
    logger = generateLogger(p+"log.log")
    log("--------------------------------------------------------------------------", logger)
    log("--------------------------------  Predict  -------------------------------", logger)

    # Get data for previous b_hr hours
    end_dt = datetime.now()
    start_dt = end_dt - timedelta(hours=b_hr+1)
    log("Get data from " + str(start_dt) + " to " + str(end_dt), logger)
    df_esdr_array_raw, df_smell_raw = getData(start_dt=start_dt, end_dt=end_dt, logger=logger)
    df_esdr, df_smell = preprocessData(df_esdr_array_raw=df_esdr_array_raw, df_smell_raw=df_smell_raw, logger=logger)
    if len(df_esdr) < b_hr+1:
        log("ERROR: Length of esdr is less than " + str(b_hr+1) + " hours", logger)
        log("Length of esdr = " + str(len(df_esdr)), logger)
        return

    # Compute features
    df_X, _, df_C = computeFeatures(df_esdr=df_esdr, df_smell=df_smell, f_hr=f_hr, b_hr=b_hr, thr=thr, is_regr=False,
        add_inter=False, add_roll=False, add_diff=False, logger=logger, in_p_mean=p+"mean.csv", in_p_std=p+"std.csv")
    if len(df_X) != 1:
        log("ERROR: Length of X is not 1", logger)
        log("Length of X = " + str(len(df_X)), logger)
        return

    # Select features
    # NOTE: currently, the best model uses all the features
    #df_feat_selected = pd.read_csv(p+"feat_selected.csv")
    #df_X = df_X[df_feat_selected.columns]

    # Load model
    log("Load model...", logger)
    model = joblib.load(p+"model.pkl")

    # Predict result
    # For the hybrid crowd classifier
    # if pred==0, no event
    # if pred==1, event predicted by the base estimator
    # if pred==2, event detected by the crowd
    # if pred==3, event both predicted by the base estimator and detected by the crowd
    y_pred = model.predict(df_X, df_C)[0]
    log("Prediction for " + str(end_dt) + " is " + str(y_pred), logger)

    # Send the predictive push notification
    if y_pred in (1, 3): pushType1(end_dt, logger)

    # Send the crowd-based notification
    # NOTE: comment out the next line when migrating its function to the rails server
    if y_pred in (2, 3): pushType2(end_dt, logger)

def pushType1(end_dt, logger):
    """
    Send type 1 push notification (predicted by the classifier)

    Input:
        end_dt: the ending time for getting the ESDR data that is used for prediction (which is the current time)
        logger: the python logger created by the generateLogger() function
    """
    p = DATA_PATH

    # Read the push notification file
    nst_p = p + "notification_sent_times.csv"
    if isFileHere(nst_p):
        # If the file that stores the notification sending time exist,
        # ...load the times and check if we already sent the notification at the same date before
        df_nst = pd.read_csv(nst_p, parse_dates=["DateTime"])
        last_date = df_nst["DateTime"].dt.date.iloc[-1] # the last date that we send the notification
        current_date = end_dt.date()
        if current_date == last_date: # check if the dates (only year, month, and day) match
            # We already sent push notifications to users today, do not send it again until next day
            log("Ignore this prediction because we already sent a push notification today", logger)
            return
    else:
        # If the file did not exist, create a new pandas Dataframe to store the time when we send notifications
        df_nst = pd.DataFrame(data=[], columns=["DateTime"])

    # Send push notification to users
    if ENABLE_RAKE_CALL:
        # NOTE: Python by default uses the sh terminal but we want it to use bash,
        # ...because "source" and "bundle" only works for bash on the Hal11 machine
        # ...(on the sh terminal we will want to use "." instead of "source")
        cmd = 'source /etc/profile ; cd /var/www/rails-apps/smellpgh/' + SERVER + '/current/ ; bundle exec rake firebase_push_notification:send_prediction["/topics/SmellReports"] RAILS_ENV=' + SERVER + ' >> /var/www/smell-pittsburgh-prediction/py/prediction/data_production/push.log 2>&1'
        subprocess.call(["bash", "-c", cmd])

    # Create a CSV file that writes the time when the system send the push notification
    log("A prediction push notification was sent to users", logger)
    df_nst = df_nst.append({"DateTime": end_dt}, ignore_index=True) # append a row to the pandas Dataframe
    df_nst.to_csv(nst_p, index=False) # re-write the Dataframe to a CSV file

def pushType2(end_dt, logger):
    """
    Send type 2 push notification (verified by the crowd)

    Input:
        end_dt: the ending time for getting the ESDR data that is used for prediction (which is the current time)
        logger: the python logger created by the generateLogger() function
    """
    p = DATA_PATH

    # Read the crowd push notification file
    cvnst_p = p + "crow_verified_notification_sent_times.csv"
    if isFileHere(cvnst_p):
        # If the file that stores the notification sending time exist,
        # ...load the times and check if we already sent the notification at the same date before
        df_cvnst = pd.read_csv(cvnst_p, parse_dates=["DateTime"])
        last_date = df_cvnst["DateTime"].dt.date.iloc[-1] # the last date that we send the notification
        current_date = end_dt.date()
        if current_date == last_date: # check if the dates (only year, month, and day) match
            # We already sent crowd-verified push notifications to users today, do not send it again until next day
            log("Ignore this crowd-verified event because we already sent a push notification today", logger)
            return
    else:
        # If the file did not exist, create a new pandas Dataframe to store the time when we send notifications
        df_cvnst = pd.DataFrame(data=[], columns=["DateTime"])

    # Send crowd-verified push notification to users
    if ENABLE_RAKE_CALL:
        # NOTE: Python by default uses the sh terminal but we want it to use bash,
        # ...because "source" and "bundle" only works for bash on the Hal11 machine
        # ...(on the sh terminal we will want to use "." instead of "source")
        cmd = 'source /etc/profile ; cd /var/www/rails-apps/smellpgh/' + SERVER + '/current/ ; bundle exec rake firebase_push_notification:send_prediction_type2["/topics/SmellReports"] RAILS_ENV=' + SERVER + ' >> /var/www/smell-pittsburgh-prediction/py/prediction/data_production/crow_verified_push.log 2>&1'
        subprocess.call(["bash", "-c", cmd])

    # Create a CSV file that writes the time when the system send the push notification
    log("A crowd-verified push notification was sent to users", logger)
    df_cvnst = df_cvnst.append({"DateTime": end_dt}, ignore_index=True) # append a row to the pandas Dataframe
    df_cvnst.to_csv(cvnst_p, index=False) # re-write the Dataframe to a CSV file

if __name__ == "__main__":
    main(sys.argv)


import pdfplumber
import os,re
file_path = "/home/FDDC_announcements_round2_train_pdf/"

def  pdf_tbl2txt(file):
    pdf = pdfplumber.open(file_path + "my.pdf")
    for i in pdf.pages:
        # page = pdf.pages[0]
        # i.extract_table()
        if i.find_tables(table_settings={}):
            i.crop(boundiipng_box)


import json
import torch
from transformers import GPT2Tokenizer
from transformers import GPT2DoubleHeadsModel
from MTDNN import MTDNN
from tqdm import trange, tqdm
from keras_preprocessing import sequence
import pandas as pd
import Utils
import pickle
import os
from torch.utils.data import TensorDataset, DataLoader
from torch.utils.tensorboard import SummaryWriter
import datetime

SPECIAL_TOKENS = ['<pad>', '<eos>', '<rstokn>', '<bos>', '<question>', '<commonsensetask>', '<cose>', '<openbook>']
ATTR_TO_SPECIAL_TOKEN = {'bos_token': '<bos>', 'pad_token': '<pad>', 'eos_token': '<eos>',
                         'additional_special_tokens': ['<rstokn>', '<question>', '<reply>', '<commonsensetask>', '<cose>', '<openbook>']
                         }

current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
logs_dir_tensorboard = "runs2nomcs/" + (str(current_time) + "morecheckpoint-melco-update-ros")
writer = SummaryWriter(logs_dir_tensorboard)

device = 'cuda:5'
def data_preprocess():
    final_data = []
    questions = []
    choices = []
    label = []
    facts = []

    file_name = 'data/OpenBookFacts/train_complete.jsonl'
    for line in open(file_name, 'r') :
        data = (json.loads(line))

        questions.append(data['question']['stem'])
        choices.append([data['question']['choices'][0]['text'], data['question']['choices'][1]['text'],
                       data['question']['choices'][2]['text'], data['question']['choices'][3]['text']])
        if data['answerKey'] == 'A' :
            answer = 0
        elif data['answerKey'] == 'B' :
            answer = 1
        elif data['answerKey'] == 'C' :
            answer = 2
        else:
            answer = 3
        label.append(answer)
        facts.append(data['fact1'])

    openBook_Data = [["openBook"],  questions, choices, label, facts]
    final_data.append(openBook_Data)

    file_name = 'data/CoS-E/cose_train_data.csv'
    data = pd.read_csv(file_name)
    final_data.append([["CoS-E"], data])

    file_name_1 = 'data/commonsense/subtaskB_data_all-2.csv'
    file_name_2 = 'data/commonsense/subtaskC-alldata.csv'
    data1 = pd.read_csv(file_name_1)
    data2 = pd.read_csv(file_name_2)
    data = data1.merge(data2, on='FalseSent').dropna()
    final_data.append([["commonsense"], data]) # leave the last 500

    return final_data

def convert_to_tokens(input, tokenizer):
    if isinstance(input, str):
        return tokenizer.convert_tokens_to_ids(tokenizer.tokenize(input))
    elif isinstance(input, list):
        return [
            tokenizer.convert_tokens_to_ids(tokenizer.tokenize(val))
            if not isinstance(val, int) else val
            for val in input
        ]
    elif isinstance(input, pd.Series):
        input = input.tolist()
        return [
            tokenizer.convert_tokens_to_ids(tokenizer.tokenize(val))
            if not isinstance(val, int) else val
            for val in input
        ]
    else:
        import sys
        print("Conversion Error")
        sys.exit()

def padding_falsesent_choices(datas, tokenizer):

    pad = [tokenizer.convert_tokens_to_ids(SPECIAL_TOKENS[0])]
    eos = [tokenizer.convert_tokens_to_ids(SPECIAL_TOKENS[1])]
    rstokn = [tokenizer.convert_tokens_to_ids(SPECIAL_TOKENS[2])]
    bos = [tokenizer.convert_tokens_to_ids(SPECIAL_TOKENS[3])]
    questons = [tokenizer.convert_tokens_to_ids(SPECIAL_TOKENS[4])]
    commonsensetask = [tokenizer.convert_tokens_to_ids(SPECIAL_TOKENS[5])]
    COSE = [tokenizer.convert_tokens_to_ids(SPECIAL_TOKENS[6])]
    openBook = [tokenizer.convert_tokens_to_ids(SPECIAL_TOKENS[7])]
    choice_padding = -1

    input_ids = []
    lm_labels = []
    token_type_ids = []
    mc_token_ids = []
    mc_labels = []

    max_length = 128

    for data in datas:
        if data[0] == ["openBook"]:

            for question, choices, labels, facts in zip( data[1], data[2], data[3], data[4]):

                # /mc_labels = []

                question, choices, facts = convert_to_tokens(question, tokenizer), convert_to_tokens(choices, tokenizer), convert_to_tokens(facts, tokenizer)
                input1 = [bos + openBook + rstokn + question + rstokn + choices[0] + rstokn + choices[1] + rstokn + choices[2] + rstokn + choices[3] + eos]
                input2 = [bos + openBook + rstokn + question + rstokn + facts + eos]

                mc_token_ids.append(len(input1[0]))
                mc_token_ids.append(len(input2[0]))

                input1 = sequence.pad_sequences(input1, maxlen=max_length, padding='post', value=pad)
                input_ids.append(input1[0])

                fakechoice = sequence.pad_sequences([[-1]], maxlen=max_length, padding='post', value=choice_padding)
                lm_labels.append(fakechoice[0])

                tt_id1 = [
                    (len(openBook) + 1) * rstokn + (len(question) + 1) * questons + (len(choices[0]) + 1) * rstokn +
                    (len(choices[1]) + 1) * questons + (len(choices[2]) + 1) * rstokn + (len(choices[3]) + 2) * questons
                ]
                tt_id1 = sequence.pad_sequences(tt_id1, maxlen=max_length, padding='post', value=pad)
                token_type_ids.append(tt_id1[0])

                input2 = sequence.pad_sequences(input2, maxlen=max_length, padding='post', value=pad)
                input_ids.append(input2[0])

                choice = [[-1] * (len(openBook) + 2) + [-1] * len(question) + [-1] + facts + eos]
                choice = sequence.pad_sequences(choice, maxlen=max_length, padding='post', value=choice_padding)
                lm_labels.append(choice[0])

                tt_id2 = [(len(openBook) + 1) * rstokn + (len(question) + 1) * questons + (len(choices[labels]) + 2) * rstokn]
                tt_id2 = sequence.pad_sequences(tt_id2, maxlen=max_length, padding='post', value=pad)
                token_type_ids.append(tt_id2[0])

                mc_labels.append(labels)
                mc_labels.append(labels)

        elif data[0] == ["CoS-E"]:

            for idx, value in data[1].iterrows():
                value = value[1:]
                value = convert_to_tokens(value, tokenizer)
                input1 = [bos + COSE + rstokn + value[1] + rstokn + value[2] + rstokn + value[3] + rstokn + value[4] +
                          rstokn + value[5] + rstokn + value[6] + eos]
                input2 = [bos + COSE + rstokn + value[1] + rstokn + value[8] + eos]

                mc_token_ids.append(len(input1[0]))
                mc_token_ids.append(len(input2[0]))

                input1 = sequence.pad_sequences(input1, maxlen= max_length, padding='post', value=pad)
                input_ids.append(input1[0])

                fakechoice = sequence.pad_sequences([[-1]], maxlen=max_length, padding='post', value=choice_padding)
                lm_labels.append(fakechoice[0])

                tt_id1 = [(len(COSE) + 1) * rstokn + (len(value[1]) + 1) * questons + (len(value[2]) + 1) * rstokn +
                    (len(value[3]) + 1) * questons + (len(value[4]) + 1) * rstokn + (len(value[5]) + 1) * questons +
                           (len(value[6]) + 2) * rstokn]
                tt_id1 = sequence.pad_sequences(tt_id1, maxlen= max_length, padding='post', value=pad)
                token_type_ids.append(tt_id1[0])

                input2 = sequence.pad_sequences(input2, maxlen=max_length, padding='post', value=pad)
                input_ids.append(input2[0])

                choice = [[-1] * (len(COSE) + 2) + [-1] * len(value[1]) + [-1] + value[8] + eos]
                choice = sequence.pad_sequences(choice, maxlen=max_length, padding='post', value=choice_padding)
                lm_labels.append(choice[0])

                tt_id2 = [(len(COSE) + 1) * rstokn + (len(value[1]) + 1) * questons + (len(value[8]) +2) * rstokn]
                tt_id2 = sequence.pad_sequences(tt_id2, maxlen=max_length, padding='post', value=pad)
                token_type_ids.append(tt_id2[0])

                mc_labels.append(value[7])
                mc_labels.append(value[7])

        elif data[0] == ["commonsense"]:
            for idx, value in data[1].iterrows():
                # call tokenizer
                value = convert_to_tokens(value, tokenizer)
                input1 = [bos + commonsensetask + rstokn + value[1] + rstokn + value[2] + rstokn + value[3] + rstokn + value[4]+ eos]
                ml = input1
                input1 = sequence.pad_sequences(input1, maxlen=max_length, padding='post', value=pad)
                fakechoice = sequence.pad_sequences([[-1]], maxlen=max_length, padding='post', value=choice_padding)
                tt_id1 = [
                    (len(commonsensetask) + 1) * rstokn + (len(value[1]) + 1) * questons + (len(value[2]) + 1) * rstokn +
                    (len(value[3]) + 1) * questons + (len(value[4]) + 2) * rstokn
                ]
                tt_id1 = sequence.pad_sequences(tt_id1, maxlen=max_length, padding='post', value=pad)

                for i in range(3):

                    mc_token_ids.append(len(ml))
                    input_ids.append(input1[0])
                    lm_labels.append(fakechoice[0])
                    token_type_ids.append(tt_id1[0])

                    input2 = [bos + commonsensetask + rstokn + value[1] + rstokn + value[7 + i] + eos]
                    mc_token_ids.append(len(input2[0]))
                    input2 = sequence.pad_sequences(input2, maxlen=max_length, padding='post', value=pad)
                    input_ids.append(input2[0])

                    choice = [[-1] + [-1] * len(commonsensetask) + [-1] + [-1] * len(value[1]) + [-1] + value[7 + i] + eos]
                    choice = sequence.pad_sequences(choice, maxlen=max_length, padding='post', value=choice_padding)
                    lm_labels.append(choice[0])

                    tt_id2 = [(len(commonsensetask) + 1) * rstokn + (len(value[1]) + 1) * questons + (len(value[7 + i]) + 2) * rstokn]
                    tt_id2 = sequence.pad_sequences(tt_id2, maxlen=max_length, padding='post', value=pad)
                    token_type_ids.append(tt_id2[0])

                    mc_labels.append(value[5])
                    mc_labels.append(value[5])
                    # mc_labels.append(0)

    return input_ids, token_type_ids, mc_token_ids, lm_labels, mc_labels

def converting_tokens(data, tokenizer):

    print("Converting tokens to ids ...", flush=True)

    input_ids, token_type_ids, mc_token_ids, lm_labels, mc_labels = padding_falsesent_choices(data, tokenizer)

    input_ids = torch.tensor(input_ids)
    input_ids = input_ids.view((-1, 2) + input_ids.shape[1:])

    mc_token_ids = torch.tensor(mc_token_ids)
    mc_token_ids = mc_token_ids.view((-1, 2) + mc_token_ids.shape[1:])

    lm_labels = torch.tensor(lm_labels)
    lm_labels = lm_labels.view((-1, 2) + lm_labels.shape[1:])

    token_type_ids = torch.tensor(token_type_ids)
    token_type_ids = token_type_ids.view((-1, 2) + token_type_ids.shape[1:])

    mc_labels = torch.tensor(mc_labels)
    mc_labels = mc_labels.view((-1, 2) + mc_labels.shape[1:])

    pickle.dump(input_ids, open("data/pickle/input_ids.p", "wb"))
    pickle.dump(mc_token_ids, open("data/pickle/mc_token_ids.p", "wb"))
    pickle.dump(lm_labels, open("data/pickle/lm_labels.p", "wb"))
    pickle.dump(token_type_ids, open("data/pickle/token_type_ids.p", "wb"))
    pickle.dump(mc_labels, open("data/pickle/mc_labels.p", "wb"))

    return input_ids, token_type_ids, mc_token_ids, lm_labels, mc_labels

def train(model, optimizer, scheduler, train_data, output_dir, num_train_epochs, tokenizer, lm_coef, mc_coef,gradient_accumulation_steps, mgpu, temp=[], valid_data = []):

    training_loss = {}
    evaluation_loss = {}
    global_steps = 0
    for epochs in range(num_train_epochs):
        model.train()
        print("Training start for epoch {}".format(epochs), flush=True)
        nb_tr_steps, tr_loss = 0, 0
        optimizer.zero_grad()
        lm_sub_batch_loss, mc_sub_batch_loss, sub_batch_loss = 0, 0, 0
        print("sub_batch_loss \t lm_sub_batch_loss \t mc_sub_batch_loss")
        for step, batch in (enumerate(train_data)):
            model.train()
            batch = tuple(t.to(device).type(torch.cuda.LongTensor) for t in batch)
            input_ids, token_type_ids, mc_token_ids, lm_labels, mc_labels = batch

            lm_loss, mc_loss, *_ = model(
                input_ids, token_type_ids=token_type_ids, mc_token_ids=mc_token_ids,
                mc_labels=mc_labels, lm_labels=lm_labels, task=input_ids[0][0][1]
            )
            mc_loss = mc_loss[0]
            del input_ids, token_type_ids, mc_token_ids, lm_labels, mc_labels
            loss = (lm_loss * lm_coef) + (mc_loss * mc_coef)
            loss = loss.mean()
            loss /= gradient_accumulation_steps
            loss.backward()
            tr_loss += loss.item()
            lm_sub_batch_loss += lm_loss.item()
            mc_sub_batch_loss += mc_loss.item()
            sub_batch_loss += loss.item()

            if (global_steps + 1) % gradient_accumulation_steps == 0:
                optimizer.step()
                scheduler.step()
                # global_steps +=1
                optimizer.zero_grad()
                print("{} \t {} \t {}".format(sub_batch_loss, lm_sub_batch_loss/gradient_accumulation_steps, mc_sub_batch_loss/gradient_accumulation_steps))
                writer.add_scalar('Training batch loss', sub_batch_loss, global_steps+1)
                writer.add_scalar('Training lm batch loss', lm_sub_batch_loss/gradient_accumulation_steps, global_steps+1)
                writer.add_scalar('Training mc batch loss', mc_sub_batch_loss/gradient_accumulation_steps, global_steps+1)
                training_loss[(global_steps+1)] = (sub_batch_loss, lm_sub_batch_loss/gradient_accumulation_steps, mc_sub_batch_loss/gradient_accumulation_steps)
                lm_sub_batch_loss, mc_sub_batch_loss, sub_batch_loss = 0, 0, 0

            if (global_steps + 1) % 800 == 0:
                eval_loss, eval_lm_loss, eval_mc_loss = evaluate_gpt2(model, valid_data)
                print("{} \t {} \t {}".format(eval_mc_loss, eval_lm_loss, eval_mc_loss))
                writer.add_scalar('Eval total loss - 100', eval_loss, (global_steps + 1))
                writer.add_scalar('Eval total LM loss - 100', eval_lm_loss, (global_steps + 1))
                writer.add_scalar('Eval total MC loss - 100', eval_mc_loss, (global_steps + 1))
                evaluation_loss[(global_steps + 1)] = (eval_loss, eval_lm_loss, eval_mc_loss)

                if not os.path.exists(output_dir + '/' + str(global_steps + 1)):
                    os.makedirs(output_dir + '/' + str(global_steps + 1))
                torch.save(model, output_dir + '/' + str(global_steps + 1) + '/' + str(global_steps + 1) + '.pt')
                # model.save_state_dict(output_dir + '/' + str(global_steps + 1))
            global_steps += 1
        print("Epoch Completed at Step Size {}".format(global_steps))
        if not os.path.exists(output_dir + '/' + '_epoch_' + str(epochs)):
            os.makedirs(output_dir + '/' + '_epoch_' + str(epochs))
        torch.save(model, output_dir + '/' + '_epoch_' + str(epochs) + '/' + str(epochs) + '.pt')
        # model.save_state_dict(output_dir + '/' + '_epoch_' + str(epochs))

    pickle.dump(training_loss, open("data/pickle/training_loss-melco-update.p", "wb"))
    pickle.dump(evaluation_loss, open("data/pickle/evaluation_loss-melco-update.p", "wb"))

    return model
def evaluate_gpt2(model, valid_data):

    lm_sub_batch_loss, mc_sub_batch_loss = 0, 0
    model.eval()
    print("\n *************************Evaluation************************************ \n")
    for step, batch in (enumerate(valid_data)):
        batch = tuple(t.to(device).type(torch.cuda.LongTensor) for t in batch)
        input_ids, token_type_ids, mc_token_ids, lm_labels, mc_labels = batch

        lm_loss, mc_loss, *_ = model(
            input_ids, token_type_ids=token_type_ids, mc_token_ids=mc_token_ids,
            mc_labels=mc_labels, lm_labels=lm_labels, task=input_ids[0][0][1]
        )
        del input_ids, token_type_ids, mc_token_ids, lm_labels, mc_labels
        lm_sub_batch_loss += lm_loss.item()
        mc_sub_batch_loss += mc_loss[0].item()

    return (lm_sub_batch_loss + mc_sub_batch_loss)/len(valid_data), (lm_sub_batch_loss)/len(valid_data), (mc_sub_batch_loss)/len(valid_data)

def main():
    flag = True
    mgpu = True
    output_dir= 'checkpoints-More-melco-new'
    epochs = 3
    gradient_accumulation_steps = 8
    lm_coef, mc_coef = 1, 0

    token_class = GPT2Tokenizer
    model_Class = MTDNN

    gpt_model = model_Class.from_pretrained('omcs/-Final')
    # gpt_model = model_Class.from_pretrained('gpt2-large')
    gpt_tokenizer = token_class.from_pretrained('omcs/-Final', do_lower_case=True)
    # gpt_tokenizer = token_class.from_pretrained('gpt2-large', do_lower_case=True)

    gpt_model, gpt_tokenizer = Utils.add_special_tokens(gpt_model, gpt_tokenizer, ATTR_TO_SPECIAL_TOKEN)
    gpt_model.to(device)
    #gpt_model = torch.nn.DataParallel(gpt_model, output_device=1, device_ids=[0, 1])

    cache_input_ids, cache_mc_token_ids, cache_lm_labels, cache_token_type_ids, cache_mc_labels = \
            "data/pickle/input_ids.p", "data/pickle/mc_token_ids.p", "data/pickle/lm_labels.p", "data/pickle/token_type_ids.p", "data/pickle/mc_labels.p"
    if flag and os.path.exists(cache_input_ids) and os.path.exists(cache_mc_token_ids) and os.path.exists(
                cache_lm_labels) and os.path.exists(cache_token_type_ids) and os.path.exists(cache_mc_labels):
            print("Token ids loaded from previous processed file ... ", flush=True)
            input_ids, mc_token_ids, lm_labels, token_type_ids, mc_labels = pickle.load(open(cache_input_ids, "rb")), pickle.load(open(cache_mc_token_ids, "rb")), \
                                                                            pickle.load(open(cache_lm_labels, "rb")), pickle.load(open(cache_token_type_ids, "rb")), \
                                                                            pickle.load(open(cache_mc_labels, "rb"))
    else:
        data = data_preprocess()
        input_ids, token_type_ids, mc_token_ids, lm_labels, mc_labels = converting_tokens(data, gpt_tokenizer)

    temp = [input_ids, token_type_ids, mc_token_ids, lm_labels, mc_labels]
    train_data, valid_data = Utils.build_dataloader((input_ids, token_type_ids, mc_token_ids, lm_labels, mc_labels))
    train_data, valid_data = Utils.generate_batch(train_data, valid_data, 1)

    t_total = len(train_data) / epochs
    learning_rate, adam_epsilon, weight_decay, warmup_steps = 1e-5, 1e-8, 0, 0

    optimizer, scheduler = Utils.optimizer_generater(gpt_model, learning_rate, adam_epsilon, weight_decay, warmup_steps, t_total)

    model = train(gpt_model, optimizer, scheduler, train_data, output_dir, epochs, gpt_tokenizer, lm_coef, mc_coef, gradient_accumulation_steps,
                  mgpu, temp, valid_data)
    print("End of execution", flush=True)
    output_dir = output_dir + '/' + 'final'
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)
    gpt_tokenizer.save_pretrained(output_dir)

if __name__ == '__main__':
    main()


from django.db.models import Q
from django_filters.rest_framework import DjangoFilterBackend
from rest_framework.decorators import action
from rest_framework.response import Response
from rest_framework.viewsets import ModelViewSet

from gcloud.core.apis.drf.exceptions import ValidationException
from gcloud.core.apis.drf.viewsets import ApiMixin, permissions
from gcloud.label.models import Label, TemplateLabelRelation
from gcloud.label.serilaziers import LabelSerializer
from gcloud.iam_auth import IAMMeta, get_iam_client, res_factory
from iam.contrib.drf.shortcuts import allow_or_raise_immediate_response
from iam import Subject, Action

iam = get_iam_client()

class LabelViewSet(ApiMixin, ModelViewSet):
    queryset = Label.objects.all()
    serializer_class = LabelSerializer
    permission_classes = [permissions.IsAuthenticated]
    filter_backends = [DjangoFilterBackend]
    filterset_fields = "__all__"

    def list(self, request, *args, **kwargs):
        project_id = request.query_params.get("project_id")
        if not project_id:
            raise ValidationException("project_id should be provided.")
        allow_or_raise_immediate_response(
            iam=iam,
            system=IAMMeta.SYSTEM_ID,
            subject=Subject("user", request.user.username),
            action=Action(IAMMeta.PROJECT_VIEW_ACTION),
            resources=res_factory.resources_for_project(project_id),
        )
        return super(LabelViewSet, self).list(request, *args, **kwargs)

    def update(self, request, *args, **kwargs):
        label = self.get_object()
        if label.is_default:
            raise ValidationException("default label cannot be updated.")
        project_id = label.project_id
        allow_or_raise_immediate_response(
            iam=iam,
            system=IAMMeta.SYSTEM_ID,
            subject=Subject("user", request.user.username),
            action=Action(IAMMeta.PROJECT_EDIT_ACTION),
            resources=res_factory.resources_for_project(project_id),
        )
        return super(LabelViewSet, self).update(request, *args, **kwargs)

    def destroy(self, request, *args, **kwargs):
        label = self.get_object()
        if label.is_default:
            raise ValidationException("default label cannot be deleted.")
        project_id = label.project_id
        allow_or_raise_immediate_response(
            iam=iam,
            system=IAMMeta.SYSTEM_ID,
            subject=Subject("user", request.user.username),
            action=Action(IAMMeta.PROJECT_EDIT_ACTION),
            resources=res_factory.resources_for_project(project_id),
        )
        self.perform_destroy(label)
        return Response({"result": True, "message": "success"})

    @action(methods=["get"], detail=False)
    def list_with_default_labels(self, request, *args, **kwargs):
        project_id = request.query_params.get("project_id")
        if not project_id:
            raise ValidationException("project_id should be provided.")
        allow_or_raise_immediate_response(
            iam=iam,
            system=IAMMeta.SYSTEM_ID,
            subject=Subject("user", request.user.username),
            action=Action(IAMMeta.PROJECT_VIEW_ACTION),
            resources=res_factory.resources_for_project(project_id),
        )
        queryset = Label.objects.filter(Q(project_id=project_id) | Q(is_default=True))
        serializer = self.get_serializer(queryset, many=True)
        return Response(serializer.data)

    @action(methods=["get"], detail=False)
    def get_templates_labels(self, request):
        return self._fetch_label_or_template_ids(request, fetch_label=True)

    @action(methods=["get"], detail=False)
    def get_label_template_ids(self, request):
        return self._fetch_label_or_template_ids(request, fetch_label=False)

    @staticmethod
    def _fetch_label_or_template_ids(request, fetch_label):
        base_id_name = "template_ids" if fetch_label else "label_ids"
        if fetch_label:
            fetch_func = TemplateLabelRelation.objects.fetch_templates_labels
        else:
            fetch_func = TemplateLabelRelation.objects.fetch_label_template_ids
        base_ids = request.query_params.get(base_id_name)
        if not base_ids:
            raise ValidationException("{} must be provided.".format(base_id_name))
        project_id = request.query_params.get("project_id")
        allow_or_raise_immediate_response(
            iam=iam,
            system=IAMMeta.SYSTEM_ID,
            subject=Subject("user", request.user.username),
            action=Action(IAMMeta.PROJECT_VIEW_ACTION),
            resources=res_factory.resources_for_project(project_id),
        )
        base_ids = [int(base_id) for base_id in base_ids.strip().split(",")]
        return Response(fetch_func(base_ids))


"""
Name: Timothy James Duffy, Kevin Falconett
File: metrics.py
Class: CSc 483; Spring 2023
Project: TextSummarizer

Provides methods to calculate the ROUGE metric and print the results.
"""

# Filter tensorflow warnings.
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
from database import *
from rouge import Rouge
from config import DATABASE_NAME
from summarizer import generate_summary

def get_rouge_scores(num_docs, offset):
    """Gets the rogue scores using documents in the database. Documents can be offset by id."""
    # Get articles and summaries from the database.
    database = Database(DATABASE_NAME)
    articles, summaries = zip(*database.get_data(num_docs, offset))

    # Holds the generated summaries.
    generated_summaries = []

    # Generate summaries for the articles.
    for i, article in enumerate(articles):
        summary = generate_summary(article)
        generated_summaries.append(summary)
        print('Actual summary:\n{}'.format(summaries[i]))
        print('Generated summary:\n{}\n'.format(generated_summaries[i]))

    # Calculate ROUGE scores for the sample set.
    rouge = Rouge()
    scores = rouge.get_scores(generated_summaries, list(summaries), avg=True)
    return scores

def print_scores(scores):
    """Prints the given ROUGE scores in a nice format."""
    # Get ROUGE dictionaries. Each contains recall, precision, accuracy scores.
    r1 = scores['rouge-1']
    r2 = scores['rouge-2']
    rl = scores['rouge-l']

    # Print out the scores for Rouge-1, Rouge-2, and Rouge-l.
    print('Rouge-1\trecall:\t{:.2f}\tprecision:\t{:.2f}\tf1_score:\t{:.2f}'.format(r1['r'], r1['p'], r1['f']))
    print('Rouge-2\trecall:\t{:.2f}\tprecision:\t{:.2f}\tf1_score:\t{:.2f}'.format(r2['r'], r2['p'], r2['f']))
    print('Rouge-l\trecall:\t{:.2f}\tprecision:\t{:.2f}\tf1_score:\t{:.2f}\n'.format(rl['r'], rl['p'], rl['f']))

def main():
    # Prints the ROUGE results for data the model has been trained on.
    print('Trained Data ROUGE Scores:')
    trained_data = get_rouge_scores(1, 0)
    print_scores(trained_data)

    # Prints the ROUGE results for data the model has NOT been trained on.
    print('Untrained Data ROUGE Scores:')
    untrained_data = get_rouge_scores(1, 1200)
    print_scores(untrained_data)

if __name__ == '__main__':
    main()


__authors__ = ["T. Vincent"]
__license__ = "MIT"
__date__ = "28/06/2018"

import logging
import numpy
import weakref
import functools
from typing import Optional

from ....utils.weakref import WeakList
from ... import qt
from .. import items
from ..items import core
from ...colors import rgba

logger = logging.getLogger(__name__)

class _RegionOfInterestBase(qt.QObject):
    """Base class of 1D and 2D region of interest

    :param QObject parent: See QObject
    :param str name: The name of the ROI
    """

    sigAboutToBeRemoved = qt.Signal()
    """Signal emitted just before this ROI is removed from its manager."""

    sigItemChanged = qt.Signal(object)
    """Signal emitted when item has changed.

    It provides a flag describing which property of the item has changed.
    See :class:`ItemChangedType` for flags description.
    """

    def __init__(self, parent=None):
        qt.QObject.__init__(self)
        if parent is not None:
            self.setParent(parent)
        self.__name = ''

    def getName(self):
        """Returns the name of the ROI

        :return: name of the region of interest
        :rtype: str
        """
        return self.__name

    def setName(self, name):
        """Set the name of the ROI

        :param str name: name of the region of interest
        """
        name = str(name)
        if self.__name != name:
            self.__name = name
            self._updated(items.ItemChangedType.NAME)

    def _updated(self, event=None, checkVisibility=True):
        """Implement Item mix-in update method by updating the plot items

        See :class:`~silx.gui.plot.items.Item._updated`
        """
        self.sigItemChanged.emit(event)

    def contains(self, position):
        """Returns True if the `position` is in this ROI.

        :param tuple[float,float] position: position to check
        :return: True if the value / point is consider to be in the region of
                 interest.
        :rtype: bool
        """
        return False  # Override in subclass to perform actual test

class RoiInteractionMode(object):
    """Description of an interaction mode.

    An interaction mode provide a specific kind of interaction for a ROI.
    A ROI can implement many interaction.
    """

    def __init__(self, label, description=None):
        self._label = label
        self._description = description

    @property
    def label(self):
        """Short name"""
        return self._label

    @property
    def description(self):
        """Longer description of the interaction mode"""
        return self._description

class InteractionModeMixIn(object):
    """Mix in feature which can be implemented by a ROI object.

    This provides user interaction to switch between different
    interaction mode to edit the ROI.

    This ROI modes have to be described using `RoiInteractionMode`,
    and taken into account during interation with handles.
    """

    sigInteractionModeChanged = qt.Signal(object)

    def __init__(self):
        self.__modeId = None

    def _initInteractionMode(self, modeId):
        """Set the mode without updating anything.

        Must be one of the returned :meth:`availableInteractionModes`.

        :param RoiInteractionMode modeId: Mode to use
        """
        self.__modeId = modeId

    def availableInteractionModes(self):
        """Returns the list of available interaction modes

        Must be implemented when inherited to provide all available modes.

        :rtype: List[RoiInteractionMode]
        """
        raise NotImplementedError()

    def setInteractionMode(self, modeId):
        """Set the interaction mode.

        :param RoiInteractionMode modeId: Mode to use
        """
        self.__modeId = modeId
        self._interactiveModeUpdated(modeId)
        self.sigInteractionModeChanged.emit(modeId)

    def _interactiveModeUpdated(self, modeId):
        """Called directly after an update of the mode.

        The signal `sigInteractionModeChanged` is triggered after this
        call.

        Must be implemented when inherited to take care of the change.
        """
        raise NotImplementedError()

    def getInteractionMode(self):
        """Returns the interaction mode.

        Must be one of the returned :meth:`availableInteractionModes`.

        :rtype: RoiInteractionMode
        """
        return self.__modeId

    def createMenuForInteractionMode(self, parent: qt.QWidget) -> qt.QMenu:
        """Create a menu providing access to the different interaction modes"""
        availableModes = self.availableInteractionModes()
        currentMode = self.getInteractionMode()
        submenu = qt.QMenu(parent)
        modeGroup = qt.QActionGroup(parent)
        modeGroup.setExclusive(True)
        for mode in availableModes:
            action = qt.QAction(parent)
            action.setText(mode.label)
            action.setToolTip(mode.description)
            action.setCheckable(True)
            if mode is currentMode:
                action.setChecked(True)
            else:
                callback = functools.partial(self.setInteractionMode, mode)
                action.triggered.connect(callback)
            modeGroup.addAction(action)
            submenu.addAction(action)
        submenu.setTitle("Interaction mode")
        return submenu

class RegionOfInterest(_RegionOfInterestBase, core.HighlightedMixIn):
    """Object describing a region of interest in a plot.

    :param QObject parent:
        The RegionOfInterestManager that created this object
    """

    _DEFAULT_LINEWIDTH = 1.
    """Default line width of the curve"""

    _DEFAULT_LINESTYLE = '-'
    """Default line style of the curve"""

    _DEFAULT_HIGHLIGHT_STYLE = items.CurveStyle(linewidth=2)
    """Default highlight style of the item"""

    ICON, NAME, SHORT_NAME = None, None, None
    """Metadata to describe the ROI in labels, tooltips and widgets

    Should be set by inherited classes to custom the ROI manager widget.
    """

    sigRegionChanged = qt.Signal()
    """Signal emitted everytime the shape or position of the ROI changes"""

    sigEditingStarted = qt.Signal()
    """Signal emitted when the user start editing the roi"""

    sigEditingFinished = qt.Signal()
    """Signal emitted when the region edition is finished. During edition
    sigEditionChanged will be emitted several times and
    sigRegionEditionFinished only at end"""

    def __init__(self, parent=None):
        # Avoid circular dependency
        from ..tools import roi as roi_tools
        assert parent is None or isinstance(parent, roi_tools.RegionOfInterestManager)
        _RegionOfInterestBase.__init__(self, parent)
        core.HighlightedMixIn.__init__(self)
        self.__text = None
        self._color = rgba('red')
        self._editable = False
        self._selectable = False
        self._focusProxy = None
        self._visible = True
        self._child = WeakList()

    def _connectToPlot(self, plot):
        """Called after connection to a plot"""
        for item in self.getItems():
            # This hack is needed to avoid reentrant call from _disconnectFromPlot
            # to the ROI manager. It also speed up the item tests in _itemRemoved
            item._roiGroup = True
            plot.addItem(item)

    def _disconnectFromPlot(self, plot):
        """Called before disconnection from a plot"""
        for item in self.getItems():
            # The item could be already be removed by the plot
            if item.getPlot() is not None:
                del item._roiGroup
                plot.removeItem(item)

    def _setItemName(self, item):
        """Helper to generate a unique id to a plot item"""
        legend = "__ROI-%d__%d" % (id(self), id(item))
        item.setName(legend)

    def setParent(self, parent):
        """Set the parent of the RegionOfInterest

        :param Union[None,RegionOfInterestManager] parent: The new parent
        """
        # Avoid circular dependency
        from ..tools import roi as roi_tools
        if (parent is not None and not isinstance(parent, roi_tools.RegionOfInterestManager)):
            raise ValueError('Unsupported parent')

        previousParent = self.parent()
        if previousParent is not None:
            previousPlot = previousParent.parent()
            if previousPlot is not None:
                self._disconnectFromPlot(previousPlot)
        super(RegionOfInterest, self).setParent(parent)
        if parent is not None:
            plot = parent.parent()
            if plot is not None:
                self._connectToPlot(plot)

    def addItem(self, item):
        """Add an item to the set of this ROI children.

        This item will be added and removed to the plot used by the ROI.

        If the ROI is already part of a plot, the item will also be added to
        the plot.

        It the item do not have a name already, a unique one is generated to
        avoid item collision in the plot.

        :param silx.gui.plot.items.Item item: A plot item
        """
        assert item is not None
        self._child.append(item)
        if item.getName() == '':
            self._setItemName(item)
        manager = self.parent()
        if manager is not None:
            plot = manager.parent()
            if plot is not None:
                item._roiGroup = True
                plot.addItem(item)

    def removeItem(self, item):
        """Remove an item from this ROI children.

        If the item is part of a plot it will be removed too.

        :param silx.gui.plot.items.Item item: A plot item
        """
        assert item is not None
        self._child.remove(item)
        plot = item.getPlot()
        if plot is not None:
            del item._roiGroup
            plot.removeItem(item)

    def getItems(self):
        """Returns the list of PlotWidget items of this RegionOfInterest.

        :rtype: List[~silx.gui.plot.items.Item]
        """
        return tuple(self._child)

    @classmethod
    def _getShortName(cls):
        """Return an human readable kind of ROI

        :rtype: str
        """
        if hasattr(cls, "SHORT_NAME"):
            name = cls.SHORT_NAME
        if name is None:
            name = cls.__name__
        return name

    def getColor(self):
        """Returns the color of this ROI

        :rtype: QColor
        """
        return qt.QColor.fromRgbF(*self._color)

    def setColor(self, color):
        """Set the color used for this ROI.

        :param color: The color to use for ROI shape as
           either a color name, a QColor, a list of uint8 or float in [0, 1].
        """
        color = rgba(color)
        if color != self._color:
            self._color = color
            self._updated(items.ItemChangedType.COLOR)

    def isEditable(self):
        """Returns whether the ROI is editable by the user or not.

        :rtype: bool
        """
        return self._editable

    def setEditable(self, editable):
        """Set whether the ROI can be changed interactively.

        :param bool editable: True to allow edition by the user,
           False to disable.
        """
        editable = bool(editable)
        if self._editable != editable:
            self._editable = editable
            self._updated(items.ItemChangedType.EDITABLE)

    def isSelectable(self):
        """Returns whether the ROI is selectable by the user or not.

        :rtype: bool
        """
        return self._selectable

    def setSelectable(self, selectable):
        """Set whether the ROI can be selected interactively.

        :param bool selectable: True to allow selection by the user,
           False to disable.
        """
        selectable = bool(selectable)
        if self._selectable != selectable:
            self._selectable = selectable
            self._updated(items.ItemChangedType.SELECTABLE)

    def getFocusProxy(self):
        """Returns the ROI which have to be selected when this ROI is selected,
        else None if no proxy specified.

        :rtype: RegionOfInterest
        """
        proxy = self._focusProxy
        if proxy is None:
            return None
        proxy = proxy()
        if proxy is None:
            self._focusProxy = None
        return proxy

    def setFocusProxy(self, roi):
        """Set the real ROI which will be selected when this ROI is selected,
        else None to remove the proxy already specified.

        :param RegionOfInterest roi: A ROI
        """
        if roi is not None:
            self._focusProxy = weakref.ref(roi)
        else:
            self._focusProxy = None

    def isVisible(self):
        """Returns whether the ROI is visible in the plot.

        .. note::
            This does not take into account whether or not the plot
            widget itself is visible (unlike :meth:`QWidget.isVisible` which
            checks the visibility of all its parent widgets up to the window)

        :rtype: bool
        """
        return self._visible

    def setVisible(self, visible):
        """Set whether the plot items associated with this ROI are
        visible in the plot.

        :param bool visible: True to show the ROI in the plot, False to
            hide it.
        """
        visible = bool(visible)
        if self._visible != visible:
            self._visible = visible
            self._updated(items.ItemChangedType.VISIBLE)

    def getText(self) -> str:
        """Returns the currently displayed text for this ROI"""
        return self.getName() if self.__text is None else self.__text

    def setText(self, text: Optional[str] = None) -> None:
        """Set the displayed text for this ROI.

        If None (the default), the ROI name is used.
        """
        if self.__text != text:
            self.__text = text
            self._updated(items.ItemChangedType.TEXT)

    def _updateText(self, text: str) -> None:
        """Update the text displayed by this ROI

        Override in subclass to custom text display
        """
        pass

    @classmethod
    def showFirstInteractionShape(cls):
        """Returns True if the shape created by the first interaction and
        managed by the plot have to be visible.

        :rtype: bool
        """
        return False

    @classmethod
    def getFirstInteractionShape(cls):
        """Returns the shape kind which will be used by the very first
        interaction with the plot.

        This interactions are hardcoded inside the plot

        :rtype: str
        """
        return cls._plotShape

    def setFirstShapePoints(self, points):
        """Initialize the ROI using the points from the first interaction.

        This interaction is constrained by the plot API and only supports few
        shapes.
        """
        raise NotImplementedError()

    def creationStarted(self):
        """Called when the ROI creation interaction was started.
        """
        pass

    def creationFinalized(self):
        """Called when the ROI creation interaction was finalized.
        """
        pass

    def _updateItemProperty(self, event, source, destination):
        """Update the item property of a destination from an item source.

        :param items.ItemChangedType event: Property type to update
        :param silx.gui.plot.items.Item source: The reference for the data
        :param event Union[Item,List[Item]] destination: The item(s) to update
        """
        if not isinstance(destination, (list, tuple)):
            destination = [destination]
        if event == items.ItemChangedType.NAME:
            value = source.getName()
            for d in destination:
                d.setName(value)
        elif event == items.ItemChangedType.EDITABLE:
            value = source.isEditable()
            for d in destination:
                d.setEditable(value)
        elif event == items.ItemChangedType.SELECTABLE:
            value = source.isSelectable()
            for d in destination:
                d._setSelectable(value)
        elif event == items.ItemChangedType.COLOR:
            value = rgba(source.getColor())
            for d in destination:
                d.setColor(value)
        elif event == items.ItemChangedType.LINE_STYLE:
            value = self.getLineStyle()
            for d in destination:
                d.setLineStyle(value)
        elif event == items.ItemChangedType.LINE_WIDTH:
            value = self.getLineWidth()
            for d in destination:
                d.setLineWidth(value)
        elif event == items.ItemChangedType.SYMBOL:
            value = self.getSymbol()
            for d in destination:
                d.setSymbol(value)
        elif event == items.ItemChangedType.SYMBOL_SIZE:
            value = self.getSymbolSize()
            for d in destination:
                d.setSymbolSize(value)
        elif event == items.ItemChangedType.VISIBLE:
            value = self.isVisible()
            for d in destination:
                d.setVisible(value)
        else:
            assert False

    def _updated(self, event=None, checkVisibility=True):
        if event == items.ItemChangedType.TEXT:
            self._updateText(self.getText())
        elif event == items.ItemChangedType.HIGHLIGHTED:
            style = self.getCurrentStyle()
            self._updatedStyle(event, style)
        else:
            styleEvents = [items.ItemChangedType.COLOR,
                           items.ItemChangedType.LINE_STYLE,
                           items.ItemChangedType.LINE_WIDTH,
                           items.ItemChangedType.SYMBOL,
                           items.ItemChangedType.SYMBOL_SIZE]
            if self.isHighlighted():
                styleEvents.append(items.ItemChangedType.HIGHLIGHTED_STYLE)

            if event in styleEvents:
                style = self.getCurrentStyle()
                self._updatedStyle(event, style)

        super(RegionOfInterest, self)._updated(event, checkVisibility)

        # Displayed text has changed, send a text event
        if event == items.ItemChangedType.NAME and self.__text is None:
            self._updated(items.ItemChangedType.TEXT, checkVisibility)

    def _updatedStyle(self, event, style):
        """Called when the current displayed style of the ROI was changed.

        :param event: The event responsible of the change of the style
        :param items.CurveStyle style: The current style
        """
        pass

    def getCurrentStyle(self):
        """Returns the current curve style.

        Curve style depends on curve highlighting

        :rtype: CurveStyle
        """
        baseColor = rgba(self.getColor())
        if isinstance(self, core.LineMixIn):
            baseLinestyle = self.getLineStyle()
            baseLinewidth = self.getLineWidth()
        else:
            baseLinestyle = self._DEFAULT_LINESTYLE
            baseLinewidth = self._DEFAULT_LINEWIDTH
        if isinstance(self, core.SymbolMixIn):
            baseSymbol = self.getSymbol()
            baseSymbolsize = self.getSymbolSize()
        else:
            baseSymbol = 'o'
            baseSymbolsize = 1

        if self.isHighlighted():
            style = self.getHighlightedStyle()
            color = style.getColor()
            linestyle = style.getLineStyle()
            linewidth = style.getLineWidth()
            symbol = style.getSymbol()
            symbolsize = style.getSymbolSize()

            return items.CurveStyle(
                color=baseColor if color is None else color,
                linestyle=baseLinestyle if linestyle is None else linestyle,
                linewidth=baseLinewidth if linewidth is None else linewidth,
                symbol=baseSymbol if symbol is None else symbol,
                symbolsize=baseSymbolsize if symbolsize is None else symbolsize)
        else:
            return items.CurveStyle(color=baseColor,
                                    linestyle=baseLinestyle,
                                    linewidth=baseLinewidth,
                                    symbol=baseSymbol,
                                    symbolsize=baseSymbolsize)

    def _editingStarted(self):
        assert self._editable is True
        self.sigEditingStarted.emit()

    def _editingFinished(self):
        self.sigEditingFinished.emit()

    def populateContextMenu(self, menu: qt.QMenu):
        """Populate a menu used as a context menu"""
        pass

class HandleBasedROI(RegionOfInterest):
    """Manage a ROI based on a set of handles"""

    def __init__(self, parent=None):
        RegionOfInterest.__init__(self, parent=parent)
        self._handles = []
        self._posOrigin = None
        self._posPrevious = None

    def addUserHandle(self, item=None):
        """
        Add a new free handle to the ROI.

        This handle do nothing. It have to be managed by the ROI
        implementing this class.

        :param Union[None,silx.gui.plot.items.Marker] item: The new marker to
            add, else None to create a default marker.
        :rtype: silx.gui.plot.items.Marker
        """
        return self.addHandle(item, role="user")

    def addLabelHandle(self, item=None):
        """
        Add a new label handle to the ROI.

        This handle is not draggable nor selectable.

        It is displayed without symbol, but it is always visible anyway
        the ROI is editable, in order to display text.

        :param Union[None,silx.gui.plot.items.Marker] item: The new marker to
            add, else None to create a default marker.
        :rtype: silx.gui.plot.items.Marker
        """
        return self.addHandle(item, role="label")

    def addTranslateHandle(self, item=None):
        """
        Add a new translate handle to the ROI.

        Dragging translate handles affect the position position of the ROI
        but not the shape itself.

        :param Union[None,silx.gui.plot.items.Marker] item: The new marker to
            add, else None to create a default marker.
        :rtype: silx.gui.plot.items.Marker
        """
        return self.addHandle(item, role="translate")

    def addHandle(self, item=None, role="default"):
        """
        Add a new handle to the ROI.

        Dragging handles while affect the position or the shape of the
        ROI.

        :param Union[None,silx.gui.plot.items.Marker] item: The new marker to
            add, else None to create a default marker.
        :rtype: silx.gui.plot.items.Marker
        """
        if item is None:
            item = items.Marker()
            color = rgba(self.getColor())
            color = self._computeHandleColor(color)
            item.setColor(color)
            if role == "default":
                item.setSymbol("s")
            elif role == "user":
                pass
            elif role == "translate":
                item.setSymbol("+")
            elif role == "label":
                item.setSymbol("")

        if role == "user":
            pass
        elif role == "label":
            item._setSelectable(False)
            item._setDraggable(False)
            item.setVisible(True)
        else:
            self.__updateEditable(item, self.isEditable(), remove=False)
            item._setSelectable(False)

        self._handles.append((item, role))
        self.addItem(item)
        return item

    def removeHandle(self, handle):
        data = [d for d in self._handles if d[0] is handle][0]
        self._handles.remove(data)
        role = data[1]
        if role not in ["user", "label"]:
            if self.isEditable():
                self.__updateEditable(handle, False)
        self.removeItem(handle)

    def getHandles(self):
        """Returns the list of handles of this HandleBasedROI.

        :rtype: List[~silx.gui.plot.items.Marker]
        """
        return tuple(data[0] for data in self._handles)

    def _updated(self, event=None, checkVisibility=True):
        """Implement Item mix-in update method by updating the plot items

        See :class:`~silx.gui.plot.items.Item._updated`
        """
        if event == items.ItemChangedType.VISIBLE:
            for item, role in self._handles:
                visible = self.isVisible()
                editionVisible = visible and self.isEditable()
                if role not in ["user", "label"]:
                    item.setVisible(editionVisible)
                else:
                    item.setVisible(visible)
        elif event == items.ItemChangedType.EDITABLE:
            for item, role in self._handles:
                editable = self.isEditable()
                if role not in ["user", "label"]:
                    self.__updateEditable(item, editable)
        super(HandleBasedROI, self)._updated(event, checkVisibility)

    def _updatedStyle(self, event, style):
        super(HandleBasedROI, self)._updatedStyle(event, style)

        # Update color of shape items in the plot
        color = rgba(self.getColor())
        handleColor = self._computeHandleColor(color)
        for item, role in self._handles:
            if role == 'user':
                pass
            elif role == 'label':
                item.setColor(color)
            else:
                item.setColor(handleColor)

    def __updateEditable(self, handle, editable, remove=True):
        # NOTE: visibility change emit a position update event
        handle.setVisible(editable and self.isVisible())
        handle._setDraggable(editable)
        if editable:
            handle.sigDragStarted.connect(self._handleEditingStarted)
            handle.sigItemChanged.connect(self._handleEditingUpdated)
            handle.sigDragFinished.connect(self._handleEditingFinished)
        else:
            if remove:
                handle.sigDragStarted.disconnect(self._handleEditingStarted)
                handle.sigItemChanged.disconnect(self._handleEditingUpdated)
                handle.sigDragFinished.disconnect(self._handleEditingFinished)

    def _handleEditingStarted(self):
        super(HandleBasedROI, self)._editingStarted()
        handle = self.sender()
        self._posOrigin = numpy.array(handle.getPosition())
        self._posPrevious = numpy.array(self._posOrigin)
        self.handleDragStarted(handle, self._posOrigin)

    def _handleEditingUpdated(self):
        if self._posOrigin is None:
            # Avoid to handle events when visibility change
            return
        handle = self.sender()
        current = numpy.array(handle.getPosition())
        self.handleDragUpdated(handle, self._posOrigin, self._posPrevious, current)
        self._posPrevious = current

    def _handleEditingFinished(self):
        handle = self.sender()
        current = numpy.array(handle.getPosition())
        self.handleDragFinished(handle, self._posOrigin, current)
        self._posPrevious = None
        self._posOrigin = None
        super(HandleBasedROI, self)._editingFinished()

    def isHandleBeingDragged(self):
        """Returns True if one of the handles is currently being dragged.

        :rtype: bool
        """
        return self._posOrigin is not None

    def handleDragStarted(self, handle, origin):
        """Called when an handler drag started"""
        pass

    def handleDragUpdated(self, handle, origin, previous, current):
        """Called when an handle drag position changed"""
        pass

    def handleDragFinished(self, handle, origin, current):
        """Called when an handle drag finished"""
        pass

    def _computeHandleColor(self, color):
        """Returns the anchor color from the base ROI color

        :param Union[numpy.array,Tuple,List]: color
        :rtype: Union[numpy.array,Tuple,List]
        """
        return color[:3] + (0.5,)


import concurrent.futures
import logging
import time

def thread_function(name):
    logging.info("Thread %s: starting", name)
    time.sleep(2)
    logging.info("Thread %s: finishing", name)

class FakeDatabase:
    def __init__(self):
        self.value = 0

    def update(self, name):
        logging.info("Thread %s: starting update", name)
        local_copy = self.value
        local_copy += 1
        time.sleep(0.1)
        self.value = local_copy
        logging.info("Thread %s: finishing update", name)

if __name__ == "__main__":
    format = "%(asctime)s: %(message)s"
    logging.basicConfig(format=format, level=logging.INFO,
                        datefmt="%H:%M:%S")

    # threads = list()
    # for index in range(3):
    #     logging.info("Main    : create and start thread %d.", index)
    #     x = threading.Thread(target=thread_function, args=(index,))
    #     threads.append(x)
    #     x.start()
    #
    # for index, thread in enumerate(threads):
    #     logging.info("Main    : before joining thread %d.", index)
    #     thread.join()
    #     logging.info("Main    : thread %d done", index)
    #
    # with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor: # replace Max_Workers with number of threads and execute it using .map
    #     executor.map(thread_function, range(3))

    format = "%(asctime)s: %(message)s"
    logging.basicConfig(format=format, level=logging.INFO,
                        datefmt="%H:%M:%S")

    database = FakeDatabase()
    logging.info("Testing update. Starting value is %d.", database.value)
    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
        for index in range(2):
            executor.submit(database.update, index)
    logging.info("Testing update. Ending value is %d.", database.value)


import random
import re
import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
from keras.preprocessing.text import Tokenizer
import torch

def transform_to_index_tensor(pairs,rus_w2i,en_w2i,device):
    rus_tensor = []
    en_tensor = []

    for word in range(len(pairs[0])):
        en_tensor.append(en_w2i[pairs[0][word]])
    for word in range(len(pairs[1])):
        rus_tensor.append(rus_w2i[pairs[1][word]])

    return en_tensor,rus_tensor

def split_dataset(pairs,val_size=20.000,test_size=20.000):
    pairs = random.sample(pairs,len(pairs))
    pairs_test = pairs[:test_size]
    pairs_val = pairs[test_size:test_size+val_size]
    pairs_train = pairs[test_size+val_size:len(pairs)]

    return pairs_train,pairs_val,pairs_test

def custom_index_tokenizer(phrase,w2i,i2w):
    for word in phrase:
        if word in w2i:
            continue
        else:
            if bool(w2i) == False:
                w2i[word] = 0
                i2w[0] = word
            else:
                new_idx = list(i2w)[-1]+1
                w2i[word] = new_idx
                i2w[new_idx] = word
    return w2i,i2w

def filter_double_spaces(word_list):
    updated_word_list = []
    for w in range(len(word_list)):
        if not word_list[w] == '':
            updated_word_list.append(word_list[w])
    return updated_word_list

def get_data(filename='fra.txt',max_words=-1,plot_res=False):
    russian_word_list = []
    english_word_list = []

    #dictionaries for converting a word to a unique integer and the opposite
    russian_word_to_idx = {'<SOS>':0,'<EOS>':1,'<PAD>':2}
    english_word_to_idx = {'<SOS>':0,'<EOS>':1,'<PAD>':2}
    russian_idx_to_word = {0:'<SOS>',1:'<EOS>',2:'<PAD>'}
    english_idx_to_word = {0:'<SOS>',1:'<EOS>',2:'<PAD>'}

    pairs = []

    #read the dataset from the file
    with open(filename, "r", encoding="utf-8") as f:
        lines_list = f.read().split("\n")
        print("The file total translated words/phrases are: "+str(len(lines_list)))

    #get the phrases for each language to a different list
    word_counter = 0
    for i in range(len(lines_list)):
        if not max_words == -1:
            word_counter += 1
            if word_counter > max_words:
                break
        try:
            lines_list[i].split('\t')[1]
        except:
            continue
        russian_word_list.append(lines_list[i].split('\t')[1])
        english_word_list.append(lines_list[i].split('\t')[0])
    print("The total english phrases are: " + str(len(english_word_list)))
    print("The total russian phrases are: "+str(len(russian_word_list)))

    russian_lengths = []
    english_lengths = []
    russian_words_final = []
    english_words_final = []
    for phrase in range(len(russian_word_list)):
        #remove punc
        russian_words = re.sub(r'[^\w\s]', '', russian_word_list[phrase])
        english_words = re.sub(r'[^\w\s]', '', english_word_list[phrase])

        #to lower case
        russian_words = russian_words.lower()
        english_words = english_words.lower()

        russian_lengths.append(len(russian_words))
        english_lengths.append(len(english_words))

        #split to space
        russian_words = russian_words.split(' ')
        english_words = english_words.split(' ')

        #filter double spaces
        russian_words = filter_double_spaces(russian_words)
        english_words = filter_double_spaces(english_words)

        #add SOS and EOS tokens
        russian_words.insert(0, "<SOS>")
        russian_words.append("<EOS>")

        english_words.insert(0, "<SOS>")
        english_words.append("<EOS>")
        pairs.append([english_words,russian_words])

        russian_word_to_idx,russian_idx_to_word = custom_index_tokenizer(russian_words,russian_word_to_idx,russian_idx_to_word)
        english_word_to_idx, english_idx_to_word = custom_index_tokenizer(english_words, english_word_to_idx,english_idx_to_word)

        russian_words_final.append(russian_words)
        english_words_final.append(english_words)

    if plot_res:
        plt.hist(english_lengths, 15, alpha=0.5, label='English Lengths',edgecolor = "black")
        plt.hist(russian_lengths, 30, alpha=0.5, label='Russian Lengths',edgecolor = "black")
        plt.legend(loc='upper right')
        plt.show()

    print("Found "+str(len(list(russian_idx_to_word.keys())))+" unique russian words.")
    print("Found " + str(len(list(english_idx_to_word.keys()))) + " unique english words.")

    return russian_words_final,english_words_final,russian_word_to_idx,russian_idx_to_word,english_word_to_idx,english_idx_to_word,pairs,russian_words_final,english_words_final


import requests
from prettytable import PrettyTable

#write this into terminal if using linux or cmd if using windows
#pip3 install Prettytable requests

city = input('Enter City => ')#City name here
api = "your api here"#api of https://openweathermap.org get urs from the website
response = requests.get(f"http://api.openweathermap.org/data/2.5/weather?q={city}&appid={api}")#request a response

#if the response equal 200 = OK thats mean everything is okey

if response:
    weather = response.json()["weather"]#get weather info
    temp = response.json()["main"]#get temperature info
    wind = response.json()["wind"]#get wind info
    clouds = response.json()["clouds"]#get clouds info
    coord = response.json()["coord"]#get coordinates

    weather_cloud = weather[0]["description"]

    temp_feels = temp["temp"]
    temp_feels_like = temp["feels_like"]
    temp_feels_min = temp["temp_min"]
    temp_feels_max = temp["temp_max"]

    wind_speed = wind["speed"]

    clouds_deg = clouds["all"]

    coordination_lon = coord["lon"]
    coordination_lat= coord["lat"]
    #write everything into a table
    table = PrettyTable()

    table.field_names = ["City",
                        "weather Description",
                        "Temperature",
                        "Feels Like",
                        "Temperature Min",
                        "Temperature Max",
                        "Win Speed",
                        "Cloud",
                        "Coordinates Long",
                        "Coordinates Lat"]
    table.add_row([city.capitalize(),
                weather_cloud,
                temp_feels,
                temp_feels_like,
                temp_feels_min,
                temp_feels_max,
                wind_speed,
                clouds_deg,
                coordination_lon,
                coordination_lat])

    print(table)
else:
    print("Invalid Input")


import pandas as pd

from morpheus import SequentialComposition, ParallelComposition
from morpheus.algo.selection import base_selection_algorithm, random_selection_algorithm
from morpheus.utils.encoding import *
from morpheus.utils import debug_print

from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor

VERBOSITY = 0

def default_dataset(n_features=7, random_state=997):
    """
    Generate a dataset to be used in tests.

    Returns:

    """
    X, y = make_classification(
        n_samples=10 ** 3,
        n_features=n_features,
        n_informative=n_features,
        n_repeated=0,
        n_redundant=0,
        n_clusters_per_class=2,
        random_state=random_state,
    )

    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.2, random_state=random_state
    )

    train = pd.DataFrame(X_train)
    train = train.assign(y=y_train)

    test = pd.DataFrame(X_test)
    test = test.assign(y=y_test)

    return train, test

def default_chain(random_state=997):
    """
    Default classifier chain.

    For use in further tests.

    Returns:

    """
    train, _ = default_dataset(random_state=random_state)

    m_list = default_m_list_for_chain(train.values)

    sc = SequentialComposition()
    for m in m_list:
        sc.add_estimator(m, location="back")

    return sc

def default_ensemble(random_state=997):
    """
    Default classifier ensmeble.

    For use in further tests.

    Returns:

    """
    train, _ = default_dataset(random_state=random_state)

    m_list = default_m_list_for_ensemble(train.values)

    pc = ParallelComposition()
    for m in m_list:
        pc.add_estimator(m)

    return pc

def default_m_list_for_chain(data):
    targ_ids_1 = [4, 5]
    desc_ids_1 = [0, 1, 2]

    targ_ids_2 = [7]
    desc_ids_2 = [1, 2, 5]

    all_desc_ids = [desc_ids_1, desc_ids_2]
    all_targ_ids = [targ_ids_1, targ_ids_2]

    m_list = []
    ids = zip(all_desc_ids, all_targ_ids)

    for desc_ids, targ_ids in ids:
        msg = """
        Learning model with desc ids:    {}
                            targ ids:    {}
        """.format(
            desc_ids, targ_ids
        )
        print(msg)

        if set(targ_ids).issubset({6, 7}):
            learner = RandomForestClassifier
        elif set(targ_ids).issubset({0, 1, 2, 3, 4, 5}):
            learner = RandomForestRegressor
        else:
            msg = """
            Cannot learn mixed (nominal/numeric) models
            """
            raise ValueError(msg)

        # Learn a model for desc_ids-targ_ids
        m = learn_model(data, desc_ids, targ_ids, learner, max_depth=5, n_estimators=5)
        m_list.append(m)

    return m_list

def default_m_list_for_ensemble(data):
    targ_ids_1 = [5]
    desc_ids_1 = [0, 1, 2]

    targ_ids_2 = [4, 5]
    desc_ids_2 = [0, 1, 3]

    all_desc_ids = [desc_ids_1, desc_ids_2]
    all_targ_ids = [targ_ids_1, targ_ids_2]

    m_list = []
    ids = zip(all_desc_ids, all_targ_ids)

    for desc_ids, targ_ids in ids:
        msg = """
        Learning model with desc ids:    {}
                            targ ids:    {}
        """.format(
            desc_ids, targ_ids
        )
        print(msg)

        if set(targ_ids).issubset({6, 7}):
            learner = RandomForestClassifier
        elif set(targ_ids).issubset({0, 1, 2, 3, 4, 5}):
            learner = RandomForestRegressor
        else:
            msg = """
            Cannot learn mixed (nominal/numeric) models
            """
            raise ValueError(msg)

        # Learn a model for desc_ids-targ_ids
        m = learn_model(data, desc_ids, targ_ids, learner, max_depth=5, n_estimators=5)
        m_list.append(m)

    return m_list

def default_m_list_for_mercs(data):
    n, m = data.shape
    attributes = list(range(m))

    metadata = {"nb_atts": m}
    settings = {"param": 1, "its": 1}

    m_codes = base_selection_algorithm(metadata, settings)

    all_desc_ids, all_targ_ids = [], []
    for m_code in m_codes:
        desc_ids, targ_ids, _ = code_to_query(m_code)
        all_desc_ids.append(desc_ids)
        all_targ_ids.append(targ_ids)

    m_list = []
    ids = zip(all_desc_ids, all_targ_ids)

    for desc_ids, targ_ids in ids:
        msg = """
        Learning model with desc ids:    {}
                            targ ids:    {}
        """.format(
            desc_ids, targ_ids
        )
        print(msg)

        if set(targ_ids).issubset(attributes[-1:]):
            learner = RandomForestClassifier
        elif set(targ_ids).issubset(attributes[:-1]):
            learner = RandomForestRegressor
        else:
            msg = """
            Cannot learn mixed (nominal/numeric) models
            """
            raise ValueError(msg)

        # Learn a model for desc_ids-targ_ids
        m = learn_model(data, desc_ids, targ_ids, learner, max_depth=5, n_estimators=5)
        m_list.append(m)

    return m_list

def random_m_list_for_mercs(data, its=1, fraction=0.3, random_state=997):
    n, m = data.shape
    attributes = list(range(m))

    metadata = {"nb_atts": m}
    settings = {"param": 1, "its": its, "fraction": fraction}

    m_codes = random_selection_algorithm(metadata, settings, random_state=random_state)

    all_desc_ids, all_targ_ids = [], []
    for m_code in m_codes:
        desc_ids, targ_ids, _ = code_to_query(m_code)
        all_desc_ids.append(desc_ids)
        all_targ_ids.append(targ_ids)

    m_list = []
    ids = zip(all_desc_ids, all_targ_ids)

    for desc_ids, targ_ids in ids:
        msg = """
        Learning model with desc ids:    {}
                            targ ids:    {}
        """.format(
            desc_ids, targ_ids
        )
        debug_print(msg, level=1, V=VERBOSITY)

        if set(targ_ids).issubset(attributes[-1:]):
            learner = RandomForestClassifier
        elif set(targ_ids).issubset(attributes[:-1]):
            learner = RandomForestRegressor
        else:
            msg = """
            Cannot learn mixed (nominal/numeric) models
            """
            raise ValueError(msg)

        # Learn a model for desc_ids-targ_ids
        m = learn_model(
            data,
            desc_ids,
            targ_ids,
            learner,
            max_depth=5,
            n_estimators=5,
            random_state=random_state,
        )
        m_list.append(m)

    return m_list

def learn_model(data, desc_ids, targ_ids, model, **kwargs):
    """
    Learn a model from the data.

    The desc ids and targ ids identify which algo task
    you should try to learn from the data.

    Model is a machine learning method that has a .fit() method.

    Args:
        data:
        desc_ids:
        targ_ids:
        model:
        **kwargs:

    Returns:

    """

    X, Y = data[:, desc_ids], data[:, targ_ids]

    if X.shape[1] == 1:
        X = X.ravel()
    if Y.shape[1] == 1:
        Y = Y.ravel()

    try:
        clf = model(**kwargs)
        clf.fit(X, Y)
    except ValueError as e:
        print(e)

    # Bookkeeping
    clf.desc_ids = desc_ids
    clf.targ_ids = targ_ids
    return clf


#!/usr/bin/env python
# -*- coding: UTF-8 -*-

import SPARQLWrapper

# REF [site] >> https://sparqlwrapper.readthedocs.io/en/latest/main.html
def select_example():
	sparql = SPARQLWrapper.SPARQLWrapper("http://vocabs.ardc.edu.au/repository/api/sparql/csiro_international-chronostratigraphic-chart_geologic-time-scale-2020")
	sparql.setReturnFormat(SPARQLWrapper.JSON)

	# Gets the first 3 geological ages from a Geological Timescale database, via a SPARQL endpoint.
	sparql.setQuery("""
PREFIX gts: <http://resource.geosciml.org/ontology/timescale/gts#>

SELECT *
WHERE {
	?a a gts:Age .
}
ORDER BY ?a
LIMIT 3
"""
	)

	try:
		ret = sparql.queryAndConvert()

		for r in ret["results"]["bindings"]:
			print(r)
	except Exception as ex:
		print(ex)

# REF [site] >> https://sparqlwrapper.readthedocs.io/en/latest/main.html
def ask_example():
	sparql = SPARQLWrapper.SPARQLWrapper("http://dbpedia.org/sparql")
	sparql.setQuery("""
ASK WHERE {
	<http://dbpedia.org/resource/Asturias> rdfs:label "Asturias"@es
}
"""
	)
	sparql.setReturnFormat(SPARQLWrapper.XML)

	try:
		results = sparql.query().convert()
		print(results.toxml())
	except Exception as ex:
		print(ex)

# REF [site] >> https://sparqlwrapper.readthedocs.io/en/latest/main.html
def construct_example():
	sparql = SPARQLWrapper.SPARQLWrapper("http://dbpedia.org/sparql")

	sparql.setQuery("""
PREFIX dbo: <http://dbpedia.org/ontology/>
PREFIX sdo: <https://schema.org/>

CONSTRUCT {
	?lang a sdo:Language ;
	sdo:alternateName ?iso6391Code .
}
WHERE {
	?lang a dbo:Language ;
	dbo:iso6391Code ?iso6391Code .
	FILTER (STRLEN(?iso6391Code)=2)  # To filter out non-valid values.
}
LIMIT 3
"""
	)

	try:
		results = sparql.queryAndConvert()
		print(results.serialize())
	except Exception as ex:
		print(ex)

# REF [site] >> https://sparqlwrapper.readthedocs.io/en/latest/main.html
def describe_example():
	sparql = SPARQLWrapper.SPARQLWrapper("http://dbpedia.org/sparql")
	sparql.setQuery("DESCRIBE <http://dbpedia.org/resource/Asturias>")

	try:
		results = sparql.queryAndConvert()
		print(results.serialize(format="json-ld"))
	except Exception as ex:
		print(ex)

# REF [site] >> https://sparqlwrapper.readthedocs.io/en/latest/main.html
def update_example():
	sparql = SPARQLWrapper.SPARQLWrapper("https://example.org/sparql")
	sparql.setHTTPAuth(SPARQLWrapper.DIGEST)
	sparql.setCredentials("some-login", "some-password")
	sparql.setMethod(SPARQLWrapper.POST)

	sparql.setQuery("""
PREFIX dbp:  <http://dbpedia.org/resource/>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>

WITH <http://example.graph>
DELETE {
	dbo:Asturias rdfs:label "Asturies"@ast
}
"""
	)

	try:
		results = sparql.query()
		print(results.response.read())
	except Exception as ex:
		print(ex)

# REF [site] >> https://sparqlwrapper.readthedocs.io/en/latest/main.html
def SPARQLWrapper2_example():
	sparql = SPARQLWrapper.SPARQLWrapper2("http://dbpedia.org/sparql")
	sparql.setQuery("""
PREFIX dbp:  <http://dbpedia.org/resource/>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>

SELECT ?label
WHERE {
	dbp:Asturias rdfs:label ?label
}
LIMIT 3
"""
	)

	try:
		for result in sparql.query().bindings:
			print(f"{result['label'].lang}, {result['label'].value}")
	except Exception as ex:
		print(ex)

# REF [site] >> https://sparqlwrapper.readthedocs.io/en/latest/main.html
def partial_interpretation_of_results():
	sparql = SPARQLWrapper.SPARQLWrapper2("http://example.org/sparql")
	sparql.setQuery("""
SELECT ?subj ?prop
WHERE {
	?subj ?prop ?obj
}
"""
	)

	try:
		ret = sparql.query()
		print(ret.variables)  # This is an array consisting of "subj" and "prop".
		for binding in ret.bindings:
			# Each binding is a dictionary. Let us just print the results.
			print(f"{binding['subj'].value}, {binding['subj'].type}")
			print(f"{binding['prop'].value}, {binding['prop'].type}")
	except Exception as ex:
		print(ex)

	#-----
	sparql.setQuery("""
SELECT ?subj ?obj ?opt
WHERE {
	?subj <http://a.b.c> ?obj .
	OPTIONAL {
		?subj <http://d.e.f> ?opt
	}
}
"""
	)

	try:
		ret = sparql.query()
		print(ret.variables)  # This is an array consisting of "subj", "obj", "opt".
		if ("subj", "prop", "opt") in ret:
			# There is at least one binding covering the optional "opt", too.
			bindings = ret["subj", "obj", "opt"]
			# Bindings is an array of dictionaries with the full bindings.
			for b in bindings:
				subj = b["subj"].value
				o = b["obj"].value
				opt = b["opt"].value

				# Do something nice with subj, o, and opt.

		# Another way of accessing to values for a single variable: take all the bindings of the "subj", "obj", "opt".
		subjbind = ret.getValues("subj")  # An array of Value instances.
		objbind = ret.getValues("obj")  # An array of Value instances.
		optbind = ret.getValues("opt")  # An array of Value instances.
	except Exception as ex:
		print(ex)

def dbpedia_test():
	sparql = SPARQLWrapper.SPARQLWrapper("http://dbpedia.org/sparql")
	sparql.setReturnFormat(SPARQLWrapper.JSON)

	if True:
		sparql.setQuery("""
SELECT ?uri ?name ?page ?nick
WHERE {
	?uri a foaf:Person ;
	foaf:name ?name;
	foaf:page ?page;
	foaf:nick ?nick.
}
LIMIT 100
"""
		)
	elif False:
		sparql.setQuery("""
SELECT ?name ?birth ?role
WHERE{
	?x a foaf:Person ;
	dbpprop:fullname ?name;
	dbpprop:countryofbirth ?birth;
	dbpprop:role ?role.

	FILTER regex(?birth, "land$").
	FILTER regex(?birth, "^Eng").
	FILTER regex(?birth, "England").
} LIMIT 100
"""
		)

	try:
		ret = sparql.queryAndConvert()
		print(ret["results"]["bindings"])
	except Exception as ex:
		print(ex)

def dbpedia_ko_test():
	sparql = SPARQLWrapper.SPARQLWrapper("http://ko.dbpedia.org/sparql")
	sparql.setReturnFormat(SPARQLWrapper.JSON)

	if False:
		sparql.setQuery("""
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
PREFIX dbo: <http://dbpedia.org/ontology/>
PREFIX dbp: <http://ko.dbpedia.org/property/>

SELECT DISTINCT ?comment
WHERE {
	?s foaf:name ?name;
	rdfs:comment ?comment;
	dbp:occupation ?occupation.
	FILTER(REGEX(STR(?occupation), '정치'))
}
LIMIT 30
"""
		)
	elif False:
		sparql.setQuery("""
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
PREFIX dbo: <http://dbpedia.org/ontology/>
PREFIX dbp: <http://ko.dbpedia.org/property/>

SELECT ?comment, ?relative, ?parent
WHERE {
	?s foaf:name ?name;
	rdfs:comment ?comment.
	FILTER(STR(?name) = '하정우')
	OPTIONAL{?relative dbo:relative ?s.}
	OPTIONAL{?parent dbo:child ?s.}
}
LIMIT 30
"""
		)
	elif True:
		sparql.setQuery("""
select * where {
	?s <http://ko.dbpedia.org/property/장소> ?o
} LIMIT 100
"""
		)
	elif False:
		sparql.setQuery("""
PREFIX dbo: <http://dbpedia.org/ontology/>
PREFIX dbp: <http://ko.dbpedia.org/property/>
PREFIX res: <http://ko.dbpedia.org/resource/>
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>

select * where {
	?s rdf:type foaf:Person.
	?s <http://ko.dbpedia.org/property/국가> '대한민국'@ko.
}
"""
		)
	elif False:
		sparql.setQuery("""
PREFIX dbo: <http://dbpedia.org/ontology/>
PREFIX dbp: <http://ko.dbpedia.org/property/>
PREFIX res: <http://ko.dbpedia.org/resource/>
PREFIX foaf: <http://xmlns.com/foaf/0.1/>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
select count(*) where {
	?s rdf:type foaf:Person.
	{?s dbp:출생일 ?Bdate.} UNION {?s dbp:사망일 ?Ddate.}
	?s dbo:abstract ?abstract.
	?s dbp:국적 ?nation.
}
"""
		)

	try:
		ret = sparql.queryAndConvert()
		print(ret["results"]["bindings"])
	except Exception as ex:
		print(ex)

def main():
	#select_example()
	#ask_example()
	#construct_example()
	#describe_example()
	#update_example()

	#SPARQLWrapper2_example()

	#partial_interpretation_of_results()

	#-----
	dbpedia_test()
	dbpedia_ko_test()

#--------------------------------------------------------------------

if "__main__" == __name__:
	main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Mar 11 14:27:38 2021

@author: maximelucas
"""

import numpy as np
import matplotlib.pyplot as plt

rho_max=250
v_max=130
long=1000
A=150
B=40000
ga=[0 for i in range(B)]
gb=[0 for i in range(B)]
rho0=[200 for i in range(A//5)]+[0 for i in range(4*A//5)]
deltaX = long/A
deltaT=deltaX * (1/(v_max+1))
rho_critique = 125

def gG(s1, s2):
    if (s1 <= 125 and s2 <= 125):
        res = f(s1)
    elif (s1 >= 125 and s2 >= 125):
        res = f(s2)
    elif (s1 < 125 and s2 > 125):
        res = min(f(s1),f(s2))
    else :
        res = f(125)
    return res
    #return f(s1)

def f(rho):
    res = rho*v(rho)
    return res

#def v(rho):
    #res = v_max
    #res = (v_max/rho_max)*(rho_max-rho)
#    return res

def v(rho):
    res = v_max
    if rho != 0:
        res = np.sqrt(100*((1000/rho)-4))
    if res > v_max :
        res = v_max
    return res

def rho_c(X):
    l = (deltaT)/(deltaX)
    Y = [0 for i in range(len(X))]
    for i in range(1, len(X)-1):
        Y[i] = X[i] -( l * (gG(X[i], X[i+1]) - gG(X[i-1], X[i])))
    return Y

v = [v(i) for i in range(250)]
plt.plot(v)

#line, = plt.plot(rho0)
#X=rho0

#for i in range(1000):
#    X = rho_c(X)
#    line.set_ydata(X)
#    plt.pause(1e-4)
#    plt.draw()


# This file is part of "Junya's self learning project about Neural Network."
#
# "Junya's self learning project about Neural Network"
# is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# "Junya's self learning project about Neural Network"
# is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Foobar.  If not, see <http://www.gnu.org/licenses/>.
#
# (c) Junya Kaneko <jyuneko@hotmail.com>

import numpy as np
from matplotlib import pyplot
import logging

__author__ = 'Junya Kaneko <jyuneko@hotmail.com>'

def sigmoid(z, alpha=1, theta=0):
    return 1 / (1 + np.exp(-alpha * (z - theta)))

def draw_sigmoid_graphs():
    dom = np.array([x / 100 for x in range(-300, 300)])
    alphas = [1, 10, 50, 100]

    nrows = 2
    ncols = int(len(alphas)/2)
    for i, alpha in enumerate(alphas):
        values = np.array([0.0] * len(dom))
        for j, x in enumerate(dom):
            values[j] = sigmoid(x, alpha=alpha)
        pyplot.subplot(nrows, ncols, i + 1)
        pyplot.title('Alpha = %s' % alpha)
        pyplot.xlabel('x')
        pyplot.ylabel('sigmoid(x)')
        pyplot.plot(dom, values)
    pyplot.tight_layout()
    pyplot.show()

class Element:
    def __init__(self, w, alpha, theta):
        self._w = w if isinstance(w, np.ndarray) else np.array(w)
        self._alpha = alpha
        self._theta = theta

    def out(self, input, alpha=None, theta=None):
        _input = input if isinstance(input, np.ndarray) else np.array(input)
        _alpha = alpha if alpha else self._alpha
        _theta = theta if theta else self._theta
        logging.debug('PARAMETERS (alpha, theta): (%s, %s)' % (_alpha, _theta))
        logging.debug('INPUT: (%s), SIGMOID: %s' % (','.join([str(i) for i in _input]), sigmoid(self._w.dot(_input), _alpha, _theta)))
        if sigmoid(self._w.dot(_input), _alpha, _theta) > 0.5:
            logging.debug('OUTPUT: %s' % 1)
            return 1
        else:
            logging.debug('OUTPUT: %s' % 0)
            return 0

    def __call__(self, input, alpha=None, theta=None):
        return self.out(input, alpha, theta)

class AndElement(Element):
    def __init__(self):
        super().__init__(w=[1, 1], alpha=1, theta=1.5)

class OrElement(Element):
    def __init__(self):
        super().__init__(w=[1, 1], alpha=1, theta=0.5)

class XorElement:

    def out(self, input):
        or_element = OrElement()

        i1 = np.array([input[0], -input[1]])
        i2 = np.array([-input[0], input[1]])

        o1 = or_element(i1)
        o2 = or_element(i2)
        return or_element([o1, o2])

    def __call__(self, input):
        return self.out(input)

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)

    draw_sigmoid_graphs()

    and_element = AndElement()
    or_element = OrElement()
    xor_element = XorElement()

    assert and_element([1, 1])
    assert not and_element([1, 0])
    assert not and_element([0, 1])
    assert not and_element([0, 0])

    assert or_element([1, 1])
    assert or_element([1, 0])
    assert or_element([0, 1])
    assert not or_element([0, 0])

    assert not xor_element([1, 1])
    assert xor_element([1, 0])
    assert xor_element([0, 1])
    assert not xor_element([0, 0])


import contextlib
import os
import shutil
import tempfile
import mandrel
import unittest

class TestCase(unittest.TestCase):
    def assertIs(self, a, b):
        # python 2.6/2.7 compatibility
        self.assertTrue(a is b)

@contextlib.contextmanager
def tempdir(dir=None):
    """Context manager that yields a temporary directory.  Cleans up afterwards."""
    if dir is not None:
        dir = os.path.realpath(os.path.expanduser(dir))

    path = os.path.realpath(tempfile.mkdtemp(dir=dir))
    try:
        yield path
    finally:
        shutil.rmtree(path, ignore_errors=True)

@contextlib.contextmanager
def chdir(path):
    """Context manager that moves to path in context; returns to original dir afterwards."""
    start_path = os.path.realpath('.')
    try:
        os.chdir(os.path.realpath(os.path.expanduser(path)))
        yield
    finally:
        os.chdir(start_path)

@contextlib.contextmanager
def workdir(dir=None):
    """Context manager that creates a temp dir, moves to it, and yields the path.

    Moves back to original dir and cleans up afterwards."""
    with tempdir(dir) as path:
        with chdir(path):
            yield path

def refresh_bootstrapper():
    if hasattr(mandrel, 'bootstrap'):
        reload(mandrel.bootstrap)
    else:
        __import__('mandrel.bootstrap')

BOOTSTRAP_FILE = 'Mandrel.py'
@contextlib.contextmanager
def bootstrap_scenario(text="", dir=None):
    with workdir(dir=dir) as path:
        bootstrapper = os.path.join(path, BOOTSTRAP_FILE)
        with open(bootstrapper, 'w') as f:
            f.write(text)
        yield path, bootstrapper


import serial,os,sys

ser = serial.Serial('/dev/ttyAMA1', 115200)
drawPath = "/home/ubuntu/my_ws/src/test_odom_rap/drawLcd/drawLcd.py"
byte_list = [0x55, 0x0E, 0x01, 0x02,
                    int(0 / 256), int(0 % 256),
                   0,0,
                    0,0,
                    0, 0, 1]
k = 0
for i in range(len(byte_list)):
    k += byte_list[i]
    k = k % 256
byte_list.append(k)
contr_law = b"%c%c%c%c%c%c%c%c%c%c%c%c%c%c" % (byte_list[0], byte_list[1], byte_list[2], byte_list[3],
                                                    byte_list[4], byte_list[5], byte_list[6], byte_list[7],
                                                    byte_list[8], byte_list[9], byte_list[10], byte_list[11],
                                                    byte_list[12], byte_list[13])

ser.write(contr_law)
try:
    if ord(ser.read())==85:
        flag = False
        receive = [ord(ser.read()) for i  in range(13)]
        bettery = int(27.322*((receive[9] * 256 + receive[10]) /100.0) - 245.9)
        print(bettery)
        if bettery <0:
            bettery = 0
        elif bettery>100:
            bettery = 100
        try:
            print(bettery)
            os.popen("python "+" "+drawPath+" "+str(bettery)+" 0 0")
        except IOError:
            res = os.system("sudo chmod 777 /dev/i2c-1&sudo chown root.gpio /dev/gpiomem&sudo chmod g+rw /dev/gpiomem&sudo chmod 777 /dev/spidev0.*")
            os.popen("python "+" "+drawPath+" "+str(bettery)+" 0 0")
except TypeError:
    print('error')


import os
import sqlite3
import shutil
import sys
import psutil

def is_browser_running(browser_name):
    for process in psutil.process_iter():
        try:
            if browser_name in process.name().lower():
                return True
        except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
            pass
    return False

if is_browser_running('firefox'):
    print("Firefox가 실행 중입니다. 브라우저를 닫고 다시 시도하세요.")

# 환경에 따른 경로 설정
if os.name == 'nt':  # Windows 환경
    base_path = "C:/Users"
else:  # WSL 환경
    base_path = "/mnt/c/Users"

# Windows 사용자 이름
windows_username = "user"  # 여기에 실제 Windows 사용자 이름을 입력하세요

# Firefox 프로필 경로
firefox_path = os.path.join(base_path, windows_username, "AppData", "Roaming", "Mozilla", "Firefox")

# Chrome 프로필 경로
chrome_path = os.path.join(base_path, windows_username, "AppData", "Local", "Google", "Chrome", "User Data", "Default", "History")

# SQLite 데이터를 추출하는 함수
def extract_sqlite_data(db_path, query):
    try:
        conn = sqlite3.connect(db_path)
        cursor = conn.cursor()
        cursor.execute(query)
        rows = cursor.fetchall()
        conn.close()
        return rows
    except Exception as e:
        print(f"Error reading {db_path}: {str(e)}")
        return []

if not os.access(firefox_path, os.R_OK):
    print("Firefox 경로에 접근할 수 없습니다. 권한을 확인하세요.")

# 출력을 파일에 저장할 경로
output_file_path = "linuxweboutput.txt"

# 기존 출력 스트림을 백업하고 파일로 변경
original_stdout = sys.stdout
with open(output_file_path, "w") as f:
    sys.stdout = f

    # Firefox 인터넷 기록을 추출합니다.
    for profile_directory in os.listdir(firefox_path):
        if profile_directory.endswith('.default-release'):
            places_db = os.path.join(firefox_path, profile_directory, 'places.sqlite')
            if os.path.exists(places_db):
                temp_copy = os.path.join("/tmp", "temp_places.sqlite")
                shutil.copy(places_db, temp_copy)
                query = "SELECT url, title, last_visit_date FROM moz_places"
                records = extract_sqlite_data(temp_copy, query)
                os.remove(temp_copy)
                for record in records:
                    print(record)

    # Chrome 인터넷 기록을 추출합니다.
    if os.path.exists(chrome_path):
        temp_copy = os.path.join("/tmp", "temp_history.sqlite")
        shutil.copy(chrome_path, temp_copy)
        query = "SELECT url, title, last_visit_time FROM urls"
        records = extract_sqlite_data(temp_copy, query)
        os.remove(temp_copy)
        for record in records:
            print(record)

# 기존 출력 스트림을 복원
sys.stdout = original_stdout