import sys
import numpy as np
import math
import copy
import random
import time

import os
import signal
import datetime
import subprocess

import json
from pydoc import locate
import shutil
from datetime import datetime

import glob

from dask.distributed import Client, LocalCluster
from dask_jobqueue import SLURMCluster

def get_config_PLAIN_ES_DIRECTIONAL_ANT(config):
    config = copy.deepcopy(config)
    config["n_iterations"] = 200
    config["ES_type"] = "plain_ES"  # can be plain_ES,maxvar_ES,maxent_ES,nondominated_ES
    config["env_name"] = "DirectionalAnt"  # Ant
    EXPERIMENT_NAME = config["ES_type"] + "_" + config["env_name"]
    return config,EXPERIMENT_NAME

def get_config_PLAIN_ES_NORMAL_ANT(config):
    config = copy.deepcopy(config)
    config["n_iterations"] = 200
    config["ES_type"] = "plain_ES"  # can be plain_ES,maxvar_ES,maxent_ES,nondominated_ES
    config["env_name"] = "Ant"  # Ant
    EXPERIMENT_NAME = config["ES_type"] + "_" + config["env_name"]
    return config,EXPERIMENT_NAME

def get_config_PLAIN_ES_DECEPTIVE_ANT(config):
    config = copy.deepcopy(config)
    config["n_iterations"] = 200
    config["ES_type"] = "plain_ES"  # can be plain_ES,maxvar_ES,maxent_ES,nondominated_ES
    config["env_name"] = "DeceptiveAnt"
    EXPERIMENT_NAME = config["ES_type"] + "_" + config["env_name"]
    return config,EXPERIMENT_NAME

def get_config_NONDOMINATED_ES_DIRECTIONAL_ANT(config):
    config = copy.deepcopy(config)
    config["n_iterations"] = 200
    config["ES_type"] = "nondominated_ES"  # can be plain_ES,maxvar_ES,maxent_ES,nondominated_ES
    config["env_name"] = "DirectionalAnt"  # Ant
    EXPERIMENT_NAME = config["ES_type"] + "_" + config["env_name"]
    return config,EXPERIMENT_NAME

def get_config_NONDOMINATED_ES_NORMAL_ANT(config):
    config = copy.deepcopy(config)
    config["n_iterations"] = 200
    config["ES_type"] = "nondominated_ES"  # can be plain_ES,maxvar_ES,maxent_ES,nondominated_ES
    config["env_name"] = "Ant"  # Ant
    EXPERIMENT_NAME = config["ES_type"] + "_" + config["env_name"]
    return config,EXPERIMENT_NAME

def get_config_NONDOMINATED_ES_DECEPTIVE_ANT(config):
    config = copy.deepcopy(config)
    config["n_iterations"] = 200
    config["ES_type"] = "nondominated_ES"  # can be plain_ES,maxvar_ES,maxent_ES,nondominated_ES
    config["env_name"] = "DeceptiveAnt"  # Ant
    EXPERIMENT_NAME = config["ES_type"] + "_" + config["env_name"]
    return config,EXPERIMENT_NAME

def get_config_EVO_ES_DIRECTIONAL_ANT(config):
    config = copy.deepcopy(config)
    config["n_iterations"] = 200
    config["ES_type"] = "maxvar_ES"  # can be plain_ES,maxvar_ES,maxent_ES,nondominated_ES
    config["env_name"] = "DirectionalAnt"  # Ant
    EXPERIMENT_NAME = config["ES_type"] + "_" + config["env_name"]
    return config,EXPERIMENT_NAME

def get_config_EVO_ES_NORMAL_ANT(config):
    config = copy.deepcopy(config)
    config["n_iterations"] = 200
    config["ES_type"] = "maxvar_ES"  # can be plain_ES,maxvar_ES,maxent_ES,nondominated_ES
    config["env_name"] = "Ant"  # Ant
    EXPERIMENT_NAME = config["ES_type"] + "_" + config["env_name"]
    return config,EXPERIMENT_NAME

def get_all_ant_configs():

    configs = []
    experiment_names = []

    get_conf_funs = [
        get_config_PLAIN_ES_DIRECTIONAL_ANT,
        get_config_PLAIN_ES_NORMAL_ANT,
        get_config_PLAIN_ES_DECEPTIVE_ANT,

        get_config_NONDOMINATED_ES_DIRECTIONAL_ANT,
        get_config_NONDOMINATED_ES_NORMAL_ANT,
        get_config_NONDOMINATED_ES_DECEPTIVE_ANT,

        get_config_EVO_ES_DIRECTIONAL_ANT,
        get_config_EVO_ES_NORMAL_ANT,
    ]

    for fun in get_conf_funs:
        config,EXPERIMENT_NAME = fun(default_config)
        configs.append(config)
        experiment_names.append(EXPERIMENT_NAME)

    return configs,experiment_names

def get_deceptive_ant_configs():

    configs = []
    experiment_names = []

    get_conf_funs = [
        get_config_PLAIN_ES_DECEPTIVE_ANT,
        get_config_NONDOMINATED_ES_DECEPTIVE_ANT,
    ]

    for fun in get_conf_funs:
        config,EXPERIMENT_NAME = fun(default_config)
        configs.append(config)
        experiment_names.append(EXPERIMENT_NAME)

    return configs,experiment_names

def get_config_PLAIN_ES(env_name,config):
    config = copy.deepcopy(config)
    config["ES_type"] = "plain_ES"  # can be plain_ES,maxvar_ES,maxent_ES,nondominated_ES
    config["env_name"] = env_name
    EXPERIMENT_NAME = config["ES_type"] + "_" + config["env_name"]
    return config,EXPERIMENT_NAME

def get_config_NONDOMINATED_ES(env_name,config):
    config = copy.deepcopy(config)
    config["ES_type"] = "nondominated_ES"  # can be plain_ES,maxvar_ES,maxent_ES,nondominated_ES
    config["env_name"] = env_name
    EXPERIMENT_NAME = config["ES_type"] + "_" + config["env_name"]
    return config,EXPERIMENT_NAME

def get_config_EVO_ES(env_name,config):
    config = copy.deepcopy(config)
    config["ES_type"] = "maxvar_ES"  # can be plain_ES,maxvar_ES,maxent_ES,nondominated_ES
    config["env_name"] = env_name
    EXPERIMENT_NAME = config["ES_type"] + "_" + config["env_name"]
    return config,EXPERIMENT_NAME

def get_all_humanoid_configs():

    configs = []
    experiment_names = []

    env_name = "humanoid"
    config,EXPERIMENT_NAME = get_config_PLAIN_ES(env_name,default_config)
    config["n_iterations"] = 1200
    configs.append(config)
    experiment_names.append(EXPERIMENT_NAME)

    config,EXPERIMENT_NAME = get_config_NONDOMINATED_ES(env_name,default_config)
    config["n_iterations"] = 1200
    configs.append(config)
    experiment_names.append(EXPERIMENT_NAME)

    config,EXPERIMENT_NAME = get_config_EVO_ES(env_name,default_config)
    config["n_iterations"] = 1200
    configs.append(config)
    experiment_names.append(EXPERIMENT_NAME)

    env_name = "directionalhumanoid"
    config,EXPERIMENT_NAME = get_config_PLAIN_ES(env_name,default_config)
    config["n_iterations"] = 1200
    configs.append(config)
    experiment_names.append(EXPERIMENT_NAME)

    config,EXPERIMENT_NAME = get_config_NONDOMINATED_ES(env_name,default_config)
    config["n_iterations"] = 1200
    configs.append(config)
    experiment_names.append(EXPERIMENT_NAME)

    #config,EXPERIMENT_NAME = get_config_EVO_ES(env_name,default_config) # SKIP THIS,

    env_name = "deceptivehumanoid"
    config,EXPERIMENT_NAME = get_config_PLAIN_ES(env_name,default_config)
    config["n_iterations"] = 1200
    configs.append(config)
    experiment_names.append(EXPERIMENT_NAME)

    config,EXPERIMENT_NAME = get_config_NONDOMINATED_ES(env_name,default_config)
    config["n_iterations"] = 1200
    configs.append(config)
    experiment_names.append(EXPERIMENT_NAME)

    config,EXPERIMENT_NAME = get_config_EVO_ES(env_name,default_config)
    config["n_iterations"] = 1200
    configs.append(config)
    experiment_names.append(EXPERIMENT_NAME)

    return configs,experiment_names

def get_some_humanoid_configs():
    configs = []
    experiment_names = []

    env_name = "humanoid"
    config,EXPERIMENT_NAME = get_config_NONDOMINATED_ES(env_name,default_config)
    config["n_iterations"] = 1200
    configs.append(config)
    experiment_names.append(EXPERIMENT_NAME)

    env_name = "directionalhumanoid"
    config,EXPERIMENT_NAME = get_config_NONDOMINATED_ES(env_name,default_config)
    config["n_iterations"] = 1200
    configs.append(config)
    experiment_names.append(EXPERIMENT_NAME)

    env_name = "deceptivehumanoid"
    config,EXPERIMENT_NAME = get_config_NONDOMINATED_ES(env_name,default_config)
    config["n_iterations"] = 1200
    configs.append(config)
    experiment_names.append(EXPERIMENT_NAME)

    return configs,experiment_names

def generate_job_file(job_folder_path,experiment_name,USE_WEEK_QUEUE):

    lines = []
    lines.append("#!/bin/bash")
    if USE_WEEK_QUEUE is True:
        lines.append("#SBATCH --partition=week")
        lines.append("#SBATCH --time=168:00:00")
    else:
        # we dont specify a partition, will default to normal queue
        lines.append("#SBATCH --time=48:00:00")

    lines.append("#SBATCH --job-name=dask_scheduler_" + experiment_name)
    lines.append("#SBATCH --account=CS-DCLABS-2019")
    lines.append("#SBATCH --export=ALL")
    lines.append("#SBATCH --ntasks=1")
    # --cpus-per-task can be used to request multiple cpus on the same node, so multithreaded code works.
    lines.append("#SBATCH --cpus-per-task=4") # The main thread is responsible for calulation the weighted average, few extra cpus are good to have
    lines.append("#SBATCH --mem=16gb")

    lines.append("")
    lines.append("python /users/ak1774/scratch/Evolvability_cheetah/Evolvability-ES/run_with_jobfile.py " + job_folder_path)

    return lines

def get_last_iteration_of_run(run_folder):
    files = glob.glob(run_folder + "/*")
    files = [file.split("/")[-1] for file in files]
    latest_iteration = max([int(file.replace("niche_","").replace(".pickle","")) for file in files if "niche" in file])
    return latest_iteration

def get_continue_job_details(CONFIG_FOLDERS):

    runs_to_continue = []
    for CONFIG_FOLDER in CONFIG_FOLDERS:
        subfolders = glob.glob(CONFIG_FOLDER + "/*/")
        subfolders = [folder for folder in subfolders if "dask-worker-space" not in folder]

        for folder in subfolders:
            last_iter = get_last_iteration_of_run(folder)

            if last_iter < 799:
                runs_to_continue.append({
                    "config_folder" : CONFIG_FOLDER,
                    "run_folder" : folder,
                    "last_iter" : last_iter,
                    "niche_filename" : folder + "niche_"+str(last_iter)+".pickle",
                    "pop_filename" : folder + "pop_"+str(last_iter)+".pickle",
                })

    return runs_to_continue

def get_all_config_folders():

    root_folders = [
        "LONG_RESUBMIT_01_18___14:54",
        "NODES_01_14___12:19",
        "NODES_01_09___12:04",
        "NODES_12_28___14:50",
        #"TEST_12_23___15:08",  # fitness is fucked up
        #"NODES_01_14___10:09",   # just started running
        "NODES_12_31___18:46",
        #"WEEK_12_28___14:45",  # some did not run properly because not enough workers
        "WEEK_NONDOMINATED_ONLY_01_14___12:25"
    ]

    CONFIG_TYES = ["plain_ES_humanoid","nondominated_ES_humanoid","maxvar_ES_humanoid",
        "plain_ES_deceptivehumanoid","nondominated_ES_deceptivehumanoid",
        "plain_ES_directionalhumanoid","nondominated_ES_directionalhumanoid"]

    root_folders = ["/users/ak1774/scratch/Evolvability_cheetah/job_results/"+f for f in root_folders]
    all_config_folders = []
    for root_f in root_folders:
        for c in CONFIG_TYES:
            all_config_folders.append(root_f+"/"+c)

    return all_config_folders

def run_continue_jobs():

    # for this job we dont need to specify configs and stuff, because we just want to continue existing jobs.

    NUM_PARRALEL_JOBS = 4
    USE_WEEK_QUEUE = False

    CONFIG_FOLDERS = get_all_config_folders()

    continue_jobs = get_continue_job_details(CONFIG_FOLDERS)
    print("Number of jobs to continue: ",len(continue_jobs))

    current_i = 0
    while True:

        done = False
        # very sophisticated method to check how many jobs are running
        cmd = "squeue -u ak1774 | grep sch | wc -l"
        ps = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
        process_output = ps.communicate()[0]
        if len(process_output) > 10:
            # the process output is too large, there were probably some error
            # For example squeue can fail with: slurm_load_jobs error: Socket timed out on send/recv operation
            continue

        num_schedulers_running = int(process_output.decode("utf-8"))

        free_slots = NUM_PARRALEL_JOBS - num_schedulers_running
        for _ in range(free_slots):

            if current_i < len(continue_jobs):
                print("submitting continue job: ",current_i,"/",len(continue_jobs))
                print(continue_jobs[current_i]["run_folder"],)

                current_continue_job = continue_jobs[current_i]

                run_dir = current_continue_job["run_folder"]
                continue_dir_path = current_continue_job["run_folder"] + "continue"
                os.makedirs(continue_dir_path,exist_ok=True)
                os.chdir(continue_dir_path)

                current_continue_job
                with open("continue_job.json", 'w') as outfile:
                    json.dump(current_continue_job,outfile, indent=4)

                lines = generate_job_file(job_folder_path=continue_dir_path,experiment_name="continue",USE_WEEK_QUEUE=USE_WEEK_QUEUE)

                # save job file
                with open("job.job", 'w') as f:
                    for line in lines:
                        f.write("%s\n" % line)

                # submit job file
                subprocess.run(["sbatch", "job.job"])
                print("JOB SUBMITTED, ",current_i)

                current_i = current_i + 1

            else:
                print("Last job submitted, we are DONE here.")
                done = True

        if done is True:
            break

        time.sleep(30)

if __name__ == '__main__':

    if True:
        run_continue_jobs()
        print("QUIT")
        sys.exit()

    print("QUIT FAILED")

    default_config = {
        "ES_type" : "nondominated_ES",   # can be plain_ES,maxvar_ES,maxent_ES,nondominated_ES
        "ModelType" : "plain_FF", #"CPPN", # can be plain_FF,CPPN
        "ModelExtraArgs" : {"use_positional_encoding" : True,
                            "use_bottleneck" : False,
                            "query_list_positional_encoding_max_wavelength" : 2000,
                            "query_list_positional_encoding_width" : 32,
                            "embedding_positional_encoding_max_wavelength" : 2000,
                            "embedding_positional_encoding_width" : 32,
                            "embedding_positional_encoding_resolution" : 500},

        "learning_rate" : 0.01,
        "batch_size" : 10,#10,
        "pop_size" : 10000,#10000,
        "l2_coeff" : 0.005,
        "noise_std" : 0.02,
        "noise_decay" : 1.0,
        "n_iterations" : 1500,  # was 200 for ant, was 800 for humanoid
        "n_rollouts" : 1,
        "env_deterministic" : True,
        "returns_normalization" : "centered_ranks",
        "evals_per_step" : 100, # was 100
        "eval_batch_size" : 10, # was 10
        "single_threaded" : True,
        "gpu_mem_frac" : 0.2,
        "action_noise" : 0.00,
        "progress" : False,
        "seed" : None, # was 42, only Ant was using it, probably only relevant for eval, because on pop we have random noise in parameters.
        "niche_path" : "niche_{iteration}.pickle",
        "pop_path" : "pop_{iteration}.pickle",
        "po_path" : "po_gen_{iteration}.pickle",
        "device" : "cpu",
        "log_fnm" : "log.json",
        "env_name" : "DirectionalAnt",
        "extra_config" : {
            "HUMANOID_USE_BUILT_IN_REWARDS" : False,
        }
        }

    RUN_LOCALLY = False

    RUN_CONTINUOUS_SUBMISSION = True

    if RUN_CONTINUOUS_SUBMISSION is True:

        # for this mode, we continuously monitor the running jobs, and if there is a spot, we submit a new one.
        # I dont have to resubmit every 48 hours, and there is no half finished run.
        # each job is only running a single run.
        # we go around the configs, so each config will have the same amount of runs.

        # To get the number of active jobs, we check the number of schedulaer jobs running.
        # "squeue -u ak1774 | grep "sch" | wc -l"

        NUM_PARRALEL_JOBS = 4
        USE_WEEK_QUEUE = False

        SUBMISSION_NAME = "RESUBMIT_DECEPTIVE_ANT_PLUS"
        RESULTS_ROOT = "/users/ak1774/scratch/Evolvability_cheetah/job_results"
        os.makedirs(RESULTS_ROOT,exist_ok=True)

        SUBMISSION_ROOT = os.path.join(RESULTS_ROOT, SUBMISSION_NAME + datetime.now().strftime("_%m_%d___%H:%M"))
        os.makedirs(SUBMISSION_ROOT,exist_ok=True)

        #configs,experiment_names = get_all_humanoid_configs()
        configs,experiment_names = get_deceptive_ant_configs()

        current_i = 0
        while True:

            cmd = "squeue -u ak1774 | grep sch | wc -l"
            ps = subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
            process_output = ps.communicate()[0]
            num_schedulers_running = int(process_output.decode("utf-8"))

            free_slots = NUM_PARRALEL_JOBS - num_schedulers_running
            for _ in range(free_slots):

                current_experiment_i = current_i % len(configs)
                current_experiment_name = experiment_names[current_experiment_i]
                current_config = configs[current_experiment_i]

                main_experiment_dir = os.path.join(SUBMISSION_ROOT, current_experiment_name)
                os.makedirs(main_experiment_dir,exist_ok=True)
                os.chdir(main_experiment_dir)
                # save job config list, dont care if it already exsist
                with open("config.json", 'w') as outfile:
                    json.dump(current_config,outfile, indent=4)
                # generate job file
                lines = generate_job_file(main_experiment_dir,current_experiment_name,USE_WEEK_QUEUE)

                # save job file
                with open("job.job", 'w') as f:
                    for line in lines:
                        f.write("%s\n" % line)

                # submit job file
                subprocess.run(["sbatch", "job.job"])
                print("JOB SUBMITTED, ",current_i," ",current_experiment_name)

                current_i += 1

            time.sleep(30)

    if RUN_LOCALLY == False:

        SUBMISSION_NAME = "NONDOMINATED_ONLY"
        USE_WEEK_QUEUE = False

        RESULTS_ROOT = "/users/ak1774/scratch/Evolvability_cheetah/job_results"
        os.makedirs(RESULTS_ROOT,exist_ok=True)

        SUBMISSION_ROOT = os.path.join(RESULTS_ROOT, SUBMISSION_NAME + datetime.now().strftime("_%m_%d___%H:%M"))
        os.makedirs(SUBMISSION_ROOT,exist_ok=True)

        #################################
        # CREATE A LIST OF CONFIGS
        ##################################

        #configs,experiment_names = get_all_humanoid_configs()
        #configs,experiment_names = get_some_humanoid_configs()  # some runs take longer, only run those so we have enough of those too
        configs,experiment_names = get_deceptive_ant_configs()

        ##############
        # SUBMIT JOBS
        ##############

        for config,EXPERIMENT_NAME in zip(configs,experiment_names):

            main_experiment_dir = os.path.join(SUBMISSION_ROOT, EXPERIMENT_NAME)

            os.makedirs(main_experiment_dir,exist_ok=True)
            os.chdir(main_experiment_dir)

            # save job config list
            with open("config.json", 'w') as outfile:
                json.dump(config,outfile, indent=4)

            # generate job file
            lines = generate_job_file(main_experiment_dir,EXPERIMENT_NAME,USE_WEEK_QUEUE)

            # save job file
            with open("job.job", 'w') as f:
                for line in lines:
                    f.write("%s\n" % line)

            # submit job file
            subprocess.run(["sbatch", "job.job"])
            print("JOB SUBMITTED, "+EXPERIMENT_NAME)

        print("All jobs submitted!!")

    else:

        RESULTS_ROOT = "/users/ak1774/scratch/Evolvability_cheetah/runs_humanoid"

        # Select environment
        ENV = "SLURM"
        #ENV = "LOCAL"
        #ENV = "CSGPU_LOCAL"

        NUM_WORKERS = 60  # max worker per user is 500, with 80 we can run 6 instances concurrently
        NUM_REPEAT_EXPERIMENT = 500   # repeat the experiment this many times, to see the variance in results

        queue_name = "nodes"
        job_time_limit = '48:00:00'

        USE_WEEK_QUEUE = False  # else it is the normal queue, with 48 hours
        if USE_WEEK_QUEUE is True:
            queue_name = "week"
            job_time_limit = '168:00:00'
            NUM_WORKERS = 40
            # week queue has 400 cpus total
            # maybe aim to use half 200 / 5 = 40 cpu per config

        DUBUG_SCALE = False
        if DUBUG_SCALE is True:
            NUM_WORKERS = 5
            config["pop_size"] = 10
            config["batch_size"] = 2
            config["evals_per_step"] = 4
            config["eval_batch_size"] = 2

        #################################
        # SELCT 1 CONFIG TO RUN LOCALLY
        ##################################

        #config,EXPERIMENT_NAME = get_config_PLAIN_ES_DIRECTIONAL_ANT(default_config)
        #config,EXPERIMENT_NAME = get_config_PLAIN_ES_NORMAL_ANT(default_config)
        #config,EXPERIMENT_NAME = get_config_NONDOMINATED_ES_DIRECTIONAL_ANT(default_config)
        #config,EXPERIMENT_NAME = get_config_NONDOMINATED_ES_NORMAL_ANT(default_config)
        #config,EXPERIMENT_NAME = get_config_EVO_ES_DIRECTIONAL_ANT(default_config)   # SKIP THIS, since it is almost the same as EVO_ES_NORMAL_ANT
        #config,EXPERIMENT_NAME = get_config_EVO_ES_NORMAL_ANT(default_config)

        env_name = "humanoid"
        #config,EXPERIMENT_NAME = get_config_PLAIN_ES(env_name,default_config)
        #config,EXPERIMENT_NAME = get_config_NONDOMINATED_ES(env_name,default_config)
        #config,EXPERIMENT_NAME = get_config_EVO_ES(env_name,default_config)

        env_name = "directionalhumanoid"
        #config,EXPERIMENT_NAME = get_config_PLAIN_ES(env_name,default_config)
        #config,EXPERIMENT_NAME = get_config_NONDOMINATED_ES(env_name,default_config)
        #config,EXPERIMENT_NAME = get_config_EVO_ES(env_name,default_config) # SKIP THIS,

        env_name = "deceptivehumanoid"
        #config,EXPERIMENT_NAME = get_config_PLAIN_ES(env_name,default_config)
        #config,EXPERIMENT_NAME = get_config_NONDOMINATED_ES(env_name,default_config)
        config,EXPERIMENT_NAME = get_config_EVO_ES(env_name,default_config)

        main_experiment_dir = os.path.join(RESULTS_ROOT, EXPERIMENT_NAME)
        os.makedirs(main_experiment_dir,exist_ok=True)
        os.chdir(main_experiment_dir)

        if "SLURM" in ENV:
            PROJECT_PATH = "/users/ak1774/scratch/Evolvability_cheetah/Evolvability-ES"
        elif ENV == "LOCAL":
            PROJECT_PATH = "/users/ak1774/scratch/Evolvability_cheetah/Evolvability-ES"
        elif ENV == "CSGPU_LOCAL":
            PROJECT_PATH = "/home/userfs/a/ak1774/workspace/Evolvability_cheetah/Evolvability-ES"
        else:
            raise "set up project path!!"

        # This is needed so the dask workers can import project files
        export_python_path_command = "export PYTHONPATH=${PYTHONPATH}:" + PROJECT_PATH

        ####################
        ## SET UP CLUSTER ##
        ####################

        # Befor we set up the cluster, increase the number of allowed open files
        import resource
        soft_limit,hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
        print("Open file descriptor limit: ",soft_limit," ",hard_limit)
        if soft_limit < hard_limit:
            resource.setrlimit(resource.RLIMIT_NOFILE,(hard_limit,hard_limit))
            print("Raised the limit to: ",resource.getrlimit(resource.RLIMIT_NOFILE))

        if ENV == "SLURM":

            cluster = SLURMCluster(  # these params describe a single worker
            queue= queue_name,  # for viking: nodes or week
            #cores=1,
            #memory='1GB',

            cores=1,
            processes = 1,
            memory='4GB',   # was 1GB for Ant, tried 2GB still had warning about not enough memory,

            walltime= job_time_limit, # '48:00:00',
            project="CS-DCLABS-2019",
            scheduler_options={'interface': "bond0"},  # this is the interface on the login node (the default is not visible to the workers)
            job_extra=["--export=ALL"], # extra #SBATCH options
            env_extra=[export_python_path_command,
                        #"conda activate evo"
                        "source /users/ak1774/scratch/torch_env/bin/activate"]  # extra commands to execute before starting the dask-worker
            )

            print("##### THE JOB SCRIPT: ######")
            print(cluster.job_script())
            print("############################")

            #cluster.scale(jobs=NUM_WORKERS)  # this will submit the jobs, they will register to the sceduler whenever they got executed
            cluster.scale(cores=NUM_WORKERS)
            client = Client(cluster)

        elif ENV == "LOCAL" or ENV == "CSGPU_LOCAL":
            cluster = LocalCluster(n_workers=NUM_WORKERS,processes=True,threads_per_worker=1)
            client = Client(cluster)
        else:
            raise "Choose a valid env"

        # dump config
        with open("config.json", 'w') as outfile:
            json.dump(config,outfile)

        # start evolution
        print("Waiting for workers...")
        client.wait_for_workers(1) # wait until there is at least a single worker (it can ramp up later)
        print("First worker registered!")

        for experiment_number in range(NUM_REPEAT_EXPERIMENT):

            sub_experiment_dir = os.path.join(main_experiment_dir, str(experiment_number) + datetime.now().strftime("_%m_%d___%H:%M"))
            os.makedirs(sub_experiment_dir,exist_ok=True)
            os.chdir(sub_experiment_dir)

            import main
            main.main(client,**config)
            print("Finished experiment ",experiment_number)

            # go back to the main experiment dir
            os.chdir(main_experiment_dir)


from rest_framework.status import HTTP_200_OK, HTTP_201_CREATED, HTTP_204_NO_CONTENT
from django.urls import reverse

from orders.models import CartItem
from tests.settings import TEST_QUANTITY_ONE_PRODUCT_IN_CART_ITEM, TEST_QUANTITY_PRODUCT

class TestCart:

    def test_cart_item_list(self, get_client_of_cart_item):
        client, _ = get_client_of_cart_item

        url_cart_list = reverse("orders:cart-list")

        response = client.get(url_cart_list)
        assert response.status_code == HTTP_200_OK
        assert len(response.data) == 1

    def test_cart_item_create(self, get_auth_client, create_product):
        product = create_product
        client, _ = get_auth_client

        url_cart_create = reverse("orders:cart-list")

        response = client.post(
            url_cart_create,
            data={
                'product_id': product.id
            }
        )
        assert response.status_code == HTTP_201_CREATED
        assert response.data.get('product_id') == product.id

    def test_cart_item_diminish_product(self, get_client_of_cart_item):
        client, cart_item = get_client_of_cart_item

        url_cart_create = reverse("orders:cart-diminish-product")

        response = client.post(
            url_cart_create,
            data={
                'product_id': cart_item.product.id
            }
        )
        cart_item.refresh_from_db()

        assert response.status_code == HTTP_204_NO_CONTENT
        assert cart_item.quantity == TEST_QUANTITY_ONE_PRODUCT_IN_CART_ITEM - 1
        assert cart_item.product.quantity == TEST_QUANTITY_PRODUCT + 1

    def test_cart_item_increase_product(self, get_client_of_cart_item):
        client, cart_item = get_client_of_cart_item

        url_cart_create = reverse("orders:cart-increase-product")

        response = client.post(
            url_cart_create,
            data={
                'product_id': cart_item.product.id
            }
        )
        cart_item.refresh_from_db()

        assert response.status_code == HTTP_204_NO_CONTENT
        assert cart_item.quantity == TEST_QUANTITY_ONE_PRODUCT_IN_CART_ITEM + 1
        assert cart_item.product.quantity == TEST_QUANTITY_PRODUCT - 1

    def test_cart_item_destroy_product(self, get_client_of_cart_item):
        client, cart_item = get_client_of_cart_item

        url_cart_create = reverse(
            "orders:cart-detail",
            kwargs={
                'product_id': cart_item.product.id
            }
        )

        response = client.delete(url_cart_create)

        is_product_in_cart_item = CartItem.objects.filter(
            user=cart_item.user,
            product_id=cart_item.product
        ).exists()

        assert response.status_code == HTTP_204_NO_CONTENT
        assert is_product_in_cart_item is False


import pygame
import random
import time
main_loop = True
timer = 5
main_screen, run_game, help = True, False, False
level_define, level = 1, 0
screen_width, screen_height = 900, 600
heightObject, widthObject = 80, 25
xPlayer, yPlayer = 10, screen_height/2
xComp, yComp = screen_width-40, screen_height/2
xBall, yBall = screen_width/2 - 10, screen_height/2
vel, score = 5, 0
stop_attack, check_lose = True, True
stop_attack_list = [False, True, True, False, False,True, False, True, True]
global velBall
velBall = 3

velBall_list = [velBall, velBall - 2*velBall, velBall - 2*velBall, velBall - 2*velBall]
global began_motion, began_motion
began_motion = True
moveBall = False
class Player():
    def __init__(self, win, x, y):
        pygame.draw.rect(win, (150, 0, 255), (x,y, widthObject, heightObject))

class Comp():
    def __init__(self, win, x, y):
        pygame.draw.rect(win, (150, 0, 255), (x,y, widthObject, heightObject))

class Ball():
    def __init__(self, win, x, y):
        pygame.draw.circle(win, (243, 2, 21), (x,y), 20)

pygame.init()
font = pygame.font.SysFont("Arial.tff", 45, False, True)

class mn_images():
    def __init__(self, win):
        bg_img = pygame.image.load('mn_scrbg.jpg')
        bg_img = pygame.transform.scale(bg_img, (screen_width, screen_height))
        win.blit(bg_img, (0,0))

class button():
    def __init__(self, win, pos):
        button = pygame.image.load('button_.png')
        button = pygame.transform.scale(button, (300, 150))
        win.blit(button, (screen_width- 800, screen_height- 400))
        win.blit(button, (screen_width - 400, screen_height-400))
        mn_font = pygame.font.SysFont("snapitc.tff", 50)
        text1 = mn_font.render("Play", True, (200, 0, 100))
        text2 = mn_font.render("Help", True, (200, 0, 100))
        win.blit(text1, (screen_width- 700, screen_height- 350))
        win.blit(text2, (screen_width - 300, screen_height- 350))
help_font = pygame.font.SysFont("snapitc.tff", 50)
help_text_font = pygame.font.SysFont('snapitic.tff', 40)
class Help():
    def __init__(self, win):
        bg_img = pygame.image.load('mn_scrbg.jpg')
        bg_img = pygame.transform.scale(bg_img,(screen_width, screen_height))
        win.blit(bg_img, (0,0))
    def inst(self, win):
        round = pygame.image.load('rect.png')
        round = pygame.transform.scale(round, (int(screen_width/2)+ 400, int(screen_height/2)+ 100))
        win.blit(round, (screen_width/2- 420, screen_height/2- 250))
        text = help_text_font.render("It is a single player game", True, (200, 200, 10))
        win.blit(text, (screen_width/2- 190, screen_height/2- 180))
        text1 = help_text_font.render("You can control it by UP & DOWN arrow", True, (200, 200, 10))
        win.blit(text1, (screen_width / 2 - 260, screen_height / 2 - 130))
        text2 = help_text_font.render("As you win, the score will increase as well as the level", True, (200, 200, 10))
        win.blit(text2, (screen_width / 2 - 360, screen_height / 2 - 80))
        text3 = help_text_font.render("and the speed of the ball too", True, (200, 200, 10))
        win.blit(text3, (screen_width / 2 - 200, screen_height / 2- 30))
        text4 = help_text_font.render("If you fail to hit the ball, the game will reset", True, (200, 200, 10))
        win.blit(text4, (screen_width / 2 - 300, screen_height /2 + 20))

    def help_button(self, win):
        but = pygame.image.load("button_.png")
        but = pygame.transform.scale(but, (200, 100))
        win.blit(but, (screen_width/2- 100, screen_height/2 + 150))
        text = help_font.render("Home", True, (200, 0, 100))
        win.blit(text,(screen_width/2- 50, screen_height/2 + 180))

class game():
    def __init__(self):
        super(game, self).__init__()
        global main_screen, run, run_game, timer, lose_text, lose_timer, help, main_loop
        xPlayer, yPlayer = 10, screen_height/2
        self.window = pygame.display.set_mode((screen_width, screen_height))
        ico = pygame.image.load("pong-ball-ico.png")
        pygame.display.set_caption("Pong Ball")
        pygame.display.set_icon(ico)
        run = False
        while main_loop:
            for event in pygame.event.get():
                mouse_pos = pygame.mouse.get_pos()
                if event.type == pygame.QUIT:
                    main_loop = False
            while main_screen:
                for event in pygame.event.get():
                    mouse_pos = pygame.mouse.get_pos()
                    if event.type == pygame.QUIT:
                        main_loop = False
                        main_screen = False
                    if event.type == pygame.MOUSEBUTTONDOWN:
                        if mouse_pos[0] in range(120, 375) and mouse_pos[1] in range(225, 323):
                            main_screen = False
                            run = True
                        elif mouse_pos[0] in range(525, 775) and mouse_pos[1] in range(225, 322):
                            run = False

                            main_screen = False
                            help = True
                mn_images(self.window)
                button(self.window, mouse_pos)
                pygame.display.flip()
            while help:
                for event in pygame.event.get():
                    mouse_pos = pygame.mouse.get_pos()
                    if event.type == pygame.QUIT:
                        help = False
                        main_loop = False
                    if event.type == pygame.MOUSEBUTTONDOWN:
                        if mouse_pos[0] in range(364, 533) and mouse_pos[1] in range(466, 529):

                            main_screen = True
                            help = False

                Help(self.window)
                Help.inst(self, self.window)
                Help.help_button(self, self.window)
                pygame.display.update()
            ###########################################################################################################

            while run:
                global text_timer_text
                text_timer_text = "Match starts in {} sec".format(timer)
                pygame.time.delay(10)
                for event in pygame.event.get():
                    mouse_pos = pygame.mouse.get_pos()
                    if event.type == pygame.QUIT:
                        run = False
                        main_loop = False
                    if event.type == pygame.MOUSEBUTTONDOWN:
                        if mouse_pos[0] in range(11, 106) and mouse_pos[1] in range(17, 49):
                            run_game = False

                            #
                global velBall, vel
                if timer!=0:
                    timer-=1
                    time.sleep(1)
                elif timer<1:
                    run_game = True
                if run_game == True:
                    for event in pygame.event.get():
                        if event.type == pygame.QUIT:
                            run = False
                    keys = pygame.key.get_pressed()
                    if keys[pygame.K_DOWN] and yPlayer < screen_height - (heightObject + 10):
                        yPlayer+= vel
                    if keys[pygame.K_UP] and yPlayer > 10:
                        yPlayer-= vel

                    global began_motion, xBall, yBall, yComp, moveBall, check_lose, score, level_define, level
                    # Player hit ball
                    if xPlayer + 40 >= xBall and xPlayer + 10 <=xBall:
                        if yBall in range(int(yPlayer - 10), int(yPlayer + 80)):
                            xBall_random = velBall
                    if began_motion == True:
                        xBall_random = random.choice(velBall_list)
                        yBall_random = random.choice(velBall_list)
                        pygame.time.delay(100)
                        moveBall = True

                    if moveBall == True:
                        began_motion = False
                        if yBall < screen_height - 20 and yBall > 20:
                            xBall+= xBall_random
                            yBall+= yBall_random
                        else:
                            if yBall_random < 0:
                                yBall_random -= yBall_random * 2

                            elif yBall_random > 0:
                                yBall_random -= yBall_random * 2

                            yBall += yBall_random
                            xBall += xBall_random
                    ##########################################################################
                    if check_lose == True:
                        random_win_lose = random.choice(stop_attack_list)
                        check_lose = False
                    ##########################################################################
                    if xBall > screen_width/2:

                        if stop_attack == random_win_lose:
                            if yBall - 20 > yComp and xBall< screen_width - 30:
                                if yComp + heightObject + 10 <= screen_height:
                                    yComp+= vel
                            elif yBall < yComp:
                                yComp-= vel
                        else:
                            if yBall - 90 > yComp and xBall < screen_width - 30:
                                if yComp + heightObject + 10 <= screen_height:
                                    yComp += vel
                            elif yBall < yComp - 10:
                                yComp -= vel

                        if xBall >= xComp and xBall < xComp+ 30:
                            if yBall in range( int(yComp -2), int(yComp + 80)):
                                xBall_random = -velBall
                                check_lose = True

                    if xBall < -30:
                        score = 0
                        level_define = 1
                        vel = 5
                        velBall = 3
                        pygame.time.delay(100)
                        xBall, yBall = screen_width / 2 - 10, screen_height / 2
                        timer = 5
                        yPlayer, yComp = screen_height/2, screen_height/2
                        text_timer_text = "Match starts in {} sec".format(timer)
                        run_game = False

                    elif xBall > screen_width + 30:
                        check_lose = True
                        score+= 100
                        if score% 200== 0:
                            level_define+=1
                            vel+= 1
                            velBall+=1
                            stop_attack_list.append(True)
                        pygame.time.delay(1000)
                        xBall, yBall = screen_width / 2 - 10, screen_height / 2

                # self.window.blit()
                self.window.fill((0,0,0))
                pygame.draw.rect(self.window, (250,250,250), (screen_width/2-20, 0, 20, screen_height))
                #### Player ####
                Player(self.window, xPlayer, yPlayer)
                #### Comp ####
                Comp(self.window, xComp, yComp)
                #### Ball ####
                Ball(self.window, xBall, yBall)
                ####     Score     ####
                text = font.render('Score: {}'.format(score), True, (200, 0, 70))
                textRect = text.get_rect()
                textRect.center = (screen_width / 2 - 150, 18)
                self.window.blit(text, textRect)
                #### Level ####
                text_lvl = font.render('Level: {}'.format(level_define), True, (0,100, 250))
                self.window.blit(text_lvl, (screen_width/2 + 100, 3))
                #### Timer ####
                if timer==0:
                    text_timer_text = ""
                self.text_timer = font.render(text_timer_text, True, (51, 168, 52))
                self.window.blit(self.text_timer, (screen_width / 2- 150, screen_height - 50))

                pygame.display.update()

if __name__ == "__main__":
    game()


num = int(input("Enter the number: "))
str_num = str(num)
dig = len(str_num)
digits = 0

for i in str_num:
    digits = int(i)** dig

if digits == num:
    print(num,"it is a Armstrong")
else:
    print(num,"not a Armstrong")


class Solution:
  def add(self,s: str) -> str:
    ans: int = 0
    for i in s:
      ans += int(i)
    return str(ans)

  def divide(self,s: str,k: int) -> List[str]:
    l = []
    for i in range(0,len(s),k):
      if i + k <= len(s):
        l.append(self.add(s[i:i + k]))
      else:
        l.append(self.add(s[i:]))
    return "".join(l)
  def digitSum(self, s: str, k: int) -> str:
    if len(s) <= k:
      return s
    ans = "".join(self.divide(s,k))
    while len(ans) > k:
      ans = self.divide(ans,k)
    return ans


import os
import re
from html.parser import HTMLParser
from collections import defaultdict
import numpy as np
from google.cloud import storage
from google.oauth2 import service_account
import json
import time
import concurrent.futures

f = open('ds-561-am-cc3cb4be64ff.json')
creds = json.load(f)

credentials = service_account.Credentials.from_service_account_info(creds)

def read_last_run_time(types,filename="logs.json"):
    try:
        with open(filename, "r") as f:
            data = f.read()
            if not data:
                return None  # File is empty
            data_json = json.loads(data)
            return data_json.get(types + '_time', None)
    except (FileNotFoundError, json.JSONDecodeError):
        return None  # File not found or invalid JSON

def write_run_info(elapsed_upload_time,elapsed_processing_time, incoming_stats, outgoing_stats, top_5_counts, filename="logs.json"):
    with open(filename, "w") as f:
        data = {'upload_time': elapsed_upload_time}
        data['processing_time'] = elapsed_processing_time
        # incoming_stats["Quintiles"] = incoming_stats["Quintiles"].tolist()
        # outgoing_stats["Quintiles"] = outgoing_stats["Quintiles"].tolist()
        # data['incoming_stats'] = incoming_stats
        # data['outgoing_stats'] = outgoing_stats
        # data['top_5_pages'] = top_5_counts
        json.dump(data, f)

def upload_blob(bucket_name, source_file_name, destination_blob_name):
    """Uploads a file to the bucket."""
    storage_client = storage.Client.create_anonymous_client()
    bucket = storage_client.get_bucket(bucket_name)
    blob = bucket.blob(destination_blob_name)
    blob.upload_from_filename(source_file_name)
    print(f"File {source_file_name} uploaded to {destination_blob_name}.")

def upload_folder_to_gcs(bucket_name, folder_path):
    storage_client = storage.Client.create_anonymous_client()
    bucket = storage_client.get_bucket(bucket_name)

    count = 0

    last_run_time = read_last_run_time("upload","logs.json")

    if last_run_time is not None:
        print(f"Please note, this could take some time. Last run took {last_run_time} seconds.")

    start_time = time.time()

    for root, _, files in os.walk(folder_path):
        for file in files:
            local_file = os.path.join(root, file)
            remote_path = os.path.relpath(local_file, folder_path)

            # Check if blob already exists
            blob = bucket.blob('files/'+remote_path)
            if not blob.exists():
                blob.upload_from_filename(local_file)
                count+=1
            else:
                break

    end_time = time.time()
    elapsed_time = end_time - start_time

    write_run_info(elapsed_time,read_last_run_time('processing'), {}, {}, {})

    print(f"{count} files uploaded")

# Custom parser to extract links
class LinkExtractor(HTMLParser):
    def __init__(self):
        super().__init__()
        self.links = []

    def handle_starttag(self, tag, attrs):
        if tag == "a":
            for name, value in attrs:
                if name == "href":
                    self.links.append(value)

def pagerank(graph, max_iter=10, damping=0.85, tol=0.005):
    num_pages = len(graph)
    pr = {page: 1/num_pages for page in graph}
    base_pr = (1 - damping) / num_pages

    for _ in range(max_iter):
        next_pr = {page: base_pr for page in graph}
        for page, outgoing_links in graph.items():
            if not outgoing_links:
                continue  # Skip this page because it has no outgoing links

            share_pr = pr[page] / len(outgoing_links)
            for link in outgoing_links:
                if link not in next_pr:
                    next_pr[link] = base_pr
                next_pr[link] += share_pr * damping

        # Check for convergence
        diff = sum(abs(next_pr[page] - pr[page]) for page in graph)
        if diff < tol:
            break

        pr = next_pr

    return pr

def process_blob_data(blob):
    """Processes a blob and extracts links."""
    filename = blob.name.replace("files/", "").replace(".html", "")
    parser = LinkExtractor()

    with blob.open("r") as f:
        parser.feed(f.read())

    links = {re.sub(r"\.html$", "", link) for link in parser.links}
    return filename, links

def compute_stats(link_counts):
    return {
        "Average": np.mean(link_counts),
        "Median": np.median(link_counts),
        "Max": max(link_counts),
        "Min": min(link_counts),
        "Quintiles": np.percentile(link_counts, [20, 40, 60, 80])
    }

def main():
    graph = defaultdict(set)
    out_count = defaultdict(int)

    storage_client = storage.Client.create_anonymous_client()
    bucket_name = "hw-2-files-bucket"

    folder_path = "./files"

    upload_folder_to_gcs(bucket_name, folder_path)

    last_run_time = read_last_run_time("processing","logs.json")

    if last_run_time is not None:
        print(f"Please note, this could take some time. Last web run took {last_run_time} seconds.")

    start_time = time.time()

    bucket = storage_client.bucket(bucket_name)
    blobs = list(bucket.list_blobs())

    out_count = defaultdict(int)

    BATCH_SIZE = 100
    total_blobs = len(blobs)

    for i in range(0, total_blobs, BATCH_SIZE):
        batch = blobs[i:i+BATCH_SIZE]

        with concurrent.futures.ThreadPoolExecutor() as executor:
            results = list(executor.map(process_blob_data, batch))

            for filename, links in results:
                graph[filename] = links
                out_count[filename] = len(links)

    in_count = [len(graph[filename]) for filename in graph]
    out_count = [out_count[filename] for filename in graph]

    print("Incoming links stats:", compute_stats(in_count))
    print("Outgoing links stats:", compute_stats(out_count))

    pr = pagerank(graph)
    top_5_pages = sorted(pr, key=pr.get, reverse=True)[:5]
    print("Top 5 pages by PageRank:", top_5_pages)

    end_time = time.time()

    write_run_info(read_last_run_time('upload'), end_time, compute_stats(in_count), compute_stats(out_count), top_5_pages)

if __name__ == "__main__":
    main()


#!/usr/bin/env python

"""
Usage example:
1. First realization per model
./parallel_driver.py --mip cmip6 --exp historical --case_id v20200305 --modnames all --realization r1i1p1f1 --metricsCollection ENSO_perf
2. All realizations of individual models
./parallel_driver.py --mip cmip6 --exp historical --case_id v20200305 --modnames all --realization all --metricsCollection ENSO_perf

"""

from __future__ import print_function
from subprocess import Popen

from PMPdriver_lib import AddParserArgument
from PMPdriver_lib import sort_human

import datetime
import json
import os
import sys
import time

from os import makedirs as OS__makedirs
from os.path import exists as OSpath__exists
from os.path import join as OSpath__join

# To avoid below error
# OpenBLAS blas_thread_init: pthread_create failed for thread XX of 96: Resource temporarily unavailable
os.environ['OPENBLAS_NUM_THREADS'] = '1'

# Must be done before any CDAT library is called.
# https://github.com/CDAT/cdat/issues/2213
if 'UVCDAT_ANONYMOUS_LOG' not in os.environ:
    os.environ['UVCDAT_ANONYMOUS_LOG'] = 'no'

# =================================================
# Collect user defined options
# -------------------------------------------------
param = AddParserArgument()

# Metrics Collection
metric_collection = param.metricsCollection

# Pre-defined options
mip = param.mip
exp = param.exp
print('mip:', mip)
print('exp:', exp)

# Check given model option
models = param.modnames
print('models:', models)

# Realizations
realization = param.realization
if ('all' in [r.lower() for r in realization]) or (realization == 'all'):
    realization = 'all'
print('realization: ', realization)

# case id
case_id = param.case_id
print('case_id:', case_id)

path_main = "/p/user_pub/pmp/pmp_results/pmp_v1.1.2"
path_in_json = OSpath__join(path_main, "metrics_results", "enso_metric", mip, exp, case_id, metric_collection)
path_out = OSpath__join(path_main, "graphics", "enso_metric", mip, exp, case_id, metric_collection)

pattern = "_".join([mip, exp, metric_collection, case_id])

# ---------------------------------------------------#
# Adjust model list if "ALL" given
# ---------------------------------------------------#
# read json file
filename_js = OSpath__join(path_in_json, pattern + "_allModels_allRuns.json")
print('filename_js:', filename_js)
with open(filename_js) as ff:
    data_json = json.load(ff)['RESULTS']['model']

# Include all models if conditioned
if ('all' in [m.lower() for m in models]) or (models == 'all'):
    models = sort_human(list(data_json.keys()))

print('models:', models)
print('number of models:', len(models))

# Debug
debug = param.debug
print('debug:', debug)

# =================================================
# Create output directories
# -------------------------------------------------
print("path_out:", path_out)
if not OSpath__exists(path_out):
    try:
        OS__makedirs(path_out)
    except:
        pass

# =================================================
# Generates list of command
# e.g.: python PMPdriver_plot.py --mip cmip5 --exp historical --metricsCollection ENSO_perf --modnames IPSL-CM5A-LR --realization r1i1p1 --case_id v20200305
# -------------------------------------------------
cmds_list = []
for model in models:
    print(' ----- model: ', model, ' ---------------------')
    if realization is "all":
        runs_list = sort_human(list(data_json[model].keys()))
        print('runs_list (all):', runs_list)
    else:
        runs_list = [realization]
    for run in runs_list:
        cmd = ['python', 'PMPdriver_plot.py',
               '--mip', mip, '--exp', exp, '--metricsCollection', metric_collection,
               '--case_id', case_id,
               '--modnames', model,
               '--realization', run]
        cmds_list.append(cmd)

for i, cmd in enumerate(cmds_list):
    print(i+1, ' '.join(cmd))

# =================================================
# Run subprocesses in parallel
# -------------------------------------------------
# log dir
log_dir = os.path.join("log", case_id, metric_collection)

if not os.path.exists(log_dir):
    os.makedirs(log_dir)

# number of tasks to submit at the same time
#num_workers = 8
num_workers = 10
#num_workers = 30
#num_workers = 25

print("Start : %s" % time.ctime())

# submit tasks and wait for subset of tasks to complete
procs_list = []
for p, cmd in enumerate(cmds_list):
    timenow = time.ctime()
    print(timenow, p, ' '.join(cmd))
    model = cmd[-3]
    run = cmd[-1]
    log_filename = '_'.join(['log_ensoPlot', metric_collection, mip, exp, model, run, case_id])
    log_file = os.path.join(log_dir, log_filename)
    with open(log_file+"_stdout.txt", "wb") as out, open(log_file+"_stderr.txt", "wb") as err:
        procs_list.append(Popen(cmd, stdout=out, stderr=err))
        time.sleep(1)
    if ((p > 0 and p % num_workers == 0) or (p == len(cmds_list)-1)):
        print('wait...')
        for proc in procs_list:
            proc.wait()
        print("Tasks end : %s" % time.ctime())
        procs_list = []

# tasks done
print("End : %s" % time.ctime())
sys.exit('DONE')


import pandas as pd

# 读取数据
data = pd.read_csv('data/training_data.csv')  # 请替换 'your_data_file.csv' 为你的数据文件路径

# 提取縣市、鄉鎮市區和路名列
counties = data['縣市'].unique()
towns = data['鄉鎮市區'].unique()
roads = data['路名'].unique()

# 创建字典来存储縣市、鄉鎮市區和路名的编号
county_dict = {county: i for i, county in enumerate(counties, start=1)}
town_dict = {town: i for i, town in enumerate(towns, start=1)}
road_dict = {road: i for i, road in enumerate(roads, start=1)}

# 将编号添加到数据中
data['縣市編號'] = data['縣市'].map(county_dict)
data['鄉鎮市區編號'] = data['鄉鎮市區'].map(town_dict)
data['路名編號'] = data['路名'].map(road_dict)

# 打印前几行数据，包含编号
print(data[['縣市', '縣市編號', '鄉鎮市區', '鄉鎮市區編號', '路名', '路名編號']].head())

# 可以根据需要保存修改后的数据
data.to_csv('data_with_id.csv', index=False)


from nltk.stem.porter import PorterStemmer
porterStemmer = PorterStemmer()
t = [
"boys",
"boy",
"toys",
"toy",
"noises",
"noise",
"noised",
"noising",
"voices",
"voice",
"voiced",
"voicing",
"cry",
"cried",
"cries",
"tried",
"try",
"tries",
"arrive",
"arrival",
"arrived",
"arriving",
"arriver",
"said",
"say",
"meaning",
'heard',
'hear',
'heart',
'survive',
'survived',
'surviving',
'survival',
'survivor',
'trival',
'semantics',
'analysis',
'division',
'owing',
'owner',
'seeing',
'timeframe',
'longitudinal',
'latitudinal',
'mare',
'children',
'helper',
'lamented',
'lamenting',
'am',
'were',
'being',
'I',
'me',
'paste',
'his',
'was',
'has',
'gas',
'face',
'faces',
'less',
'loss',
'lose',
'lost',
'her',
'sing',
'flying',
'xerox',
'politics',
'polite',
'flight',
'pros',
'cons',
'mess',
'maze',
'is',
'does',
'aid',
'made',
'make',
'might',
'may',
'could',
'can',
'would',
'will',
'him']

print('word -> porter stemmerd word')

for num in range(0,len(t)):

	print (t[num]+' -> '+porterStemmer.stem(t[num]))


class CustomIterator:
    def __init__(self, sequence: list, start_index: int, end_index: int):
        self.__sequence = sequence
        self.__start_index = start_index
        self.__end_index = end_index

    def __iter__(self):
        return self

    def __next__(self):
        if self.__start_index > self.__end_index:
            raise Exception('start index cannot be more than end index')
        if self.__start_index < 0:
            raise Exception('start index cannot be less than 0')
        if self.__end_index > len(self.__sequence):
            self.__end_index = len(self.__sequence)
        if self.__start_index > len(self.__sequence):
            raise Exception('start index out of range')

        if self.__start_index < self.__end_index:
            item = self.__sequence[self.__start_index]
            self.__start_index += 1
            return item
        else:
            raise StopIteration

if __name__ == '__main__':
    test_list = [13, 21, 25, 123, 3124, 124, 222, 151, 21, 412, 4]
    test = CustomIterator(test_list, 0, 100)
    for element in test:
        print(element)


from model.service.external_requests import execute_order
from model.service.helpers import EXECUTION_APP_ENDPOINTS
from model.tests.setup.fixtures.internal_modules import *
from shared.utils.tests.fixtures.external_modules import *

class TestModelExternalRequests:
    def test_execute_order(
        self,
        mock_settings_env_vars,
        mock_requests_post,
        mock_redis_connection,
        requests_post_spy
    ):
        """
        GIVEN some params
        WHEN the method execute_order is called
        THEN the return value is equal to the expected response

        """

        bearer_token = "abc"

        params = {
            "pipeline_id": 1,
            "signal": 1,
        }

        res = execute_order(bearer_token=bearer_token, **params)

        assert res == response
        requests_post_spy.assert_called_with(
            EXECUTION_APP_ENDPOINTS["EXECUTE_ORDER"](
                os.getenv("EXECUTION_APP_URL")
            ),
            json=params,
            headers={"Authorization": bearer_token}
        )


from application import app
from application import db
from application.models import Quantity, Method, Recipes, Cuisine, Ingredients, Schedule, Measure
from sqlalchemy import inspect

app.app_context().push()

inspector = inspect(db.engine)
if inspector.has_table == True:
    pass
else:
    db.create_all()

if Recipes.query.filter_by(recipe_name = 'Pea and Cauliflower Curry').first():
    pass
else:

    cuisines = ('Indian', 'Salad', 'Pasta', 'Various')
    for cus in cuisines:
        i = Cuisine(cus)
        db.session.add(i)
        db.session.commit()

    ingredients = ['sunflower oil', 'cumin seeds', 'brown mustard seeds', 'asafetida', 'cauliflower', 'turmeric', 'ground coriander', 'salt', 'sugar', 'frozen peas','medium tomato', 'fresh coriander', 'fresh green chillies','root ginger']
    for ing in ingredients:
        i = Ingredients(ing)
        db.session.add(i)
        db.session.commit()

    measures = ["ml", "g", "tsp", "tbsp", "whole", "large", "medium", "small", "cloves", "cm", "handful", "cup"]
    for m in measures:
        measure = Measure(m)
        db.session.add(measure)
        db.session.commit()

    rec = Recipes('Pea and Cauliflower Curry', 'quick and easy curry, no calorie details', 1)
    db.session.add(rec)
    db.session.commit()

    quantities = [ (1, 1,"",100.0,1),(1, 2,"", 1.0, 3), (1, 3,"", 1.0, 3), (1, 4,"", 0.5, 3), (1, 5,'cut into 1cm pieces', 1.0, 7), (1, 6,"", 1.0, 3) , (1, 7,"", 1.0, 4), (1, 8,"", 1.5, 3), (1, 9,"", 1.0, 3), (1, 10,"", 400.0, 2), (1, 11,"", 1.0, 7), (1, 12, 'roughly chopped', 1.0, 11), (1, 13,"", 4.0, 5),(1, 14, 'peeled and roughly chopped', 4.0, 10)]
    for q in quantities:
        quant = Quantity(*q)
        db.session.add(quant)
        db.session.commit()

    methods = [(1, 'Crush the chillies and ginger together with a pinch of salt to make a fine masala paste.\n Heat the oil in non-stick pan over a medium heat for 30s, then add the cumin and mustard seeds.\n When the mustard seeds start to pop reduce the heat and stir in the asafetida.\n Add the cauliflower and return the heat to medium.\n Stir in the masala paste, turmeric, ground coriander, salt, and sugar.\n Cover the pan and leave to cook for 8-10 minutes, stirring every few minutes.\n Stir in the peas and tomato and cook for a further 3-5 minutes.\n Remove from the heat and sprinkle with the chopped coriander.\n Leave for 5 minutes to let flavours develop')]
    for m in methods:
        met = Method(*m)
        db.session.add(met)
        db.session.commit()

    schedules =[(1,1,1,1), (2,1,1,1), (3,1,1,1), (4,1,1,1), (0,1,1,1)]
    for s in schedules:
        sched = Schedule(*s)
        db.session.add(sched)
        db.session.commit()

if __name__ == "__main__":
    app.run(debug=True, host='0.0.0.0', port=5050)


import json
from http import HTTPStatus

import dash
import dash_table
import pandas as pd
import requests
from dash.dependencies import Output, Input, State
from dash.exceptions import PreventUpdate

import common
from common import app

def get_distinct_field_values(field_name):
    """Get distinct values from database for dropdowns with no filter in query.
    Used only while retrieving initial values for dropdown
    """
    query_input = {"field": field_name, "query": {"latest": True}}
    query_input.update(common.credentials)
    response = requests.request("GET", common.distinct_endpoint, headers=common.headers,
                                data=json.dumps(query_input))

    if response.status_code == HTTPStatus.OK:
        json_response = json.loads(response.text)
        field_values = json_response["result"]
        output = [
            {'label': val, 'value': val} for val in field_values if val.strip() != ""
        ]
        if common.DEBUG_PRINTS:
            print("Fetch {} value  for dropdown : {}".format(field_name, output))
        return output
    elif response.status_code == HTTPStatus.NOT_FOUND:
        print(f"(get_distinct_field_values) No data found for field:{field_name}")
    else:
        print(
            f"(get_distinct_field_values) Error response {response.status_code} for field:{field_name} ")
    return None

# input fields and corresponding database fields
db_field_mapping = {"query_branch.value": "buildType", "query_build.value": "buildNo",
                    "query_system_type.value": "testPlanLabel",
                    "query_feature.value": "feature",
                    "query_test_plan.value": "testPlanID",
                    "query_test_execution.value": "testExecutionID",
                    "query_test_id.value": "testID",
                    "query_execution_type.value": "executionType",
                    "query_test_results.value": "testResult"}

# Display Names for table in output.
column_names = {"buildType": "Build Type", "buildNo": "Build No", "testPlanID": "Test Plan ID",
                "testPlanLabel": "Test Plan Labels",
                "testExecutionID": "Test Execution ID",
                "testExecutionLabel": "Test Execution Labels", "testTeam": "Test Team",
                "feature": "Feature",
                "testID": "Test ID", "testName": "Test Name", "testType": "Test Type",
                "testResult": "Test Result", "logPath": "Log Path",
                "healthCheckResult": "Health Check Result",
                "testIDLabels": "Test ID Labels", "testTags": "Test Tags",
                "testStartTime": "Test Start Time", "testExecutionTime": "Test Execution Time",
                "executionType": "Execution Type", "issueIDs": "Issue IDs",
                "nodesHostname": "Node Hostnames", "noOfNodes": "No of Nodes",
                "clientHostname": "Client Hostname"}

@app.callback(
    [Output('table_query_result', 'children'),
     Output('table_query_result_count', 'children')],
    [Input('query_result_button', 'n_clicks'),
     Input('query_branch', 'value'),
     Input('query_build', 'value'),
     Input('query_system_type', 'value'),
     Input('query_feature', 'value'),
     Input('query_test_plan', 'value'),
     Input('query_test_execution', 'value'),
     Input('query_test_id', 'value'),
     Input('query_execution_type', 'value'),
     Input('query_test_results', 'value'),
     Input('query_table_view', 'value')
     ]
)
def retrieve_query_results(n_clicks, *values):
    """
    Function triggered after Get Results button to retrieve the table contents.
    """
    ctx = dash.callback_context
    # update only when get results button is clicked.
    if ctx.triggered[0]['prop_id'] == 'query_result_button.n_clicks':
        query_input = {"latest": True}

        # Check valid inputs given before get result button
        input_found = False
        for input_val in ctx.inputs:
            if ctx.inputs[input_val] not in [None, [], [None]] \
                    and input_val != "query_result_button.n_clicks" \
                    and input_val != "query_table_view.value":
                input_found = True
                break

        if not input_found:
            return "No Inputs Selected!!!"

        query = _form_query_by_inputs(ctx, query_input)
        if common.DEBUG_PRINTS:
            print("Sending Query (retrieve_query_results):{}".format(query))
        query.update(common.credentials)
        response = requests.request("GET", common.search_endpoint, headers=common.headers,
                                    data=json.dumps(query))
        data_from_db = construct_fields_to_display(ctx)

        if response.status_code == HTTPStatus.OK:
            json_response = json.loads(response.text)
            table_data = json_response["result"]

            for document in table_data:
                for db_key in data_from_db.keys():
                    if db_key in document and document[db_key] is not None:
                        data_from_db[db_key].append(document[db_key])
                    else:
                        data_from_db[db_key].append("-")
            no_docs_returned = "Number of Documents returned : " + str(len(table_data))
            df_query_output = pd.DataFrame(data_from_db)

            datatable_query_output = dash_table.DataTable(
                id="temp_table_query_result",
                columns=[{"name": column_names[i], "id": i, "deletable": True}
                         for i in df_query_output.columns],
                data=df_query_output.to_dict('records'),
                filter_action="native",
                page_action='native',
                page_current=0,
                page_size=20,
                export_format="csv",
                export_headers="display",
                style_cell_conditional=[{'if': {'column_id': 'testName'},
                                         'overflow': 'hidden',
                                         'textOverflow': 'ellipsis',
                                         'maxWidth': 350, },
                                        {'if': {'column_id': 'nodesHostname'},
                                         'overflow': 'hidden',
                                         'textOverflow': 'ellipsis',
                                         'maxWidth': 200, },
                                        {'if': {'column_id': 'testStartTime'},
                                         'overflow': 'hidden',
                                         'textOverflow': 'ellipsis',
                                         'maxWidth': 150, },
                                        ],

                style_cell={'textAlign': 'center', 'border': '1px solid black',
                            'font-size': '15px'},
                tooltip_data=[
                    {
                        column: {'value': str(value), 'type': 'markdown'}
                        for column, value in row.items()
                    } for row in df_query_output.to_dict('records')
                ],
                tooltip_duration=None,
                style_header=common.dict_style_header,
                style_header_conditional=[
                    {'whiteSpace': 'normal', 'height': 'auto', 'maxWidth': 150}],
                style_data_conditional=[{'if': {'row_index': 'odd'}, 'backgroundColor': '#F8F8F8'}],
                style_table={
                    'width': '100%',
                    'overflowX': 'scroll',
                    'height': '50%',
                    'overflowY': 'auto'
                }
            )

            return datatable_query_output,no_docs_returned
    return [],None

def construct_fields_to_display(ctx):
    """
    Fields to display based on view is detailed or not.

    :param ctx:
    :return:
    """
    data_from_db = {}
    if ctx.inputs["query_table_view.value"] == "detail":
        data_from_db = {"buildType": [], "buildNo": [], "testPlanID": [], "testPlanLabel": [],
                        "testExecutionID": [], "testExecutionLabel": [], "testTeam": [],
                        "feature": [],
                        "testID": [], "testName": [], "testType": [],
                        "testResult": [], "logPath": [], "healthCheckResult": [],
                        "testIDLabels": [], "testTags": [],
                        "testStartTime": [], "testExecutionTime": [],
                        "executionType": [], "issueIDs": [],
                        "nodesHostname": [], "noOfNodes": [], "clientHostname": []}
    else:
        data_from_db = {"buildType": [], "buildNo": [], "testPlanID": [], "testPlanLabel": [],
                        "testExecutionID": [],
                        "testID": [], "testName": [], "testType": [],
                        "testResult": [],
                        "testStartTime": [], "testExecutionTime": [],
                        "executionType": [], "issueIDs": [],
                        "nodesHostname": []}
    return data_from_db

def _form_query_by_inputs(ctx, query_input):
    """
    Form query based on context inputs.

    :param ctx:
    :param query_input:
    :return: query dict
    """
    for input_val in ctx.inputs:
        if ctx.inputs[input_val] not in [None, [], [None]] \
                and input_val != "query_result_button.n_clicks" \
                and input_val != "query_table_view.value":
            if len(ctx.inputs[input_val]) == 0:
                pass
            elif len(ctx.inputs[input_val]) == 1:
                query_input[db_field_mapping[input_val]] = ctx.inputs[input_val][0]
            else:
                temp_list = []
                for i in range(len(ctx.inputs[input_val])):
                    temp_dict = {db_field_mapping[input_val]: ctx.inputs[input_val][i]}
                    temp_list.append(temp_dict)
                query_input['$or'] = temp_list
    query = {"query": query_input}
    return query

def build_query(inputs):
    """
    Build query based on the existing state of the the dropdowns in the QA query page
    """
    query_input = {"latest": True}
    for input_val in inputs:
        if inputs[input_val] not in [None, [None], []] \
                and input_val != "query_result_button.n_clicks" \
                and input_val != "query_table_view.value":
            if len(inputs[input_val]) == 0:
                pass
            elif len(inputs[input_val]) == 1:
                query_input[db_field_mapping[input_val]] = inputs[input_val][0]
            else:
                temp_list = []
                for i in range(len(inputs[input_val])):
                    temp_dict = {db_field_mapping[input_val]: inputs[input_val][i]}
                    temp_list.append(temp_dict)
                query_input['$or'] = temp_list
    if common.DEBUG_PRINTS:
        print("(query_tab_callbacks)Build query output : {}".format(query_input))
    return query_input

@app.callback(
    [Output('query_branch', 'options')],
    [
        Input('query_build', 'value'),
        Input('query_system_type', 'value'),
        Input('query_feature', 'value'),
        Input('query_test_plan', 'value'),
        Input('query_test_execution', 'value'),
        Input('query_test_id', 'value'),
        Input('query_execution_type', 'value')
    ],
    [State('query_branch', 'value')]
)
def retrieve_query_branch(build, system_type, feature, test_plan, test_execution, test_id,
                          execution_type, current_val):
    """
    Updates Branch values dynamically in dropdown based on other entered value
    """
    # pylint: disable=unused-variable

    if current_val not in [None, [None], []]:
        raise PreventUpdate

    # form query based on inputs
    ctx = dash.callback_context
    query_input = build_query(ctx.inputs)
    query = {"field": "buildType", "query": query_input}
    query.update(common.credentials)
    if common.DEBUG_PRINTS:
        print("Sending Query (retrieve_query_branch):{}".format(query))
    response = requests.request("GET", common.distinct_endpoint, headers=common.headers,
                                data=json.dumps(query))
    if response.status_code == HTTPStatus.OK:
        json_response = json.loads(response.text)
        branches = json_response["result"]
        options = [
            {'label': branch, 'value': branch} for branch in branches
        ]
        return [options]
    else:
        print("Response (retrieve_query_branch):{}".format(response.status_code))
        return []

@app.callback(
    [Output('query_build', 'options')],
    [
        Input('query_branch', 'value'),
        Input('query_system_type', 'value'),
        Input('query_feature', 'value'),
        Input('query_test_plan', 'value'),
        Input('query_test_execution', 'value'),
        Input('query_test_id', 'value'),
        Input('query_execution_type', 'value')
    ],
    [State('query_build', 'value')]
)
def retrieve_query_build(branch, system_type, feature, test_plan, test_execution, test_id,
                         execution_type, current_val):
    """
    Updates Build values dynamically in dropdown based on other entered value
    """
    # pylint: disable=unused-variable
    # form query based on inputs
    if current_val not in [None, [None], []]:
        raise PreventUpdate

    ctx = dash.callback_context
    query_input = build_query(ctx.inputs)
    query = {"field": "buildNo", "query": query_input}
    query.update(common.credentials)
    if common.DEBUG_PRINTS:
        print("Sending Query (retrieve_query_build):{}".format(query))
    response = requests.request("GET", common.distinct_endpoint, headers=common.headers,
                                data=json.dumps(query))
    if response.status_code == HTTPStatus.OK:
        json_response = json.loads(response.text)
        results = json_response["result"]
        options = [
            {'label': res_val, 'value': res_val} for res_val in results
        ]
        return [options]
    else:
        print("Response (retrieve_query_build):{}".format(response.status_code))
        return []

@app.callback(
    [Output('query_system_type', 'options')],
    [
        Input('query_branch', 'value'),
        Input('query_build', 'value'),
        Input('query_feature', 'value'),
        Input('query_test_plan', 'value'),
        Input('query_test_execution', 'value'),
        Input('query_test_id', 'value'),
        Input('query_execution_type', 'value')
    ],
    [State('query_system_type', 'value')]
)
def retrieve_query_system_type(branch, build, feature, test_plan, test_execution, test_id,
                               execution_type, current_val):
    """
    Updates System type values dynamically in dropdown based on other entered value
    """
    # pylint: disable=unused-variable
    if current_val not in [None, [None], []]:
        raise PreventUpdate

    # form query based on inputs
    ctx = dash.callback_context
    query_input = build_query(ctx.inputs)
    query = {"field": "testPlanLabel", "query": query_input}
    query.update(common.credentials)
    if common.DEBUG_PRINTS:
        print("Sending Query (retrieve_query_system_type):{}".format(query))
    response = requests.request("GET", common.distinct_endpoint, headers=common.headers,
                                data=json.dumps(query))
    if response.status_code == HTTPStatus.OK:
        json_response = json.loads(response.text)
        results = json_response["result"]
        options = [
            {'label': res_val, 'value': res_val} for res_val in results
        ]
        return [options]
    else:
        print("Response (retrieve_query_system_type):{}".format(response.status_code))
        return []

@app.callback(
    [Output('query_feature', 'options')],
    [
        Input('query_branch', 'value'),
        Input('query_build', 'value'),
        Input('query_system_type', 'value'),
        Input('query_test_plan', 'value'),
        Input('query_test_execution', 'value'),
        Input('query_test_id', 'value'),
        Input('query_execution_type', 'value')
    ],
    [State('query_feature', 'value')]
)
def retrieve_query_feature(branch, build, system_type, test_plan, test_execution, test_id,
                           execution_type, current_val):
    """
    Updates Feature values dynamically in dropdown based on other entered value
    """
    # pylint: disable=unused-variable
    if current_val not in [None, [None], []]:
        raise PreventUpdate

    # form query based on inputs
    ctx = dash.callback_context
    query_input = build_query(ctx.inputs)
    query = {"field": "feature", "query": query_input}
    query.update(common.credentials)
    if common.DEBUG_PRINTS:
        print("Sending Query (retrieve_query_feature):{}".format(query))
    response = requests.request("GET", common.distinct_endpoint, headers=common.headers,
                                data=json.dumps(query))
    if response.status_code == HTTPStatus.OK:
        json_response = json.loads(response.text)
        results = json_response["result"]
        options = [
            {'label': res_val, 'value': res_val} for res_val in results
        ]

        return [options]
    else:
        print("Response (retrieve_query_feature):{}".format(response.status_code))
        return []

@app.callback(
    [Output('query_test_plan', 'options')],
    [
        Input('query_branch', 'value'),
        Input('query_build', 'value'),
        Input('query_system_type', 'value'),
        Input('query_feature', 'value'),
        Input('query_test_execution', 'value'),
        Input('query_test_id', 'value'),
        Input('query_execution_type', 'value')
    ],
    [State('query_test_plan', 'value')]
)
def retrieve_query_test_plan(branch, build, system_type, test_plan, test_execution, test_id,
                             execution_type, current_val):
    """
    Updates Test plan values dynamically in dropdown based on other entered value
    """
    # pylint: disable=unused-variable
    if current_val not in [None, [None], []]:
        raise PreventUpdate

    # form query based on inputs
    ctx = dash.callback_context
    query_input = build_query(ctx.inputs)
    query = {"field": "testPlanID", "query": query_input}
    query.update(common.credentials)
    if common.DEBUG_PRINTS:
        print("Sending Query (retrieve_query_test_plan):{}".format(query))
    response = requests.request("GET", common.distinct_endpoint, headers=common.headers,
                                data=json.dumps(query))
    if response.status_code == HTTPStatus.OK:
        json_response = json.loads(response.text)
        results = json_response["result"]
        options = [
            {'label': res_val, 'value': res_val} for res_val in results
        ]
        return [options]
    else:
        print("Response (retrieve_query_test_plan):{}".format(response.status_code))
        return []

@app.callback(
    [Output('query_test_execution', 'options')],
    [
        Input('query_branch', 'value'),
        Input('query_build', 'value'),
        Input('query_system_type', 'value'),
        Input('query_feature', 'value'),
        Input('query_test_plan', 'value'),
        Input('query_test_id', 'value'),
        Input('query_execution_type', 'value')
    ],
    [State('query_test_execution', 'value')]
)
def retrieve_query_test_execution(branch, build, system_type, feature, test_plan, test_id,
                                  execution_type, current_val):
    """
    Updates Test execution values dynamically in dropdown based on other entered value
    """
    # pylint: disable=unused-variable
    if current_val not in [None, [None], []]:
        raise PreventUpdate

    # form query based on inputs
    ctx = dash.callback_context
    query_input = build_query(ctx.inputs)
    query = {"field": "testExecutionID", "query": query_input}
    query.update(common.credentials)
    if common.DEBUG_PRINTS:
        print("Sending Query (retrieve_query_test_execution):{}".format(query))
    response = requests.request("GET", common.distinct_endpoint, headers=common.headers,
                                data=json.dumps(query))
    if response.status_code == HTTPStatus.OK:
        json_response = json.loads(response.text)
        results = json_response["result"]
        options = [
            {'label': res_val, 'value': res_val} for res_val in results
        ]
        return [options]
    else:
        print("Response (retrieve_query_test_execution):{}".format(response.status_code))
        return []

@app.callback(
    [Output('query_test_id', 'options')],
    [
        Input('query_branch', 'value'),
        Input('query_build', 'value'),
        Input('query_system_type', 'value'),
        Input('query_feature', 'value'),
        Input('query_test_plan', 'value'),
        Input('query_test_execution', 'value'),
        Input('query_execution_type', 'value')
    ],
    [State('query_test_id', 'value')]
)
def retrieve_query_test_id(branch, build, system_type, feature, test_plan, test_execution,
                           execution_type, current_val):
    """
    Updates Test id values dynamically in dropdown based on other entered value
    """
    # pylint: disable=unused-variable
    if current_val not in [None, [None], []]:
        raise PreventUpdate

    # form query based on inputs
    ctx = dash.callback_context
    query_input = build_query(ctx.inputs)
    query = {"field": "testID", "query": query_input}
    query.update(common.credentials)
    if common.DEBUG_PRINTS:
        print("Sending Query (retrieve_query_test_id):{}".format(query))
    response = requests.request("GET", common.distinct_endpoint, headers=common.headers,
                                data=json.dumps(query))
    if response.status_code == HTTPStatus.OK:
        json_response = json.loads(response.text)
        results = json_response["result"]
        options = [
            {'label': res_val, 'value': res_val} for res_val in results
        ]
        return [options]
    else:
        print("Response (retrieve_query_test_id):{}".format(response.status_code))
        return []

@app.callback(
    [Output('query_execution_type', 'options')],
    [
        Input('query_branch', 'value'),
        Input('query_build', 'value'),
        Input('query_system_type', 'value'),
        Input('query_feature', 'value'),
        Input('query_test_plan', 'value'),
        Input('query_test_execution', 'value'),
        Input('query_test_id', 'value')
    ],
    [State('query_execution_type', 'value')]
)
def retrieve_query_execution_type(branch, build, system_type, feature, test_plan, test_execution,
                                  test_id, current_val):
    """
    Updates Execution type values dynamically in dropdown based on other entered value
    """
    # pylint: disable=unused-variable
    if current_val not in [None, [None], []]:
        raise PreventUpdate

    # form query based on inputs
    ctx = dash.callback_context
    query_input = build_query(ctx.inputs)
    query = {"field": "executionType", "query": query_input}
    query.update(common.credentials)
    if common.DEBUG_PRINTS:
        print("Sending Query (retrieve_query_execution_type):{}".format(query))
    response = requests.request("GET", common.distinct_endpoint, headers=common.headers,
                                data=json.dumps(query))
    if response.status_code == HTTPStatus.OK:
        json_response = json.loads(response.text)
        results = json_response["result"]
        options = [
            {'label': res_val, 'value': res_val} for res_val in results
        ]
        return [options]
    else:
        print("Response (retrieve_query_execution_type):{}".format(response.status_code))
        return []


import sys
import logging

import h5py

module_logger = logging.getLogger(__name__)

__all__ = ["inspect_file_attrs", "inspect_file_structure"]

def _get_file_obj(f_path_or_f_obj):
    """
    Get an instance of h5py.File from either an existing instance
    or a path to a HDF5 file.

    Args:
        f_path_or_f_obj (str/h5py.File): Path to HDF5 or h5py.File object
    Returns:
        h5py.File
    """
    if not isinstance(f_path_or_f_obj, h5py.File):
        f_obj = h5py.File(f_path_or_f_obj, "r")
    else:
        f_obj = f_path_or_f_obj
    return f_obj

def inspect_file_attrs(f_path_or_f_obj):
    """
    List out the attributes of some HDF5 file object.
    If a path is provided, then it will open the file in
    read mode.

    Examples:

    .. code-block:: python

        import h5py

        from support.hdf5_util import inspect_file_attrs

        file_path = "example.hdf5"
        with h5py.File(file_path, "w") as f:
            f.attrs["first_name"] = "dill"
            f.attrs["last_name"] = "pickle"
            f.attrs["age"] = 0.1

        inspect_file_attrs(file_path).close()

    This will output the following:

    .. code-block:: none

        first_name: "dill"
        last_name: "pickle"
        age: 0.1

    """
    f_obj = _get_file_obj(f_path_or_f_obj)
    for attr_name in f_obj.attrs:
        print("{}: {}".format(attr_name, f_obj.attrs[attr_name]))

    return f_obj

def inspect_file_structure(f_path_or_f_obj):
    """
    Get a dictionary that reports the structure of some HDF5 file.

    Examples:

    .. code-block:: python

        import numpy as np
        import h5py

        from support.hdf5_util import inspect_file_structure

        file_path = "example.hdf5"
        with h5py.File(file_path, "w") as f:
            g = f.create_group("group0")
            g.create_dataset("dataset0",data=np.zeros(10))

        report, f_obj = inspect_file_structure(file_path)
        f_obj.close
        print(report)

    This outputs the following dictionary:

    .. code-block:: none

        {u'group0': {u'dataset0': (10,)}}

    """
    report = {}

    def _inspect_file_structure(grp, report):
        if hasattr(grp, "keys"):
            for key in grp:
                if isinstance(grp[key], h5py.Group):
                    report[key] = {}
                    _inspect_file_structure(grp[key], report[key])
                else:
                    report[key] = grp[key].shape

    f_obj = _get_file_obj(f_path_or_f_obj)
    _inspect_file_structure(f_obj, report)
    return report, f_obj

if __name__ == "__main__":
    inspect_file_attrs(sys.argv[1]).close()


import sys
import re
s=sys.stdin.read()
slist=s.split("\n")
digits=re.findall(r"\d+",slist[0])
arr1= [int(e) for e in digits ]
digits2=re.findall(r"\d+",slist[1])
arr2= [int(e) for e in digits2 ]
ans=[]
isFind=[0]*len(arr1)
left=[]
for i in range( len(arr2)):
    for j in range( len(arr1)):
        if arr2[i]==arr1[j]:
            ans.append(arr2[i])
            isFind[j]=1
for i in range(len(arr1)):
    if isFind[i]==0:
        left.append(arr1[i])
left.sort()
for e in left:
    ans.append(e)
print(ans)


import sys
sys.path.append('../src')

import time
from math import sin

from panda import Panda
import numpy as np

duration = 30
stepsize = 1e-3

robot = Panda(stepsize)
robot.setControlMode("torque")

j_trajs = np.load('../franka_panda_insertion_logs/experts/expert_joints_poses.npy', allow_pickle=True)
joints = np.vstack(j_trajs)
jmax = np.amax(joints, axis=0)
jmin = np.amin(joints, axis=0)
jmean = (jmax+jmin)/2.

robot.reset()
robot.setControlMode("torque")

for _ in range(10000):
    for j in joints:
        j = j + np.random.uniform(-1,1, size=jmean.shape)*(jmax - jmean)

        target_torque = j

        robot.step(target_torque)


import torch

MODEL_REGISTRY = {}

def register_model(model_name):
    def decorator(f):
        MODEL_REGISTRY[model_name] = f
        return f

    return decorator

def get_model(model_opt):
    model_name = model_opt['model_name']
    del model_opt['model_name']
    if model_name in MODEL_REGISTRY:
        return MODEL_REGISTRY[model_name](**model_opt)
    else:
        raise ValueError("Unknown model {:s}".format(model_name))

def euclidean_dist(x, y):
    # x: N x D
    # y: M x D
    n = x.size(0)
    m = y.size(0)
    d = x.size(1)
    assert d == y.size(1)

    x = x.unsqueeze(1).expand(n, m, d)
    y = y.unsqueeze(0).expand(n, m, d)

    return torch.pow(x - y, 2).sum(2)


import jieba

def stopwordslist(filepath):
    stopwords = [line.strip() for line in open(
        filepath, 'r', encoding='utf-8').readlines()]
    return stopwords

def init_jieba():
    jieba.enable_parallel(3)
    jieba.load_userdict("ingredients/userdict.txt")

def cutTrainData():
    input_filename = 'ingredients/train.data'
    output_filename = 'train.csv'
    print("加载原始数据...", end='')
    file = open(input_filename)
    sentence = file.read()
    print("完成")
    print("加载停用词表...", end='')
    stopwords = set(stopwordslist("ingredients/stopwords.txt"))
    print("完成")

    seg_list = jieba.cut(sentence)

    print("分词完成，正在写入文件 %s" % (output_filename))
    storefile = open(output_filename, mode='w')
    storetext = ''
    for x in seg_list:
        if x == '\n':
            storetext += x
        # 这里 if 的判断条件不可调换，因为对于 '\n', isspace 也会返回true
        elif x.isspace() or (x in stopwords) or x.isdigit() or x.isnumeric():
            continue
        else:
            storetext += x + ' '
    storefile.write(storetext)

    storefile.close()
    file.close()
    print("分词完成")

def cutTestData():
    input_filename = 'ingredients/test.data'
    output_filename = 'test.csv'
    print("加载原始数据...", end='')

    file = open(input_filename)
    sentence = ''
    # sentence = file.read()
    for line in file:
        sentence += line.split(maxsplit=1)[1]
    print("完成")

    print("加载停用词表...", end='')
    stopwords = stopwordslist("ingredients/stopwords.txt")
    print("完成")

    seg_list = jieba.cut(sentence)
    print("分词完成，正在写入文件 %s" % (output_filename))
    storefile = open(output_filename, mode='w')
    storetext = ''
    for x in seg_list:
        if x == '\n':
            storetext += x
        # 这里 if 的判断条件不可调换，因为对于 '\n', isspace 也会返回true
        elif x.isspace() or (x in stopwords) or x.isdigit() or x.isnumeric():
            continue
        else:
            storetext += x + ' '

    storefile.write(storetext)
    storefile.close()
    file.close()
    print("分词完成")

def main():
    # 加载用户 词语
    init_jieba()
    # 对训练集和测试集数据切分
    cutTestData()
    cutTrainData()

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
"""
Created on Tue Jul 26 08:12:55 2022

@author: Alumno
"""

var1=int(input("ingrese su edad:"))

pulsaciones=(200-var1)/10

print("sus pulsaciones cada 10 segundos de ejecicioes es:",pulsaciones)


# Blender tracking export to CSV
# Python script originally taken from:
# http://scummos.blogspot.cz/2012/11/blender-exporting-camera-tracking.html
# https://gist.github.com/anonymous/31c915d611f0a84e5d33
# Last edited by David Wei, University of Manchester, August 2016
import bpy
import csv
D = bpy.data

frameNums = True # include frame numbers in the csv file
relativeCoords = False # marker coords will be relative to the dimensions of the clip
filepath = "filename here"
markers = {}

for clip in D.movieclips:
    print('Clip {} found'.format(clip.name))
    if relativeCoords:
        width = 1
        height = 1
    else:
        width = clip.size[0]
        height = clip.size[1]
    for ob in clip.tracking.objects:
        print('Object {} found'.format(ob.name))
        for track in ob.tracks:
            fn = '{}_{}_tr_{}'.format(clip.name.split('.')[0], ob.name, track.name)
            markers[fn] = []
            print('track {} found'.format(track.name))
            for framenum in range(clip.frame_duration):
                markerAtFrame = track.markers.find_frame(framenum)
                if markerAtFrame:
                    coords = list(markerAtFrame.co.xy)
                    coords = [coords[0] * width, coords[1] * height]
                    markers[fn].append(coords)

for key, value in markers.items():

    print(key)
    filename = "{}{}{}".format(filepath, key, ".csv")
    open_data = open(filename, 'w', newline='')
    with open_data as writer:
        writer = csv.writer(writer, delimiter=',', quotechar='"', quoting=csv.QUOTE_NONNUMERIC)
        if frameNums:
# writer.writerow([key, "x", "y"])
            for i, data in enumerate(value):
                writer.writerow([i] + data)
        else:
# writer.writerow(["x", "y"])
            for data in value:
                writer.writerow(data)


import os, sys

dir_path = os.path.dirname(os.path.realpath(__file__))
parent_dir_path = os.path.abspath(os.path.join(dir_path, os.pardir))
sys.path.insert(0, parent_dir_path)

from Ziggeo import Ziggeo

if(len(sys.argv) < 3):
	print("Error\n")
	print("Usage: $>python videos_index.py YOUR_APP_TOKEN YOUR_PRIVATE_KEY\n")
	sys.exit()

app_token = sys.argv[1]
private_key = sys.argv[2]

ziggeo = Ziggeo(app_token, private_key)
# 'limit' will limit how much index operation will fetch the videos. default 50 max 100

video_list = ziggeo.videos().index({"limit":100})
for video in video_list:
	print(video)


# def scrape_careers_page():
#     import requests
#     from bs4 import BeautifulSoup

#     url = "https://careers.microsoft.com/us/en/search-results?keywords=&from=0&to=100&sort_by=relevance&category=&location=&job_function=&brand=&language=&"
#     page = requests.get(url)
#     soup = BeautifulSoup(page.content, 'html.parser')

#     job_listings = soup.find_all('div', class_='job-listing')

#     for job in job_listings:
#         company_name = job.find('div', class_='job-listing__company').text.strip()
#         job_title = job.find('div', class_='job-listing__title').text.strip()
#         job_url = "https://careers.microsoft.com" + job.find('a')['href']

#         print("Company Name:", company_name)
#         print("Job Title:", job_title)
#         print("Job Description URL:", job_url)

def scrape_careers_page():
    import requests
    from bs4 import BeautifulSoup

    # choose a random company from the list
    company = "Microsoft"

    # search for jobs with the keyword "AI"
    url = f"https://careers.microsoft.com/professionals/us/en/search-results?keywords=AI&from={company}"
    response = requests.get(url)
    soup = BeautifulSoup(response.content, 'html.parser')

    # click on the first entry of the results
    job_url = soup.find('a', class_='job-title-link')['href']
    job_response = requests.get(job_url)
    job_soup = BeautifulSoup(job_response.content, 'html.parser')

    # create a JSON object for the relevant information
    company_name = company
    job_title = job_soup.find('h1', class_='job-title').text.strip()
    job_description_url = job_url

    job_info = {
        "company_name": company_name,
        "job_title": job_title,
        "job_description_url": job_description_url
    }

    return job_info

scrape_careers_page()


my_dict = {"a": 1, "b": 2, "c": 3}

try:
    value = my_dict["a"]
except KeyError:
    print("A KeyError occurred!")
else:
    print("No error occurred!")
finally:
    print("This part is a final and always run, no matter what was in previous.")


import numpy as np
import matplotlib.pyplot as plt

def logtanh(x):
    return -np.log(np.tanh(x / 2))

def plot_large_x():
    import matplotlib.pyplot as plt
    x_series = np.arange(0, 35, 0.0001)
    exact_vals = logtanh(x_series)
    t = np.exp(-x_series)

    approx_vals = 2 * t
    rel_error = np.abs(exact_vals - approx_vals) / exact_vals
    plt.semilogy(x_series, rel_error)

    approx_vals = 2 * (t + t ** 3 / 3)
    rel_error = np.abs(exact_vals - approx_vals) / exact_vals
    plt.semilogy(x_series, rel_error)

    approx_vals = 2 * (t + t ** 3 / 3 + t ** 5 / 5)
    rel_error = np.abs(exact_vals - approx_vals) / exact_vals
    plt.semilogy(x_series, rel_error)
    plt.legend(['First-order', 'Third-order', 'Fifth-order'])
    plt.grid()
    plt.show()

def plot_small_x():
    logx = np.arange(-13, 0, 0.001)
    x_series = np.exp(logx)
    exact_vals = logtanh(x_series)

    approx_vals = -np.log(x_series / 2)
    rel_error = np.abs(exact_vals - approx_vals) / exact_vals
    plt.semilogy(x_series, rel_error)

    approx_vals = -np.log(x_series / 2) + x_series ** 2 / 12
    rel_error = np.abs(exact_vals - approx_vals) / exact_vals
    plt.semilogy(x_series, rel_error)
    plt.xscale('log')
    plt.legend(['First-order', 'Second-order'])
    plt.grid()
    plt.show()

if __name__ == '__main__':
    plot_small_x()
    plot_large_x()


import ply.lex as lex
import ply.yacc as yacc
# This is heavily based on Michael Fogelman's solution
# (https://www.michaelfogleman.com/aoc18/#20)
# and only refactored so I understood it better

directions = {
    'N': (0, -1),
    'S': (0, 1),
    'E': (1, 0),
    'W': (-1, 0)
    }

# lexer rules
tokens = ['LPAREN', 'RPAREN', 'OR', 'DIR']

t_LPAREN = r'\('
t_RPAREN = r'\)'
t_OR = r'\|'
t_DIR = r'[NESW]+'

# parser rules
def p_pattern(t):
    'pattern : concat'
    t[0] = t[1]

def p_pattern_or(t):
    'pattern : pattern OR concat'
    t[0] = ('or', t[1], t[3])

def p_concat(t):
    'concat : term'
    t[0] = t[1]

def p_concat_term(t):
    'concat : concat term'
    t[0] = ('concat', t[1], t[2])

def p_term(t):
    'term : DIR'
    t[0] = ('dir', t[1])

def p_term_empty(t):
    'term :'
    t[0] = ('dir', '')

def p_term_paren(t):
    'term : LPAREN pattern RPAREN'
    t[0] = t[2]

def parse(text):
    lexer = lex.lex(errorlog=lex.NullLogger())
    parser = yacc.yacc()
    text = text.strip().replace('^', '').replace('$', '')
    return parser.parse(text, lexer=lexer)

def visit(node, doors, visited):
    if node[0] == 'concat':
        visited = visit(node[1], doors, visited)
        visited = visit(node[2], doors, visited)
        return visited
    elif node[0] == 'or':
        path_a = visit(node[1], doors, visited)
        path_b = visit(node[2], doors, visited)
        return list(set(path_a) | set(path_b))
    else:
        for d in node[1]:
            dx, dy = directions[d]
            doors |= set((x, y, x + dx, y + dy) for x, y in visited)
            doors |= set((x + dx, y + dy, x, y) for x, y in visited)
            visited = [(x + dx, y + dy) for x, y in visited]
        return visited

def locate_doors(text):
    parse_tree = parse(text)
    doors = set()
    visit(parse_tree, doors, [(0, 0)])
    return doors

def search(doors):
    distances = {}
    queue = [(0, 0, 0)]
    while queue:
        x, y, d = queue.pop()
        if (x, y) in distances and distances.get((x, y)) <= d:
            continue
        distances[(x, y)] = d
        for dx, dy in directions.values():
            if (x, y, x + dx, y + dy) in doors:
                queue.append((x + dx, y + dy, d + 1))
    return distances

with open("input.txt", 'r') as myfile:
    input = myfile.readline()
doors = locate_doors(input)
distances = search(doors)
print("Part 1: ", max(distances.values()))
print("Part 2: ", sum(x >= 1000 for x in distances.values()))


a= "abcasfhajfhsaassjfhsdjfhsdfsf"
## print out this a5b1c1d2f6h4j3s7
# sorted alphabet with thier frequency

def usingcollections():
    from collections import Counter
    cc = Counter(a)
    a = [i for i in sorted(cc)] # sort dict by key value
    res = ""
    for i in a:
        res += i + str(cc[i])
    print(res)

def usinglists():

    ans = []
    b = set(a)
    for i in b:
        ans.append(i)
        count = a.count(i)
        ans.append(str(count))
    print(ans)
    print("".join(ans) , " ------> join method ")

def iterablemethod():
    b =list(a)
    b.sort()
    count = 0
    res = ""
    for i in range(len(b)):
        if i+1 < len(b) -1 :
            if b[i] == b[i+1]:
                count += 1
            else:
                res = res + str(b[i]) + str(count + 1)
                count = 0
    print(res)