from functools import reduce

lst=[]
x=int(input("Enter digit:"))
for i in range(1,x+1):
    ele=int(input("Enter a value:"))
    lst.append(ele)
print("list is:",lst)

filter_list=list(filter(lambda ele:(ele>=70 and ele<=90),lst))
print("Filtered list is",filter_list)

map_list=list(map(lambda ele:(ele+10),filter_list))
print("List after Addition by ten:",map_list)

reduce_no=reduce(lambda a,b:(a*b),map_list)
print("reduced output:",reduce_no)


import openai
from secret_key import openai_key
import json
import pandas as pd

openai.api_key = openai_key

def extract_financial_data(text):
    prompt = get_prompt_financial() + text
    response = openai.ChatCompletion.create(
        model="gpt-3.5-turbo",
        messages=[{"role": "user","content": prompt}]
    )
    content = response.choices[0]['message']['content']

    try:
        data = json.loads(content)
        return pd.DataFrame(data.items(), columns=["Measure", "Value"])

    except (json.JSONDecodeError, IndexError):
        pass

    return pd.DataFrame({
        "Measure": ["Company Name", "Stock Symbol", "Revenue", "Net Income", "EPS"],
        "Value": ["", "", "", "", ""]
    })

def get_prompt_financial():
    return '''Please retrieve company name, revenue, net income and earnings per share (a.k.a. EPS)
    from the following news article. If you can't find the information from this article
    then return "". Do not make things up.
    Then retrieve a stock symbol corresponding to that company. For this you can use
    your general knowledge (it doesn't have to be from this article). Always return your
    response as a valid JSON string. The format of that string should be this,
    {
        "Company Name": "Walmart",
        "Stock Symbol": "WMT",
        "Revenue": "12.34 million",
        "Net Income": "34.78 million",
        "EPS": "2.1 $"
    }
    News Article:
    ============

    '''

# Press the green button in the gutter to run the script.
if __name__ == '__main__':
    text = '''
    Tesla's Earning news in text format: Tesla's earning this quarter blew all the estimates. They reported 4.5 billion $ profit against a revenue of 30 billion $. Their earnings per share was 2.3 $
    '''
    df = extract_financial_data(text)

    print(df.to_string())


import numpy as np
import matplotlib.pylab as plt

def mifun(x):
    x_0 = 3.0
    a = 0.01
    return np.exp(-(x**2))/((x-x_0)**2 + a**2)

pasos=[]
iteraciones=1000

def MH(numpasos,sigma):
    pasos=np.array([])
    xviejo=np.random.uniform(-4,4,1)
    pasos=np.append(pasos,xviejo)

    for i in range(numpasos):
        xnuevo=np.random.normal(pasos[i],sigma)

        alpha=mifun(xnuevo)/mifun(pasos[i])
        if(alpha>=1):
            pasos=np.append(pasos,xnuevo)
        else:
            beta=np.random.random()
            if(beta<alpha):
                pasos=np.append(pasos,xnuevo)
            else:
                pasos=np.append(pasos,xviejo)
    return pasos

caso1=MH(100000,5)
plt.hist(caso1, bins=1000, density=True)
plt.plot(x,funcion)
plt.show()

caso2=MH(100000,0.2)
plt.hist(caso2, bins=1000, density=True)
plt.plot(x,funcion)
plt.show()

caso3=MH(100000,0.01)
plt.hist(caso3, bins=1000, density=True)
plt.plot(x,funcion)
plt.show()

caso4=MH(1000,0.1)
plt.hist(caso4, bins=1000, density=True)
plt.plot(x,funcion)
plt.show()

caso5=MH(100000,0.1)
plt.hist(caso5, bins=1000, density=True)
plt.plot(x,funcion)
plt.show()


import math
import os
import re
import sys
import gym
import random
import requests
import json
import copy
import subprocess
import logging
import numpy as np
import matplotlib.pyplot as plt
import scipy.optimize as optimize
from constraint import *
from typing import List
from gym.spaces import Discrete, Box
# from scipy.optimize import basinhopping
from IPython.display import display, clear_output
from tf_agents.specs import array_spec
from tf_agents.environments import py_environment
from tf_agents.trajectories import time_step as ts

# logging.basicConfig(format='%(asctime)s-ENV-%(levelname)s-%(message)s', level=logging.DEBUG, datefmt='%Y-%m-%d %H:%M:%S')

class CurveEnv(py_environment.PyEnvironment):

    def __init__(self, verbose: bool=False):
        self._low = -3
        self._high = 3

        self._action_spec = array_spec.BoundedArraySpec(
            shape=(), dtype=np.int32, minimum=0, maximum=1, name='action')
        self._observation_spec = array_spec.BoundedArraySpec(
            shape=(2,), dtype=np.int32, minimum=self._low, maximum=self._high, name='observation')

        self._state = 0
        self._episode_ended = False
        self._verbose = verbose

    def action_spec(self):
        return self._action_spec

    def observation_spec(self):
        return self._observation_spec

    def _reset(self):
        self._episode_ended = False
        self._target_location = self._get_target()
        self._state = self._get_default_state()

        # We will sample agent's location until
        # it does not coincide with the target location
        while np.allclose(self._state, self._target_location):
            self._state = self._get_default_state()

        self._best_state = self._state

        # self.ax.clear()

        # if self.render_mode == "human":
        #     self._render()

        if self._verbose: print(f"[RESET] state: {self._state}, target: {self._target_location}")

        return ts.restart(np.array(self._state, dtype=np.int32))

    def _step(self, action):
        if self._episode_ended:
            # The last action ended the episode. Ignore the current action and start
            # a new episode.
            return self.reset()

        action = self._transform_action(action)
        # `_state` is a np.array([x, y]), so first we update state_x
        self._state[0] += action
        # We use `np.clip` to make sure we don't leave the boundaries
        self._state[0] = np.clip(self._state[0], self._low, self._high)
        self._state[1] = self._y(self._state[0])

        # A metric (y, throughput, latency, etc) is a reward itself
        reward = -self._state[1]

        # An episode is done if the agent has reached the target.
        self._episode_ended = np.allclose(self._state, self._target_location)

        # self.ax.clear()

        # if self.render_mode == "human":
        #     self._render_frame()

        if self._verbose: print(f"[STEP] state: {self._state}, target: {self._target_location}, reward: {reward}")

        if self._episode_ended:
            return ts.termination(np.array(self._state, dtype=np.int32), reward)
        else:
            return ts.transition(
          np.array(self._state, dtype=np.int32), reward=0.0, discount=1.0)

    def _y(self, x):
        return 2*math.pow(x, 3) - 3*math.pow(x, 2) - 12 * x + 1

    def _transform_action(self, action):
        # Convert action from [0, 1] to [-1, 1]
        return 2 * action - 1

    def _get_target(self):
        x0 = [1.]
        minimizer_kwargs = {"method": "BFGS"}
        solution = optimize.basinhopping(self._y, x0, minimizer_kwargs=minimizer_kwargs,
                   niter=200)
        target_x = np.round(solution.x[0], 5)
        target_y = np.round(solution.fun, 5)
        return np.array([target_x, target_y], dtype=np.int32)

    def _get_default_state(self):
        state_x = np.random.randint(self._low, self._high, size=None, dtype=int)
        state_y = self._y(state_x)
        return np.array([state_x, state_y], dtype=np.int32)

    def _get_reward(self, next_state, current_state):
        """
        The reward is the relative difference between
        a current agent value and the next one. The
        normalization puts a large measurement range on the
        same scale.

        Han, Xue & Yu, Tingting. (2020). Automated Performance Tuning for Highly-Configurable Software Systems.
        """
        return (next_state - current_state) / current_state

    def _render(self):
        self.fig, self.ax = plt.subplots()
        self.line, = self.ax.plot([], [])
        self.ax.set_xlabel('X')
        self.ax.set_ylabel('Y')
        self.ax.set_title('Agent Learning Curve Navigation')

        x_vals = np.linspace(self._low, self._high)
        y_vals = [self._y(x) for x in x_vals]

        self.ax.clear()
        self.ax.grid(True)
        self.ax.plot(x_vals, y_vals, color='black')
        self.ax.scatter(self._target_location[0], self._target_location[1], color='red', label="Target")
        self.ax.scatter(self._state[0], self._state[1], color='blue', label="Agent")

        self.ax.set_xlabel('X')
        self.ax.set_ylabel('Y')
        self.ax.set_title('Agent Learning Curve Navigation')
        self.ax.legend()
        clear_output(wait=True)
        display(self.fig)

class CurveMultipleEnv(py_environment.PyEnvironment):

    def __init__(self, verbose: bool=False):
        self._num_variables = 2

        self._flags_min = {
            "x": -2,
            "y": -10,
        }

        self._flags_max = {
            "x": 2,
            "y": 10,
        }

        self._action_mapping = {
            0: self._decrease_MaxHeapSize,
            1: self._increase_MaxHeapSize,
            2: self._decrease_InitialHeapSize,
            3: self._increase_InitialHeapSize,
        }

        self._action_spec = array_spec.BoundedArraySpec(
            shape=(), dtype=np.int32, minimum=0, maximum=2*self._num_variables-1, name='action') # {0, 1, 2, 3}

        self._observation_spec = array_spec.BoundedArraySpec(
            shape=(self._num_variables,),
            dtype=np.int32,
            minimum=list(self._flags_min.values()),
            maximum=list(self._flags_max.values()),
            name='observation'
        )

        self._state = 0
        self._episode_ended = False
        self._verbose = verbose

    def action_spec(self):
        return self._action_spec

    def observation_spec(self):
        return self._observation_spec

    def _reset(self):
        self._episode_ended = False
        self._target_location = self._get_target()
        self._state = self._get_default_state()

        # We will sample agent's location until
        # it does not coincide with the target location
        while self._is_equal(self._state, self._target_location):
            self._state = self._get_default_state()

        self._best_state = self._state

        # self.ax.clear()

        # if self.render_mode == "human":
        #     self._render()

        if self._verbose: print(f"[RESET] state: {self._state}, target: {self._target_location}")

        return ts.restart(np.array(self._state[0], dtype=np.int32))

    def _step(self, action):
        if self._episode_ended:
            # The last action ended the episode. Ignore the current action and start
            # a new episode.
            return self.reset()

        # print(type(action))
        # Apply an action: decrease/increase <var>
        self._action_mapping.get(int(action))()

        # We use `np.clip` to make sure we don't leave the boundaries
        for i in range(len(self._state[0])):
            # Iterate through each variable in `self._state`
            # and use `np.clip` to make sure we don't leave
            # the boundaries
            min_value_i = list(self._flags_min.values())[i]
            max_value_i = list(self._flags_max.values())[i]
            self._state[0][i] = np.clip(self._state[0][i], min_value_i, max_value_i)

        self._state[1] = self._y(self._state[0])

        # An episode is done if the agent has reached the target.
        self._episode_ended = self._is_equal(self._state, self._target_location)

        # A metric (y, throughput, latency, etc) is a reward itself
        # reward = -self._state[1]

        reward = 1 if self._episode_ended else 0

        # self.ax.clear()

        # if self.render_mode == "human":
        #     self._render_frame()

        if self._verbose: print(f"[STEP] state: {self._state}, target: {self._target_location}, reward: {reward}")

        if self._episode_ended:
            return ts.termination(
                np.array(self._state[0], dtype=np.int32), reward)
        else:
            return ts.transition(
                np.array(self._state[0], dtype=np.int32), reward=0.0, discount=1.0)

    def _y(self, params):
        x, y = params
        # z_min = z(0; 1) = -2
        return 3*x**2 + x*y + 2*y**2 - x - 4*y

    def _transform_action(self, action):
        # Convert action from [0, 1] to [-1, 1]
        return 2 * action - 1

    def _get_target(self):
        initial_guess = [1, 1]
        result = optimize.minimize(self._y, initial_guess)
        if result.success:
            target_arg = np.round(result.x, 5)
            target_f = np.round(result.fun, 5)
            # returns np.array([[x, y], z], dtype=np.int32)
            return np.array([target_arg, target_f], dtype=object)
        else:
            raise ValueError(result.message)

    def _get_default_state(self):
        state_args = np.random.randint(
            list(self._flags_min.values()),
            list(self._flags_max.values()),
            size=(self._num_variables,),
            dtype=np.int32
        )
        state_f = self._y(state_args)
        return np.array([state_args, state_f], dtype=object)

    def _get_reward(self, next_state, current_state):
        """
        The reward is the relative difference between
        a current agent value and the next one. The
        normalization puts a large measurement range on the
        same scale.

        Han, Xue & Yu, Tingting. (2020). Automated Performance Tuning for Highly-Configurable Software Systems.
        """
        return (next_state - current_state) / current_state

    def _decrease_MaxHeapSize(self):
        self._state[0][0] -= 1

    def _increase_MaxHeapSize(self):
        self._state[0][0] += 1

    def _decrease_InitialHeapSize(self):
        self._state[0][1] -= 1

    def _increase_InitialHeapSize(self):
        self._state[0][1] += 1

    def _is_equal(self, state, target):
        return np.allclose(state[0], target[0])

    def _render(self):

        self.fig, self.ax = plt.subplots()
        self.line, = self.ax.plot([], [])
        self.ax.set_xlabel('X')
        self.ax.set_ylabel('Y')
        self.ax.set_title('Agent Learning Curve Navigation')

        x_vals = np.linspace(self._low, self._high)
        y_vals = [self._y(x) for x in x_vals]

        self.ax.clear()
        self.ax.grid(True)
        self.ax.plot(x_vals, y_vals, color='black')
        self.ax.scatter(self._target_location[0], self._target_location[1], color='red', label="Target")
        self.ax.scatter(self._state[0], self._state[1], color='blue', label="Agent")

        self.ax.set_xlabel('X')
        self.ax.set_ylabel('Y')
        self.ax.set_title('Agent Learning Curve Navigation')
        self.ax.legend()
        clear_output(wait=True)
        display(self.fig)

class JVMEnv(py_environment.PyEnvironment):

    def __init__(
            self,
            jdk: str,
            bm_path: str,
            callback_path: str,
            bm: str = "cassandra",
            n: int = 5,
            goal: str = "avgGCPause",
            verbose: bool=False
        ):

        self._jdk = os.path.join(jdk, "bin")
        # self._state = 0
        self._episode_ended = False
        self._verbose = verbose
        self._bm = bm
        self._bm_path = bm_path
        self._callback_path = callback_path
        self._gc_log_file = f"gc-{self._bm}.txt"
        self._n = n
        self._goal = goal

        self._env = os.environ.copy()
        self._env["PATH"] = f"{self._jdk}:{self._env['PATH']}"
        self._gc_viewer_jar = "gcviewer-1.36.jar"

        # ============= F L A G S =============
        # TODO: Add more flags
        # TODO: Find min and max values for flags
        self._num_variables = 2
        self._flags = {
            "MaxTenuringThreshold": {"min": 1, "max": 16},
            "ParallelGCThreads": {"min": 20, "max": 48},
            # "MaxHeapSize": {"min": 6.4e+7, "max": 5.0e+10},
            # "InitialHeapSize": {"min": 8.0e+6, "max": 4.0e+8},
        }

        self._action_mapping = {
            0: self._decrease_MaxTenuringThreshold,
            1: self._increase_MaxTenuringThreshold,
            2: self._decrease_ParallelGCThreads,
            3: self._increase_ParallelGCThreads,
            # 0: self._decrease_MaxHeapSize,
            # 1: self._increase_MaxHeapSize,
            # 2: self._decrease_InitialHeapSize,
            # 3: self._increase_InitialHeapSize,
        }
        # =====================================

        assert len(list(self._action_mapping.keys())) == 2*self._num_variables, f"Each flag should have 2 actions!"
        assert os.path.exists(self._gc_viewer_jar), f"{self._gc_viewer_jar} does not exist"
        assert os.path.exists(self._bm_path), f"{self._bm_path} does not exist"
        assert os.path.exists(self._jdk), f"{self._jdk} does not exist"

        self._flags_min_values = [self._flags[i]["min"] for i in self._flags.keys()]
        self._flags_max_values = [self._flags[i]["max"] for i in self._flags.keys()]

        self._action_spec = array_spec.BoundedArraySpec(
            shape=(),
            dtype=np.int32,
            minimum=0,
            maximum=2*self._num_variables-1,  # {0, 1, 2, 3}
            name='action'
        )

        self._observation_spec = array_spec.BoundedArraySpec(
            shape=(self._num_variables,),
            dtype=np.int64,
            minimum=self._flags_min_values,
            maximum=self._flags_max_values,
            name='observation'
        )

        self._default_state = self._get_default_state(mode="default")
        self._default_goal_value = self._default_state[1]
        self._current_goal_value = self._default_goal_value

        """
        A cache to store performance measurements for states.
        Han, Xue & Yu, Tingting. (2020).
        Automated Performance Tuning for Highly-Configurable Software Systems.

        0: {"args": [10, 10], "goal": 234},
        1: {"args": [10, 20], "goal": 222},
        """
        self._perf_states = {}
        self._print_welcome_msg()

    def action_spec(self):
        return self._action_spec

    def observation_spec(self):
        return self._observation_spec

    def _reset(self):
        self._episode_ended = False
        self._current_goal_value = self._default_goal_value

        # To ensure all elements within an object array are copied, use `copy.deepcopy`
        self._state = copy.deepcopy(self._default_state)
        # self.ax.clear()
        # if self.render_mode == "human":
        #     self._render()

        logging.debug(f"[RESET] {self._get_info()}, target: {self._current_goal_value}")
        return ts.restart(np.array(self._state[0], dtype=np.int64))

    def _step(self, action):
        if self._episode_ended:
            # The last action ended the episode. Ignore the current action and start
            # a new episode.
            return self.reset()

        # Apply an action based on the mapping: decrease/increase <flag.value>
        self._action_mapping.get(int(action))()
        # Make sure we don't leave the boundaries
        self._state = self._clip_state(self._state)
        flags, goal = self._state_merging(self._state[0])

        # # Get new JVM options values after setting `self._state`
        # jvm_opts = self._get_jvm_opts(self._state[0])
        # # Launch a benchmark with a new JVM configuration
        # self._run(jvm_opts, self._gc_log_file, self._bm, self._bm_path, self._n)
        # # Get the goal value
        # goal = self._get_goal_from_file()

        previous_goal_value = self._current_goal_value
        self._state[0] = flags
        self._state[1] = goal
        self._current_goal_value = goal

        # ! Termination criteria
        if self._current_goal_value <= self._default_goal_value * 0.7:
            self._episode_ended = True

        # ! Multiply by (-1) if lower is better
        # reward = -1 * self._current_goal_value
        self._reward = -1 * self._get_reward(self._current_goal_value, previous_goal_value)
        # self._reward = -1 * self._get_reward(self._current_goal_value, self._default_goal_value)

        # self._reward = -1 * (
        #     self._get_reward(self._current_goal_value, previous_goal_value)
        #     + self._get_reward(self._current_goal_value, self._default_goal_value)
        # )
        # self._reward = 1 if self._current_goal_value < self._default_goal_value else -10

        # self.ax.clear()

        # if self.render_mode == "human":
        #     self._render_frame()

        logging.debug(f"[STEP] {self._get_info()}, current_goal_value: {self._current_goal_value}, reward: {self._reward}")

        # if self._episode_ended:
        #     return ts.termination(
        #         np.array(self._state[0], dtype=np.int64), self._reward)
        # else:
        #     return ts.transition(
        #         np.array(self._state[0], dtype=np.int64), reward=0.0, discount=1.0)

        return ts.transition(
                np.array(self._state[0], dtype=np.int64), reward=self._reward, discount=0.9999)

    def _state_merging(self, flags):
        """
        Store states' JVM configurations and performance measurements
        in a cache. The cache stores the JVM flags and the metric goal.
        In each `self._step` iteration, the performance of the same
        state is queried and retrieved directly from the cache instead
        of re-running the benchmark utility.
        """
        saved_states = [self._perf_states[i]["args"] for i in self._perf_states.keys()]
        if flags in saved_states:
            for i in self._perf_states.keys():
                """
                If current state is stored in a cache,
                update the state goal value.
                """
                if flags == self._perf_states[i]["args"]:
                    goal = self._perf_states[i]["goal"]
        elif flags not in saved_states:
            """
            If current state is not stored in the cache,
            measure the performance metric, and save it
            in the cache.
            """
            try:
                last_index = list(self._perf_states.keys())[-1]
            except IndexError:
                # If `self._perf_states` is empty
                last_index = -1

            # Get new JVM options values after setting `self._state`
            jvm_opts = self._get_jvm_opts(flags = self._state[0])
            # Launch a benchmark with a new JVM configuration
            self._run(jvm_opts, self._gc_log_file, self._bm, self._bm_path, self._callback_path, self._n)
            # Get the goal value
            goal = self._get_goal_from_file()
            # Store a new state in the cache
            self._perf_states[last_index + 1] = {"args": list(flags), "goal": goal}
        return flags, goal

    def _get_JVM_opt_value(self, opt: str):
        """
        Get the defaul JVM option value from environment
        by parsing java PrintFlagsFinal output.

        Parameters:
        opt (str): JVM option name

        Returns:
        (int) Default JVM option value
        """
        opt_value = None

        flags = subprocess.check_output(
            ["java", "-XX:+PrintFlagsFinal", "-version"],
            text=True,
            env=self._env)

        assert re.search(opt, flags), f"Option {opt} was not found in JVM flags"

        for line in flags.split('\n'):
            if re.search(opt, line):
                opt_value = re.findall("\d+", line)[0]
                # TODO: Handle boolean values
                return int(opt_value)

    def _get_info(self):
        info = {}
        flags = list(self._flags.keys())
        for flag in flags:
            flag_idx = flags.index(flag)
            flag_value = self._state[0][flag_idx]
            info[flag] = flag_value
        return info

    def _get_default_state(self, mode: str="default"):
        """
        Get default values for each JVM option stored in `self._flags`
        from `java -XX:+PrintFlagsFinal -version` command output.
        Also runs a benchmark with default JVM_OPTS and get the initial
        goal value.

        Parameters:
        mode (str): The mode of generating JVM options (default/minimum/random)
                    where
                    - "default" sets JVM_OPTS default JDK options,
                    - "minimum" sets JVM_OPTS minimum possible values,
                    - "random" sets JVM_OPTS random values within options range.

        Returns:
        (np.array) The initial state of the Agent which stores the default
        JVM_OPTS and its performance measurement.
        """
        assert mode in ["default", "minimum", "random"], f"Mode '{mode}' is not supported"

        jvm_opts = []

        if mode == "default":
            for flag in self._flags.keys():
                flag_default_value = self._get_JVM_opt_value(flag)
                jvm_opts.append(flag_default_value)
        elif mode == "minimum":
            jvm_opts = self._flags_min_values.copy()
        elif mode == "random":
            jvm_opts = np.random.uniform(
                low = self._flags_min_values,
                high = self._flags_max_values,
                size = (self._num_variables,))

        # Performance measurement with default JVM_OPTS
        goal = self._get_goal_value()
        # goal = self._default_goal_value
        initial_state = copy.deepcopy(np.array([jvm_opts, goal], dtype=object))
        return initial_state

    def _get_goal_value(self, jvm_opts: List[str]=[]):
        """
        Run the benchmark with default values and
        get a start point of the goal.

        Parameters:
        jvm_opts (list): An array of JVM options.

        Returns:
        (int) A performance measurement (target metric
        or goal value).
        """
        # Run benchmark with default values
        self._run(jvm_opts, self._gc_log_file, self._bm, self._bm_path, self._callback_path, self._n)

        if os.path.exists(self._gc_log_file):
            # Get goal value from first-time generated GC log
            result = self._get_goal_from_file()
            # Clean up
            os.remove(self._gc_log_file)
            return result
        else:
            raise Exception(
                "GC log file was not generated: please, ensure that benchmark runs correctly!")

    def _clip_state(self, state):
        for i in range(len(state[0])):
            """
            Iterate through each flag value in `state`
            and use `np.clip` to make sure we don't leave
            the boundaries
            """
            min_value_i = self._flags_min_values[i]
            max_value_i = self._flags_max_values[i]
            state[0][i] = np.clip(state[0][i], min_value_i, max_value_i)
        return state

    def _get_goal_from_file(self):
        sep = ';'
        summary = "summary.csv"
        goal_value = None
        subprocess.call(
            ["java",
            "-cp", self._gc_viewer_jar,
            "com.tagtraum.perf.gcviewer.GCViewer",
            self._gc_log_file,
            summary,
            "-t", "SUMMARY"],
            stdout=subprocess.DEVNULL,
            stderr=subprocess.STDOUT,
            text = True,
            env = self._env)

        assert os.path.exists(summary), f"File {summary} does not exist"

        with open(summary, "+r") as summary_file:
            for line in summary_file.readlines():
                if self._goal + sep in line:
                    goal_value = float(line.split(sep)[1])

        if goal_value == None:
            raise Exception(f"Goal '{self._goal}' was not found in '{summary}'!")

        if os.path.exists(summary): os.remove(summary)

        return goal_value

    def _get_reward(self, next_state, current_state):
        """
        The reward is the relative difference between
        a current agent value and the next one. The
        normalization puts a large measurement range on the
        same scale.

        Han, Xue & Yu, Tingting. (2020). Automated Performance Tuning for Highly-Configurable Software Systems.
        """
        return (next_state - current_state) / current_state

    def _check_constraints(self, a, b, constraint):
        problem = Problem()
        problem.addVariable("a", [a])
        problem.addVariable("b", [b])
        problem.addConstraint(constraint, ("a", "b"))
        solutions = problem.getSolutions()
        return solutions

    def _decrease_MaxHeapSize(self):
        coef = 100 * (2**10) * (2**10) # 100 Mb
        Xmx = self._state[0][0] - coef
        Xms = self._state[0][1]
        solutions = self._check_constraints(
            Xmx,
            Xms,
            lambda a, b: a > b)

        if len(solutions) > 0:
            self._state[0][0] = Xmx

    def _increase_MaxHeapSize(self):
        coef = 100 * (2**10) * (2**10) # 100 Mb
        Xmx = self._state[0][0] + coef
        Xms = self._state[0][1]
        solutions = self._check_constraints(
            Xmx,
            Xms,
            lambda a, b: a > b)

        if len(solutions) > 0:
            self._state[0][0] = Xmx

    def _decrease_InitialHeapSize(self):
        coef = 100 * (2**10) * (2**10) # 100 Mb
        Xms = self._state[0][1] - coef
        Xmx = self._state[0][0]
        solutions = self._check_constraints(
            Xmx,
            Xms,
            lambda a, b: a > b)

        if len(solutions) > 0:
            self._state[0][1] = Xms

    def _increase_InitialHeapSize(self):
        coef = 100 * (2**10) * (2**10) # 100 Mb
        Xms = self._state[0][1] + coef
        Xmx = self._state[0][0]
        solutions = self._check_constraints(
            Xmx,
            Xms,
            lambda a, b: a > b)

        if len(solutions) > 0:
            self._state[0][1] = Xms

    def _decrease_MaxTenuringThreshold(self):
        coef = 3
        self._state[0][0] -= coef

    def _increase_MaxTenuringThreshold(self):
        coef = 3
        self._state[0][0] += coef

    def _decrease_ParallelGCThreads(self):
        coef = 4
        self._state[0][1] -= coef

    def _increase_ParallelGCThreads(self):
        coef = 4
        self._state[0][1] += coef

    def _is_equal(self, state, target):
        return np.allclose(state[0], target[0])

    def _get_jvm_opts(self, flags):
        jvm_opts = []
        for flag_name in self._flags.keys():
            i = list(self._flags.keys()).index(flag_name)
            # `state[0]` stores an array of current
            # flag values [<MaxHeapSize>, <InitialHeapSize>]
            flag_value = int(flags[i])
            # TODO: Add a condition for flags with boolean values
            jvm_opts.append(f"-XX:{flag_name}={flag_value}")
        return jvm_opts

    def _run(
            self,
            jvm_opts: List[str],
            gc_log_file: str,
            bm: str,
            bm_path: str,
            callback_path: str,
            n: int=5,
            verbose: bool=False):
        """
        Run a benchmark with the specified JVM options
        such as MaxHeapSize.

        Parameters:
        jvm_opts (array of str):  JVM options (e.g., MaxHeapSize, GCTimeRation,
                                ParallelGCThreads, etc).
        gc_log_file (str):        Path to the garbage collector log file (txt).
        bm (str):                 DaCapo benchmark name.
        bm_path (str):            DaCapo benchmark path.
        n (int):                  Total number of benchmark's iterations.
        verbose (bool):           Print debug messages.
        """

        # Clean up before running the benchmark
        if os.path.exists(gc_log_file): os.remove(gc_log_file)

        # Default flags
        jvm_opts.append("-XX:+UseParallelGC")
        jvm_opts.append("-Xmx16G")
        jvm_opts.append("-Xms16G")
        jvm_opts.append("-XX:SurvivorRatio=130")
        jvm_opts.append("-XX:TargetSurvivorRatio=66")

        # Run the benchmark (hide output)
        try:
            subprocess.check_output(
                ["java",
                "-cp", f"{callback_path}:{bm_path}",
                f"-Xlog:gc*=trace:file={gc_log_file}:tags,time,uptime,level",
                *jvm_opts,
                "-Dvmstat.enable_jfr=yes",
                "-Dvmstat.csv=yes", "Harness", "-v", "-n", f"{n}", bm],
                stderr=subprocess.STDOUT,
                text = True,
                env = self._env)
        except subprocess.CalledProcessError as e:
            raise Exception("Command failed with return code", e.returncode, e.output)

        return

    def _render(self):

        self.fig, self.ax = plt.subplots()
        self.line, = self.ax.plot([], [])
        self.ax.set_xlabel('X')
        self.ax.set_ylabel('Y')
        self.ax.set_title('Agent Learning Curve Navigation')

        x_vals = np.linspace(self._low, self._high)
        y_vals = [self._y(x) for x in x_vals]

        self.ax.clear()
        self.ax.grid(True)
        self.ax.plot(x_vals, y_vals, color='black')
        self.ax.scatter(self._target_location[0], self._target_location[1], color='red', label="Target")
        self.ax.scatter(self._state[0], self._state[1], color='blue', label="Agent")

        self.ax.set_xlabel('X')
        self.ax.set_ylabel('Y')
        self.ax.set_title('Agent Learning Curve Navigation')
        self.ax.legend()
        clear_output(wait=True)
        display(self.fig)

    def _print_welcome_msg(self):
        print("Successfully initialized a JVM Environment!\n",
              f"JDK: {self._jdk},\n",
              f"Benchmark: {self._bm} ({self._bm_path}),\n",
              f"Number of iterations: {self._n},\n",
              f"Goal: {self._goal},\n",
              f"Number of JVM options: {self._num_variables},\n",
              f"JVM options: {self._flags},\n",
              f"Env. default state: {self._default_state},\n",
              f"Env. default goal value: {self._default_goal_value},\n",)


from pyspark.sql import SparkSession
import pandas
from collections import defaultdict
import random
from math import ceil

'''
Process used to create Data

According to the use case mentioned we need to generate data that had
1. skew distribution : size s1
2. normal distribution : size s1
3. skew distribution : size s2
4. normal distribution : size s2

Here s1 and s2 are the sizes that can vary depend on use case for our e.g we had taken s1 = 10GB and s2 = 25GB

--> To tackle this I had created class called writegenerateData which can write and generate the data with the size that we want
    for this there is function called generate_huge_data which generates the data with the specified size

For the scenario part we had four scenarios with us
1. Data Skew
2. Input Data Increased
3. Input Data Increased, Skewed
4. Less resources

class create_scenarios helps us to create the wanted scenarios where we first load the desired data and then apply the load_and_groupBy function

'''

class eshtablishContext:
    def __init__(self,session_name):
        self.session = None
        self.session_name = session_name

    def build_session(self):
        self.session = SparkSession.builder.appName(self.session_name).enableHiveSupport().getOrCreate()
        return self.session

    def return_data(self,dbname,table_name):
        self.session.sql("use "+dbname)
        return self.session.sql("select * from "+table_name)

    def load_skew_data(self,table_name):
        return self.return_data(table_name)

    def load_1_gb(self,table_name):
        return self.return_data(table_name)

    def load_2_gb(self,table_name):
        return self.return_data(table_name)

    def do_groupby(self,df,col1,col2):
        return df.groupBy(col1).sum(col2)

class create_scenarios:
    def create_out_of_mem_driver(self):
        '''
        This will create OOM problem tested with 5 executors and 512mb mem
        '''
        use = eshtablishContext("gen_oom")
        session = use.build_session()
        df = session.sql("use etl_pipeline")
        df = session.sql("select * from 1gb_data")
        lst = df.collect()
        return len(lst)

    def create_out_of_mem_executor(self):
        '''
        This will create OOM problem tested with 1 executors and 512mb mem
        '''
        session = use.build_session()
        df = session.sql("use data_gen")
        df = session.sql("select * from skew_dist_1")
        for i in range(100):
            df = df.union(df)
            print("Done union with df for iteration :: ",i)
        session.sql("use bigdata")
        df.write.mode('append').saveAsTable("bigdata.data")
        print("Converted Pyspark DataFrame to Pandas DataFrame.")

    def test_execution_time_diff(self,use,session):
        import time
        df = session.sql("use etl_pipeline")
        start = time.time()
        start = time.time()
        df2 = session.sql("select * from 2gb_data")
        use.do_groupby(df2,"marital","duration").show(truncate=False)
        print("Time taken to read 2GB data :: ",time.time()-start)

    def load_and_groupBy(self,session,dbname,tabname,col1,col2):
        import time
        session.sql('use ' + dbname)
        df = session.sql('select * from ' + tabname)
        start = time.time()
        df.groupBy(col1).sum(col2).show(truncate=False)
        print("Time taken for {} is {} ".format(tabname,time.time()-start))

    def check_dist(self,session,dbname,tabname,col1):
        import time
        session.sql('use ' + dbname)
        df = session.sql('select * from ' + tabname)
        start = time.time()
        df.groupBy(col1).count().show()
        print("Time taken for {} is {} ".format(tabname,time.time()-start))

    def load(self,session,dbname,tabname):
        import time
        session.sql('use ' + dbname)
        start = time.time()
        df = session.sql('select * from ' + tabname)
        print("Time taken for {} is {} ".format(tabname,time.time()-start))

'''
can use this class generate_huge_data with the dataframe that you would like to replicate
'''

class writegenerateData:
    def __init__(self,lim):
        self.limit = lim
        self.lst = []
        self.cnt = 0
        self.df = None

    def fetch_data(self,url):
        self.df = pandas.read_csv(url,sep=';')

    def get_df_memsize(self,df):
        sample = df.limit(1).toPandas().memory_usage(index=True).sum()
        val = df.count() * sample
        return val

    def generate_small_user_dist_df(self,col_name,values,offset):
        ndf = pandas.DataFrame(columns = self.df.columns.tolist())
        for i in range(len(values)):
            value = values[i]
            for i in self.df.loc[self.df[col_name]==value].values.tolist()[:offset[i]]:
                ndf.loc[len(ndf)] = i
        return ndf

    def compute_avg_size(self,df,theta):
        avg_size = 0
        denom = int(theta*100)
        iterations = (100+denom)//denom
        print(iterations,denom)
        for _ in range(iterations):
            avg_size = avg_size + df.sample(theta).limit(1).toPandas().memory_usage(index=True).sum()
        avg_size = avg_size//iterations
        return avg_size * df.count()

    def random_sample_generator(self,session):
        df = session.createDataFrame(self.df)
        lst = df.sample(fraction = 0.06).collect()
        hsh = defaultdict(list)
        features = df.columns
        custom_df = pandas.DataFrame(columns=features)
        for row in lst:
            for key in features:
                hsh[key].append(row[key])
        for _ in range(df.count()):
            lst = []
            for key in hsh:
                lst.append(random.choice(hsh[key]))
            custom_df.loc[len(custom_df.index)] = lst
        print(custom_df.head())
        spdf = session.createDataFrame(custom_df)
        print("============================ Generated spark df with random sampling ====================")
        print(self.compute_avg_size(spdf,0.06))

    def return_dflist(self):
        return self.lst

    def commit_data_to_db(self,df,db_string):
        df.write.mode("append").saveAsTable(db_string)

    def write_to_db(self,db_string,parallel_exec=False):
        if parallel_exec:
            self.write_parallel_to_db(db_string)
        else:
            for df in self.lst:
                self.commit_data_to_db(df,db_string)

    def write_parallel_to_db(self,db_string):
        import threading
        threads = [threading.Thread(target=self.commit_data_to_db, args=[(db,db_string) for db in self.lst] )]
        for thread in threads:
            thread.start()
        for thread in threads:
            thread.join()

    def generate_huge_data(self,session,dbname,tab_name,size,theta):
        from math import ceil
        df = session.sql("use " + dbname)
        df = session.sql("select * from " + tab_name)
        sample = df.limit(theta)
        chunk_size = sample.toPandas().memory_usage(index=True).sum()
        sp = self.get_df_memsize(df)
        print("=================== size are =============== :: ",sp,size)
        sz = (size - sp)//chunk_size
        print("============== Wanted to create partitions of {} of size {} ".format(sz,chunk_size))
        for i in range(sz):
            sample.write.mode("append").saveAsTable(dbname+'.'+tab_name)
            print("============= Done with data insertion at iteration {}==============".format(i))

    def calc_df_size(self,session,dbname,tab_name):
        df = session.sql("use " + dbname)
        df = session.sql("select * from " + tab_name)
        return self.get_df_memsize(df)

    def store_data(self,session,df,dbname,tab_name):
        spdf = session.createDataFrame(df)
        spdf.write.mode("append").saveAsTable(dbname+'.'+tab_name)

    def see_partitions(self,session,dbname,tab_name):
        import pyspark.sql.functions as F
        session.sql("use " + dbname)
        df = session.sql("select * from " + tab_name)
        print("================ Showing partitions ================")
        print(df.groupBy(F.spark_partition_id()).count().show())

    def see_distribution(self,session,dbname,tab_name,colname):
        session.sql("use " + dbname)
        df = session.sql("select * from " + tab_name)
        print("================ Showing data in DB ================")
        print(df.groupBy(colname).count().show())

use = eshtablishContext("Increase_data")
session = use.build_session()
datalib = writegenerateData(int(1e6))
scene = create_scenarios()

session.sql("use skew_data")
df = session.sql("select * from skew_data")

for _ in range(2):
    df.write.mode("append").saveAsTable("skew_data"+'.'+"skew_data")

#datalib.see_partitions(session,"skew_data","skew_data_1")
#datalib.see_distribution(session,"skew_data","skew_data_1","marital")

#datalib.generate_huge_data(session,'bigdata','equal_data',10*int(1e9),int(1e6))

# Can call the necessary functions according to the need
#scene.load_and_groupBy(session,'bigdata','equal_data','marital','duration')

#scene.create_out_of_mem_executor()

'''
# Misc Section

gen_data = writegenerateData(int(1e6))
gen_data.increase_store_data(session,"etl_pipeline","2gb_data",2,int(1e5))
df = use.return_data("etl_pipeline","2gb_data")
print(datalib.get_df_memsize(df))C

#df = session.sql('select * from skew_dist')
#df = df.sample(fraction = 0.79)
#df.write.mode("overwrite").saveAsTable('data_gen'+'.'+'skew_dist_1')

#datalib.fetch_data("https://s3.amazonaws.com/apache-zeppelin/tutorial/bank/bank.csv")
#normal_df = datalib.generate_small_user_dist_df('marital',['married','single'],[49,51])
#skew_df = datalib.generate_small_user_dist_df('marital',['married','single'],[10,90])
#print("=======================Going to generate data with required distribution=========================")
#datalib.store_data(session,skew_df,'data_gen','skew_dist')
#print("====================== Stored the data ==============")
#datalib.generate_huge_data(session,'data_gen','skew_dist',10*int(1e9),1000,1000)
#print("=====================Generated the Dataset :: ",datalib.calc_df_size(session,'data_gen','skew_dist_1'))

def increase_and_store_data(self,session,dbname,tab_name,offset,theta):
        df = session.sql("use " + dbname)
        df = session.sql("select * from " + tab_name)
        sample = df.limit(theta)
        for _ in range(offset):
            df = df.union(sample)
        df = df.union(df)
        df.write.mode("append").saveAsTable(dbname+'.'+tab_name)

#datalib.fetch_data("https://s3.amazonaws.com/apache-zeppelin/tutorial/bank/bank.csv")
#scene = create_scenarios()
#scene.load_and_groupBy(session,'data_gen','normal_dist','marital','duration')
#scene.load_and_groupBy(session,'data_gen','skew_dist_1','marital','duration')
#scene.check_dist(session,'data_gen','normal_dist','marital')
#print("=========================== Finished =========================")

#scene.load_and_groupBy(session,'data_gen','skew_dist_1','marital','duration')

#normal_df = datalib.generate_small_user_dist_df('marital',['married','single'],[49,51])
#print("======================= Going to generate data with required distribution =========================")
#datalib.store_data(session,normal_df,'data_gen','normal_dist_1')
#scene.load_and_groupBy(session,'data_gen','normal_dist_1','marital','duration')
#datalib.generate_huge_data(session,'data_gen','skew_dist',25*int(1e9),int(1e6))
#print("=====================Generated the Dataset :: ",datalib.calc_df_size(session,'data_gen','skew_dist'))

df1 = session.sql("select * from 1gb_data")
        use.do_groupby(df1,"marital","duration").show(truncate=False)
        print("Time taken to read 1GB data :: ",time.time()-start)

'''


import configs.bankList as banklist
import dataMapper as dataMapper
#import vocabulary

class ConfigLoader:
    def __init__(self, data, iban):
        self.__data = data
        self.__iban = iban
        self.__headings = []
        #self.__defaultStruct = vocabulary.Vocabulary().getDictionary()

    def getStructuredData(self):
        self.setHeadings()
        mapper = dataMapper.dataMapper(self.__data)
        mappedData = mapper.map()

        bankList = banklist.BankList(self.__iban)
        bankConfig = bankList.getBankConfig()

        accountNumberIdentifier = bankConfig.getAccountNumberID(self.__headings).lower()
        transactioDateIdentifier = bankConfig.getTransactionDateID(self.__headings).lower()
        amountIdentifier = bankConfig.getAmountID(self.__headings).lower()
        currencyIdentifier = bankConfig.getCurrencyID(self.__headings).lower()

        finalDataSet = []
        for transaction in mappedData:
            item = {
                'account_number': transaction[accountNumberIdentifier],
                'transaction_date': transaction[transactioDateIdentifier],
                'amount': transaction[amountIdentifier],
                'category': transaction[currencyIdentifier]
            }
            finalDataSet.append(item)

        return finalDataSet

    def setHeadings(self):
        self.__headings = self.__data[0]


"""
@Author:    chenghua.Wang
@file:      src/GUI/API/select_window.py
@brief:     select_window called from main window.
"""
import os
import sys

from PyQt5 import QtCore, QtGui, QtWidgets
from core.SqlMan import SqlSearch_DS

class Select_Window(QtWidgets.QDialog):
    def __init__(self, MainWindow, func_class):
        super().__init__()
        self.MainWindow = MainWindow
        self.func_class = func_class
        self.setupUi()

    def setupUi(self):
        self.setObjectName("Form")
        self.resize(300, 192)
        self.verticalLayout = QtWidgets.QVBoxLayout(self)
        self.verticalLayout.setObjectName("verticalLayout")
        self.verticalLayout_2 = QtWidgets.QVBoxLayout()
        self.verticalLayout_2.setObjectName("verticalLayout_2")
        self.comboBox = QtWidgets.QComboBox(self)
        self.comboBox.setObjectName("comboBox")
        self.verticalLayout_2.addWidget(self.comboBox)
        self.lineEdit = QtWidgets.QLineEdit(self)
        self.lineEdit.setObjectName("lineEdit")
        self.verticalLayout_2.addWidget(self.lineEdit)
        self.pushButton = QtWidgets.QPushButton(self)
        self.pushButton.setEnabled(True)
        self.pushButton.setObjectName("pushButton")
        self.verticalLayout_2.addWidget(self.pushButton)
        self.verticalLayout.addLayout(self.verticalLayout_2)
        # -- Add action
        self.pushButton.clicked.connect(self.actionBottonSelect)

        self.retranslateUi()
        QtCore.QMetaObject.connectSlotsByName(self)

    def retranslateUi(self):
        _translate = QtCore.QCoreApplication.translate
        self.setWindowTitle(_translate("Form", "Form"))
        self.pushButton.setText(_translate("Form", "Select"))

    def updateSelectBar(self, describe):
        self.comboBox.clear()
        self.comboBox.addItems(describe)

    def actionBottonSelect(self):
        CurrentText = self.comboBox.currentText()
        WhereText = self.lineEdit.text()
        self.lineEdit.setText("")
        _new_describe_ = self.MainWindow.TableStatus.describe
        LOC = """
_new_data_ = []
for item in self.MainWindow.TableStatus.data.m_row:
    if item[_new_describe_.index({a})] {b}:
        _new_data_.append(item)
self.MainWindow.SearchTable_childWindow.UpdateTableData(self.func_class, SqlSearch_DS(_new_data_), _new_describe_)
self.MainWindow.SearchTable_childWindow.show()
        """.format(
            a=("\""+CurrentText+"\""),
            b=WhereText,
        )
        exec(LOC)
        self.close()

    def show(self) -> None:
        return super().show()


import cv2
import numpy as np
import os

def harsh_thresh(file):
    """
    A function that gets executed when ever the image is a screen shot.
    It creates a harsh thresholding, that isolates the document to some degree.
    """
    print(file)
    img = cv2.imread(file)
    img = cv2.threshold(img, 250, 255, cv2.THRESH_TOZERO)[1]
    cv2.imwrite('uploads/0_thresh.jpg', img)
    path = '0_thresh.jpg'
    return path


import requests
import json

class NexmoTwoFactor:

    def __init__(self, API_KEY, API_SECRET, brand):
        self.API_KEY = API_KEY
        self.API_SECRET = API_SECRET
        self.brand = brand

    def getCode(self, number):
        r = requests.post(
            'https://api.nexmo.com/verify/json',
            data={
                'api_key': self.API_KEY,
                'api_secret': self.API_SECRET,
                'brand': self.brand,
                'number': number
            }
        )
        code = None
        try:
            code = r.json()['request_id']
        except KeyError:
            pass
        return code

    def verifyCode(self, request_id, code):
        r = requests.post(
            'https://api.nexmo.com/verify/check/json',
            data={
                'api_key': self.API_KEY,
                'api_secret': self.API_SECRET,
                'request_id':  request_id,
                'code': code
            }
        )
        return r.json()['status'] == '0'


import numpy as np
from generator import ip_stream_generator
import os

def get_drifts(chunk_num, n_drifts):
    interval = chunk_num/n_drifts
    offset = 0.5*interval

    drfs = [interval*i for i in range(n_drifts)]
    return np.array(drfs) + offset

directory = 'datasets'
n_features = 8
chunk_n = 200
samples = chunk_n * 250
drifts_n = 5
d_types = ['nearest','cubic']

files = os.listdir(directory)

gt = get_drifts(chunk_n, drifts_n)
np.save('streams/drf_idx', gt)

for f in files:
    if f[0] == '.':
        continue

    ds = np.genfromtxt("%s/%s" % (directory, f), delimiter=",")
    X = ds[:, :-1]
    y = ds[:, -1].astype(int)

    for d in d_types:
        _X_s, _y = ip_stream_generator(X, y,
                        total_samples=samples,
                        stream_features=n_features,
                        random_state=13,
                        n_drifts=drifts_n,
                        interpolation=d)

        ds = np.concatenate((_X_s, _y[:, np.newaxis]), axis=1)
        np.save('streams/%s_%s' % (f.split('.')[0], d), ds)


import numpy as np
import math
from scipy.interpolate import RegularGridInterpolator, NearestNDInterpolator, LinearNDInterpolator

class vdf():

    def __init__(self, v_max, resolution, coord_sys):

        self.grid_cart = None
        self.grid_spher = None
        self.dvvv = None
        self.vdf_interp = np.zeros((resolution,resolution,resolution))
        grid_cart, grid_spher,
        grid_cyl, dvvv = self.init_grid(v_max, resolution, coord_sys)
        self.grid_cart_t = self.grid_cart.copy()
        self.grid_spher_t = self.grid_spher.copy()
        self.nb_counts = np.zeros((resolution,resolution,resolution))

    def interpolate_cart_vdf(self, grid, vdf0, interpolate='near'):

        if interpolate == 'near':
            method_str = 'nearest'
        elif interpolate == 'lin':
            method_str = 'linear'

        if interpolate in ['near', 'lin']:
            if vdf0.ndim==2:
                interpFunc = RegularGridInterpolator( (grid[0,:,0], grid[1,0,:]), vdf0,
                                                        bounds_error=False, method=method_str,
                                                        fill_value=np.nan)
                d = interpFunc(self.grid_cart[[0,2],:,:,0].reshape(2,-1).T) ## Ugly AF.
                d = d.reshape((self.vdf_interp.shape[0],self.vdf_interp.shape[0]))
                self.vdf_interp = d[:,:,None]
            elif vdf0.ndim==3:
                interpFunc = RegularGridInterpolator( (grid[0,:,0,0], grid[1,0,:,0], grid[2,0,0,:]), vdf0,
                                                        bounds_error=False, method=method_str,
                                                        fill_value=np.nan)
                d = interpFunc(self.grid_cart.reshape(3,-1).T)
                self.vdf_interp = d.T.reshape(self.vdf_interp.shape)  ## (res,res,res)

        # #
        # elif interpolate=='cub':
        #     d  = np.zeros_like(gridCart[0]).flatten()
        #     ip = tricubic.tricubic(list(distribPeriod), [distribPeriod.shape[0],distribPeriod.shape[1],distribPeriod.shape[2]])
        #     deltaSpeed = (np.log10(pSet.speed[iS,1]/normTh)-np.log10(pSet.speed[iS,0]/normTh))
        #     ds = pSet.speed[iS,1:]/normTh-pSet.speed[iS,:-1]/normTh
        #     deltaTheta = pSet.theta[1]-pSet.theta[0]
        #     deltaPhi   = pSet.phi[iS,1]-pSet.phi[iS,0]
        #     vMinSpeed  = np.log10(pSet.speed[iS,0]/normTh)
        #     vMinTheta  = 0.
        #     vMinPhi    = 0.
        #
        #     # gridS[0] = np.log10(gridS[0]) ## gridS here becomes an array of bin index, to which the coordinate belongs.
        #     # gridS[0] = (gridS[0]-vMinSpeed)/deltaSpeed #+ 1.5
        #     bin = np.digitize(gridS[0], pSet.speed[iS]/normTh)-1
        #     gridS[0] = bin + (gridS[0]-pSet.speed[iS,bin]/normTh)/ds[bin]
        #     gridS[1] = (gridS[1]-vMinTheta)/deltaTheta + .5
        #     gridS[2] = (gridS[2]-vMinPhi)/deltaPhi + .5
        #     for i, node in enumerate(gridS.reshape((3,-1)).T):
        #         d[i] = ip.ip(list(node))
        #     # itkTmp = (d<0)
        #     d = d.reshape((resFinal,resFinal,resFinal))
        #     d[gridS[0]<0] = np.nan ## "fill_value". Should also be done for values larger than, and not only smaller than.
        #     # d[itkTmp] = 0
        #     # sys.exit()
        #
        #_________________

    def interpolate_spher_vdf(self, grid, vdf0, interpolate='near', psp=False):

        speed = grid[0,:,0,0][::-1].copy()
        theta = grid[1,0,:,0].copy()
        phi   = grid[2,0,0,:].copy()
        vdf0 = np.flip(vdf0, axis=(0))

        if interpolate == 'near':
            interp_method = 'nearest'
        elif interpolate == 'lin':
            interp_method = 'linear'

        # itk = self.grid_spher[2]>np.pi
        # self.grid_spher[2,itk] -= 2.*np.pi
        if psp:
            phi -= 60.*np.pi/180.
            phi %= 2.*np.pi
            self.grid_spher_t[2] -= 60.*np.pi/180
            self.grid_spher_t[2] %= 2.*np.pi

        # phiPeriod       = np.zeros(18)
        # phiPeriod[1:-1] = phi
        # phiPeriod[0]    = phi[-1]-2*np.pi
        # phiPeriod[-1]   = phi[0]+2*np.pi
        # thetaPeriod       = np.zeros(10)
        # thetaPeriod[1:-1] = theta
        # thetaPeriod[0]    = theta[-1]-np.pi
        # thetaPeriod[-1]   = theta[0]+np.pi
        # distribPeriod              = np.zeros((32,10,18))
        # distribPeriod[:,1:-1,1:-1] = vdf0
        # distribPeriod[:,1:-1,0]    = vdf0[:,:,-1]
        # distribPeriod[:,1:-1,-1]   = vdf0[:,:,0]
        # distribPeriod[:,0]         = np.nanmean(distribPeriod[:,1], axis=1)[:,None]
        # distribPeriod[:,9]         = np.nanmean(distribPeriod[:,8], axis=1)[:,None]
        # itkR = ~np.isnan(speed)

        # interpFunc = RegularGridInterpolator( (speed, thetaPeriod, phiPeriod),
        #                                         distribPeriod,
        #                                         bounds_error=False, method=interp_method,
        #                                         fill_value=np.nan)
        interpFunc = RegularGridInterpolator( (speed, theta, phi),
                                                vdf0,
                                                bounds_error=False, method=interp_method,
                                                fill_value=np.nan)
        d = interpFunc(self.grid_spher_t.reshape(3,-1).T)
        d = d.T.reshape(self.vdf_interp.shape)  ## (res,res,res)
        d[np.isnan(d)] = 0.
        self.nb_counts += (~np.isnan(d))
        self.vdf_interp += d

    def transform_grid(self, R=None, v=None, s=None):

        if R is not None:
            gc = self.grid_cart.copy()
            self.grid_cart_t = np.dot(R, gc.reshape(3,-1)).reshape(self.grid_cart.shape)

        if v is not None:
            self.grid_cart_t -= v[:,None,None,None]

        self.grid_spher_t = self.cart2spher(self.grid_cart_t)

        # if interpolate=='near':
        #     interpFunc = RegularGridInterpolator( (speed, thetaPeriod, phiPeriod),
        #                                             (distribPeriod),
        #                                             bounds_error=False, method='nearest',
        #                                             fill_value=np.nan)
        #     d = interpFunc(self.grid_spher_t.reshape(3,-1).T)
        #     d = d.T.reshape(self.vdf_interp.shape)  ## (res,res,res)
        #     d[np.isnan(d)] = 0.
        #     self.vdf_interp += d
        #     print(np.nanmin(d), np.nanmax(d))
        #
        #
        #
        # elif interpolate=='lin':
        #     interpFunc = RegularGridInterpolator( (speed, thetaPeriod, phiPeriod),
        #                                             (distribPeriod),
        #                                             bounds_error=False, method='linear',
        #                                             fill_value=np.nan)
        #     # interpFunc = RegularGridInterpolator( (speed, theta, phi),
        #     #                                         vdf0,
        #     #                                         bounds_error=False, method='linear',
        #     #                                         fill_value=np.nan)
        #     d = interpFunc(self.grid_spher_t.reshape(3,-1).T)
        #     d = d.T.reshape(self.vdf_interp.shape)  ## (res,res,res)
        #     d[np.isnan(d)] = 0.
        #     self.vdf_interp += d
        #
        # elif interpolate=='cub':
        #     d  = np.zeros_like(gridCart[0]).flatten()
        #     ip = tricubic.tricubic(list(distribPeriod), [distribPeriod.shape[0],distribPeriod.shape[1],distribPeriod.shape[2]])
        #     deltaSpeed = (np.log10(pSet.speed[iS,1]/normTh)-np.log10(pSet.speed[iS,0]/normTh))
        #     ds = pSet.speed[iS,1:]/normTh-pSet.speed[iS,:-1]/normTh
        #     deltaTheta = pSet.theta[1]-pSet.theta[0]
        #     deltaPhi   = pSet.phi[iS,1]-pSet.phi[iS,0]
        #     vMinSpeed  = np.log10(pSet.speed[iS,0]/normTh)
        #     vMinTheta  = 0.
        #     vMinPhi    = 0.
        #
        #     # gridS[0] = np.log10(gridS[0]) ## gridS here becomes an array of bin index, to which the coordinate belongs.
        #     # gridS[0] = (gridS[0]-vMinSpeed)/deltaSpeed #+ 1.5
        #     bin = np.digitize(gridS[0], pSet.speed[iS]/normTh)-1
        #     gridS[0] = bin + (gridS[0]-pSet.speed[iS,bin]/normTh)/ds[bin]
        #     gridS[1] = (gridS[1]-vMinTheta)/deltaTheta + .5
        #     gridS[2] = (gridS[2]-vMinPhi)/deltaPhi + .5
        #     for i, node in enumerate(gridS.reshape((3,-1)).T):
        #         d[i] = ip.ip(list(node))
        #     # itkTmp = (d<0)
        #     d = d.reshape((resFinal,resFinal,resFinal))
        #     d[gridS[0]<0] = np.nan ## "fill_value". Should also be done for values larger than, and not only smaller than.
        #     # d[itkTmp] = 0
        #     # sys.exit()

        # if psp:
        #     self.grid_spher_t[2] += 60.*np.pi/180
        #     self.grid_spher_t[2] %= 2.*np.pi
        #
        #_________________

def init_grid(v_max, resolution, grid_geom):
    """Here we define the bin edges and centers, depending on the chosen
    coordinate system."""
    if grid_geom == 'cart':
        edgesX = np.linspace(-v_max, v_max, resolution + 1,
                             dtype=np.float32)
        centersX = (edgesX[:-1] + edgesX[1:]) * .5
        # 3 x res x res_phi x res/2
        grid_cart = np.mgrid[-v_max:v_max:resolution*1j,
                             -v_max:v_max:resolution*1j,
                             -v_max:v_max:resolution*1j]
        grid_cart = grid_cart.astype(np.float32)
        grid_spher = cart2spher(grid_cart)
        grid_cyl = cart2cyl(grid_cart)
        dv = centersX[1]-centersX[0]
        dvvv = np.ones((resolution, resolution, resolution)) * dv ** 3

    elif grid_geom == 'spher':
        edges_rho = np.linspace(0, v_max, resolution + 1, dtype=np.float32)
        edges_theta = np.linspace(0, np.pi, resolution + 1,
                                  dtype=np.float32)
        edges_phi = np.linspace(0, 2*np.pi, resolution + 1,
                                dtype=np.float32)
        centers_rho = (edges_rho[:-1] + edges_rho[1:]) * .5
        centers_theta = (edges_theta[:-1] + edges_theta[1:]) * .5
        centers_phi = (edges_phi[:-1] + edges_phi[1:]) * .5
        grid_spher = np.mgrid[centers_rho[0]:centers_rho[-1]:centers_rho.size*1j,
                              centers_theta[0]:centers_theta[-1]:centers_theta.size*1j,
                              centers_phi[0]:centers_phi[-1]:centers_phi.size*1j]
        grid_spher = grid_spher.astype(np.float32)
        grid_cart = spher2cart(grid_spher)
        grid_cyl = cart2cyl(grid_cart)
        d_rho = centers_rho[1]-centers_rho[0]
        d_theta = centers_theta[1]-centers_theta[0]
        d_phi = centers_phi[1]-centers_phi[0]
        dv = centers_rho[1]-centers_rho[0]
        dvvv = np.ones((resolution, resolution, resolution)) \
        * centers_rho[:, None, None] * d_rho * d_theta * d_phi

    elif grid_geom == 'cyl':
        edges_rho = np.linspace(0, v_max, resolution+1, dtype=np.float32)
        edges_phi = np.linspace(0, 2*np.pi, resolution+1, dtype=np.float32)
        edges_z = np.linspace(-v_max, v_max, resolution+1, dtype=np.float32)
        centers_rho = (edges_rho[:-1]+edges_rho[1:])*.5
        centers_phi = (edges_phi[:-1]+edges_phi[1:])*.5
        centers_z = (edges_z[:-1]+edges_z[1:])*.5
        grid_cyl = np.mgrid[centers_rho[0]:centers_rho[-1]:centers_rho.size*1j,
                            centers_phi[0]:centers_phi[-1]:centers_phi.size*1j,
                            centers_z[0]:centers_z[-1]:centers_z.size*1j]
        grid_cyl = grid_cyl.astype(np.float32)
        grid_cart = cyl2cart(grid_cyl)
        grid_spher = cart2spher(grid_cart)
        dRho = centers_rho[1]-centers_rho[0]
        dPhi = centers_phi[1]-centers_phi[0]
        dZ = centers_z[1]-centers_z[0]
        dvvv = np.ones((resolution, resolution, resolution)) \
        * centers_rho[:, None, None]*dRho*dPhi*dZ

    return grid_cart, grid_spher, grid_cyl, dvvv

def spher2cart(v_spher):
    """Coordinate system conversion
    """
    v_cart = np.zeros_like(v_spher)
    v_cart[0] = v_spher[0] * np.sin(v_spher[1]) * np.cos(v_spher[2])
    v_cart[1] = v_spher[0] * np.sin(v_spher[1]) * np.sin(v_spher[2])
    v_cart[2] = v_spher[0] * np.cos(v_spher[1])

    return v_cart

def cart2spher(v_cart):
    """Coordinate system conversion
    """
    v_spher = np.zeros_like(v_cart)
    v_spher[0] = np.sqrt(np.sum(v_cart ** 2, axis=0))
    v_spher[1] = np.arccos(v_cart[2] / v_spher[0])
    v_spher[2] = np.arctan2(v_cart[1], v_cart[0])
    itm = (v_spher[2] < 0.)
    v_spher[2][itm] += 2*np.pi

    return v_spher

def cyl2cart(v_cyl):
    """Coordinate system conversion
    """
    v_cart = np.zeros_like(v_cyl)
    v_cart[0] = v_cyl[0]*np.cos(v_cyl[1])
    v_cart[1] = v_cyl[0]*np.sin(v_cyl[1])
    v_cart[2] = v_cyl[2].copy()

    return v_cart

def cart2cyl(v_cart):
    """Coordinate system conversion
    """
    v_cyl = np.zeros_like(v_cart)
    v_cyl[0] = np.sqrt(v_cart[0]**2+v_cart[1]**2)
    v_cyl[1] = np.arctan2(v_cart[1], v_cart[0])
    v_cyl[2] = v_cart[2].copy()
    itm = (v_cyl[1] < 0.)
    v_cyl[1][itm] += 2*np.pi

    return v_cyl

def R_2vect(vector_orig, vector_fin):
    """
    Taken from:
    https://github.com/Wallacoloo/printipi/blob/master/util/rotation_matrix.py
    Calculate the rotation matrix required to rotate from one vector to another.
    For the rotation of one vector to another, there are an infinit series of
    rotation matrices possible.  Due to axially symmetry, the rotation axis
    can be any vector lying in the symmetry plane between the two vectors.
    Hence the axis-angle convention will be used to construct the matrix
    with the rotation axis defined as the cross product of the two vectors.
    The rotation angle is the arccosine of the dot product of the two unit vectors.
    Given a unit vector parallel to the rotation axis, w = [x, y, z] and the rotation angle a,
    the rotation matrix R is::
              |  1 + (1-cos(a))*(x*x-1)   -z*sin(a)+(1-cos(a))*x*y   y*sin(a)+(1-cos(a))*x*z |
        R  =  |  z*sin(a)+(1-cos(a))*x*y   1 + (1-cos(a))*(y*y-1)   -x*sin(a)+(1-cos(a))*y*z |
              | -y*sin(a)+(1-cos(a))*x*z   x*sin(a)+(1-cos(a))*y*z   1 + (1-cos(a))*(z*z-1)  |

    Parameters
    ----------
    R
        The 3x3 rotation matrix to update.
    vector_orig
        The unrotated vector defined in the reference frame.
    vector_fin
        The rotated vector defined in the reference frame.
    """

    # Convert the vectors to unit vectors.
    vector_orig = vector_orig / np.linalg.norm(vector_orig)
    vector_fin = vector_fin / np.linalg.norm(vector_fin)

    # The rotation axis (normalised).
    axis = np.cross(vector_orig, vector_fin)
    axis_len = np.linalg.norm(axis)
    if axis_len != 0.0:
        axis = axis / axis_len

    # Alias the axis coordinates.
    x = axis[0]
    y = axis[1]
    z = axis[2]

    # The rotation angle.
    angle = math.acos(np.dot(vector_orig, vector_fin))

    # Trig functions (only need to do this maths once!).
    ca = np.cos(angle)
    sa = np.sin(angle)

    # Calculate the rotation matrix elements.
    Rot_mat = np.zeros((3, 3))
    Rot_mat[0, 0] = 1.0 + (1.0 - ca)*(x**2 - 1.0)
    Rot_mat[0, 1] = -z*sa + (1.0 - ca)*x*y
    Rot_mat[0, 2] = y*sa + (1.0 - ca)*x*z
    Rot_mat[1, 0] = z*sa+(1.0 - ca)*x*y
    Rot_mat[1, 1] = 1.0 + (1.0 - ca)*(y**2 - 1.0)
    Rot_mat[1, 2] = -x*sa+(1.0 - ca)*y*z
    Rot_mat[2, 0] = -y*sa+(1.0 - ca)*x*z
    Rot_mat[2, 1] = x*sa+(1.0 - ca)*y*z
    Rot_mat[2, 2] = 1.0 + (1.0 - ca)*(z**2 - 1.0)

    return Rot_mat


import plotly.express as px
import numpy as np
import pandas as pd

data = pd.read_pickle('outputsmoothed5.pkl')

figure = px.choropleth(data,
                       locations='alpha',
                       color='birth rate',
                       animation_frame='year',
                       hover_name='country name',
                       range_color=[1, 9],
                       title='Fertility rate per country. 1955-2020 Historical Data, 2021-2100 UN Projection')
figure.show()
with open('index.html','w') as file:
    file.write(figure.to_html())


"""
Script to automatically download radiosondes from University of Wyoming website
"""

import requests
from io import StringIO
import pandas as pd
import os
from dotenv import load_dotenv

load_dotenv()

station_id = {'Ny-Alesund': '01004',
              'Singapore': '48698',
              'Essen': '10410',
              'Barbados': '78954'}

id_station = {v: k for k, v in station_id.items()}

if __name__ == '__main__':

    # set stations
    name = {'Ny-Alesund': 'ny_alesund',
            'Singapore': 'singapore',
            'Essen': 'essen',
            'Barbados': 'barbados'}

    # set times
    dates = pd.date_range(start='2019/01/01 12:00',
                          end='2019/12/31 12:00',
                          freq='24H')

    colnames = ['p [hPa]', 'z [m]', 'T [C]', 'T_dew [C]', 'RH [%]', 'r [g/kg]',
                'wdir [deg]',  'SKNT [knot]', 'THTA [K]', 'THTE [K]',
                'THTV [K]']

    stations = station_id.keys()

    for station in stations:

        print(station)

        for date in dates:

            print(date)

            date_file = date.strftime('%Y%m%d%H%M')

            day = date.strftime('%d')
            month = date.strftime('%m')
            year = date.strftime('%Y')
            hour = date.strftime('%H')

            # get data
            base = 'http://weather.uwyo.edu/cgi-bin/sounding?'
            link = base+'region=np&TYPE=TEXT%3ALIST&YEAR='+year+'&MONTH='+\
                month+'&FROM='+day+hour+'&TO='+day+hour+'&STNM='+\
                    station_id[station]
            html = requests.get(link).text

            try:
                # get station name and time from header
                header = html.split('
')[1].split('
')[0]

                data_str = html.split('
')[1].split('
')[0]

                data = pd.read_fwf(StringIO(data_str), skiprows=5, header=None,
                                   names=colnames)

                file = os.path.join(
                    os.environ['PATH_ATM'],
                    year, month, day,
                    'ID_'+station_id[station]+'_'+date_file+'.txt'
                    )

                if not os.path.exists(os.path.dirname(file)):
                    os.makedirs(os.path.dirname(file))

                with open(file, 'w') as f:
                    f.write('# '+ header + '\n')
                    data.to_csv(f, index=False)

            except IndexError:
                print('Can not find data for time {}'.format(date))


# -*- coding: utf-8 -*-
import scrapy
import json
from ..items import WalmartProductDetails

class WalmartDetailSpider(scrapy.Spider):
    """
    scrapy spider to crawl single walmart detail page using walmart products api
    input:inputDetailUrl
    you can run it using something like that
    scrapy crawl walmart_detail -o output_file.json -a category='product/api/12480393?selected=true'
    category attribute will be concatenated to 'https://www.walmart.com/' to build the start_urls list
    """
    name = "walmart_detail"
    allowed_domains = ["walmart.com"]

    def __init__(self, category='', domain=None, *args, **kwargs):
        """walmart spider constructor, build the start_urls list from category attribute"""
        super(WalmartDetailSpider, self).__init__(*args, **kwargs)
        self.start_urls = ['https://www.walmart.com/%s' % category]

    def parse(self, response):
        """
        in this function, the spider parse the crawled pages and build the walmart item
        """
        walmart_item = WalmartProductDetails()
        #init walmart_item
        walmart_item['product_name'] = ""
        walmart_item['product_url'] = ""
        walmart_item['manufacturer_sku'] = ""
        walmart_item['product_image_url'] = ""
        walmart_item['product_description'] = ""
        walmart_item['product_image_count'] = ""
        walmart_item['product_color'] = ""
        walmart_item['site_sku'] = ""
        walmart_item['product_manufacturer'] = ""
        walmart_item['product_inStock_flag'] = ""
        walmart_item['product_upc'] = ""
        walmart_item['product_available'] = ""
        walmart_item['seller_name'] = ""
        walmart_item['seller_url'] = ""
        walmart_item['product_onlinePrice'] = ""
        walmart_item['product_specifications'] = ""
        walmart_item['product_online_descriptor'] = ""
        ######################################################################################
        jsonresponse = json.loads(response.body_as_unicode())
        if "product" in jsonresponse:
            if "productName" in jsonresponse["product"]:
                walmart_item['product_name'] = jsonresponse["product"]["productName"]
            if "canonicalUrl" in jsonresponse["product"]:
                walmart_item['product_url'] = jsonresponse['product']['canonicalUrl']
            if "manufacturerProductId" in jsonresponse["product"]:
                walmart_item['manufacturer_sku'] = jsonresponse['product']['manufacturerProductId']
            if "primaryImageUrl" in jsonresponse["product"]:
                walmart_item['product_image_url'] = jsonresponse["product"]["primaryImageUrl"]
            if "longDescription" in jsonresponse["product"]:
                walmart_item['product_description'] = jsonresponse["product"]["longDescription"]
            if "imageAssets" in jsonresponse["product"]:
                walmart_item['product_image_count'] = len(jsonresponse["product"]["imageAssets"])
            if "variantInformation" in jsonresponse["product"] and "variantTypes" in jsonresponse["product"]["variantInformation"]:
                walmart_item['product_color'] = self.get_color(jsonresponse["product"]["variantInformation"]["variantTypes"])
        if "analyticsData" in jsonresponse:
            if "productId" in jsonresponse['analyticsData']:
                walmart_item['site_sku'] = jsonresponse['analyticsData']['productId']
            if "brand" in jsonresponse['analyticsData']:
                walmart_item['product_manufacturer'] = jsonresponse['analyticsData']['brand']
            if "inStock" in jsonresponse['analyticsData']:
                walmart_item['product_inStock_flag'] = jsonresponse["analyticsData"]["inStock"]
            if "upc" in jsonresponse['analyticsData']:
                walmart_item['product_upc'] = jsonresponse["analyticsData"]["upc"]
            if "productSellersMap" in jsonresponse['analyticsData']:
                sellers = jsonresponse["analyticsData"]["productSellersMap"]
                walmart_item['product_available'] = self.get_product_available_sellers(sellers)
                walmart_item['seller_name'] = self.get_sellers_names(sellers)
                #walmart_item['seller_url'] = self.get_sellers_urls(sellers)
                walmart_item['product_onlinePrice'] = self.get_product_onlinePrices(sellers)
            ##########################################################################################################
        if "idml" in jsonresponse and "specifications" in jsonresponse["idml"] and "specAttributes" in jsonresponse["idml"]["specifications"]:
            walmart_item['product_specifications'] = self.get_specifications(jsonresponse["idml"]["specifications"]["specAttributes"])

        yield walmart_item

    def get_specifications(self, specAttributes):
        """
        build a dictionary of displayName:displayValue for each item in specAttributes list
        return type: string representation of the result dictionary
        """
        result = {}
        for attribute in specAttributes:
            result[attribute["displayName"]] = attribute["displayValue"]

        return str(result)

    def get_color(self, variantTypes):
        """
        return a list of available colors for the crawled item
        """
        result = []
        for item_type in variantTypes:
            result.append(item_type["selectedValue"])

        return result

    def get_product_available_sellers(self, sellers):
        """
        return a list of available sellers for the crawled item
        """
        result = []
        for seller in sellers:
            result.append(seller["isAvail"])
        return result

    def get_sellers_names(self, sellers):
        """
        return a list of sellers names and offerId for the crawled item
        """
        result = []
        for seller in sellers:
            result.append(seller["sellerName"] + "_" + seller["offerId"])
        return result

    def get_sellers_urls(self, sellers):
        """
        return a list of sellers URLs for the crawled item
        """
        result = []
        for seller in sellers:
            result.append(seller["sellerId"])
        return result

    def get_product_onlinePrices(self, sellers):
        """
        return a list of available prices for the crawled item
        """
        result = []
        for seller in sellers:
            result.append(seller["price"])
        return result


def caesar(text, shift_amount, mode):

    new_text = ''

    if mode == 'encode':
        cipher_direction = 1
    elif mode == 'decode':
        cipher_direction = -1

    for letter in text:
        if letter in alphabet:
            position = alphabet.index(letter)

            shifted_position = position + (cipher_direction * shift_amount)

            if shifted_position > len(alphabet)-1 or shifted_position < 0:
                shifted_position = shifted_position + (-1 * cipher_direction * len(alphabet))

            new_text += alphabet[shifted_position]

        else:
            new_text += letter

    return new_text

if __name__ == '__main__':
    alphabet = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
                'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']

    is_stop = False
    while not is_stop:
        direction = input('Type \'encode\' to encrypt, type \'decode\' to decrypt:\n')

        if direction not in ['encode', 'decode']:
            print('Please type only \'encode\' or \'decode\'.')
            exit()

        text = input('Type your message:\n').lower()
        shift = int(input('Type the shift number:\n')) % 26

        new_text = caesar(text=text, shift_amount=shift, mode=direction)
        print(f'The {direction}d text is {new_text}')

        answer = input('ㅅType \'yes\' if you want to do again. Otherwise type \'no\': ').lower()
        if answer == 'no':
            print('Goodbye.')
            is_stop = True
        elif answer == 'yes':
            continue
        else:
            print('Wrong input.')
            is_stop = True


import os
import matplotlib.pyplot as plt
import sis_utils
from ersa_utils import read_tensorboard_csv

img_dir, task_dir = sis_utils.get_task_img_folder()
file_name_temp = 'run_unet_aemo_pad_PS(572, 572)_BS5_EP130_LR0.001_DS100_DR0.1-tag-{}_loss.csv'
run_type = ['train', 'valid']

plt.figure(figsize=(8, 6))

for cnt, run in enumerate(run_type):
    ax = plt.subplot(211 + cnt)
    file_name = file_name_temp.format(run)
    step, value = read_tensorboard_csv(os.path.join(task_dir, file_name), 5, 2)
    plt.plot(step, value, linewidth=2)
    plt.xlabel('Step')
    plt.ylabel('Xent')
    plt.grid(True)
    plt.title('{} Loss'.format(run.capitalize()))

plt.tight_layout()
plt.savefig(os.path.join(img_dir, 'train_crve_pad_hist.png'))
plt.show()


import math
import sys
from typing import Optional
from Leetcode.NonLinear import _tree_input

class TreeNode:
    def __init__(self, val=0, left=None, right=None):
        self.val = val
        self.left = left
        self.right = right

class Solution:
    prev = -sys.maxsize
    res = sys.maxsize
    def minDiffInBST(self, root: Optional[TreeNode]) -> int:
        if root.left:
            self.minDiffInBST(root.left)
        self.res = min(self.res, root.val - self.prev)
        self.prev = root.val

        if root.right:
            self.minDiffInBST(root.right)

        return self.res

    def minDiffInBST(self, root: TreeNode) -> int:
        def dfs(node, prev):
            if not node:
                return prev

            prev = dfs(node.left, prev)

            if prev is not None:
                self.min_dist = min(self.min_dist, node.val - prev)

            return dfs(node.right, node.val)

        self.min_dist = math.inf
        dfs(root, None)

        return self.min_dist

if __name__ == "__main__":
    t = Solution()
    root = [90,69,None,49,89,None,52]
    root = _tree_input.input_order(root)
    # _tree_input.preOrder(root)
    print(t.minDiffInBST(root))


# usr/bin/python3
# pip3 install wxpy

from time import sleep
from wxpy import *

def welcome():
    print("\n-----------------------------------------------\n")
    print("欢迎使用微信群群发工具(by Vic)")
    print("注意：该工具和电脑端微信无法同时登陆")
    input("按Enter键开始登陆，如需切换账号，请删除wxpy.pkl文件")

def menu(bot):
    print("\n-----------------------------------------------\n")
    print("1. 创建群发列表(第一次使用必选)")
    print("2. 群发消息\n")
    print("0. 输出好友状况")
    print("q. 退出程序")

    choice = input("\n请输入你需要的操作: ")

    if choice == "0":
        print("\n-----------------------------------------------\n")
        print(bot.friends().stats_text())
    elif choice == "1":
        create_send_list(bot)
    elif choice == "2":
        send_msg_to_list(bot)
    else:
        exit()

def create_send_list(bot):
    print("\n-----------------------------------------------\n")

    file_name = input("你要保存的微信群列表文件(list.txt)：") or "list.txt"
    key_word = input("请输入微信群名称关键字，空则为所有会话：")

    chats = []
    print("一些不活跃的群可能无法被获取到，将群加入到通讯录中来激活")
    print("搜索到的微信群列表：\n")
    chats += bot.groups().search(key_word)

    with open(file_name, 'w', encoding='utf8') as f:
        for chat in chats:
            try:
                f.write(chat.name + '\n')
                print("\t" + chat.name)
            except:
                pass

    print("\n一共搜索到{}个符合条件的会话，请在{}中查看或者修改".format(len(chats), file_name))

def send_msg_to_list(bot):
    print("\n-----------------------------------------------\n")

    print("请输入你要发送的内容")
    print("由 前缀 和 内容 两个部分组成，若省略前缀，将作为纯文本消息发送")
    print("前缀 部分可为: ‘@fil@’, ‘@img@’, ‘@msg@’, ‘@vid@’ (不含引号)")
    print("分别表示: 文件，图片，纯文本，视频")
    print("内容 部分可为: 文件、图片、视频的路径，或纯文本的内容\n")
    content = input()

    print("你要发送的消息是：\n\n")
    print(content)
    print("\n-----------------------------------------------\n")

    if input("请确认 (y)/n:") == 'n':
        return

    file_name = input("微信群列表文件(list.txt)：") or "list.txt"

    print("确认要发送的会话有：\n")
    chats = []
    groups = bot.groups()
    with open(file_name, 'r', encoding='utf8') as f:
        for line in f:
            chat = ensure_one(groups.search(line))
            chats.append(chat)
            print(chat.name)

    if input("\n一共将发送到{}个群聊 (y)/n:".format(len(chats))) == 'n':
        return

    print("")
    succ = 0
    fail = 0
    for chat in chats:
        # 微信限制不能发送太快
        sleep(1)
        try:
            chat.send(content)
            print("成功发送给：{}".format(chat.name))
            succ += 1
        except:
            print("发送给{}失败！！！！！！".format(chat.name))
            fail += 1
            pass

    print("\n发送成功：{}个，发送失败{}个".format(succ, fail))

if __name__ == "__main__":
    welcome()
    bot = Bot(cache_path=True)
    while True:
        menu(bot)


hello = "Hello World!"

xabar = "Nima gaplar ?"
print(xabar)
xabar = "Yangi xabar!"
print(xabar)

# class = "bo'lmagan ish"
# print(class)

radius = 5
pi = 3.14
area = pi * radius**2
print("Radiusi ", radius, " bo'lgan aylananing yuzi ", area, " ga teng.")


from fastapi import FastAPI, Header, Response, Request
from starlette.responses import JSONResponse

from pi import build_pi_get_resource_metrics_response
from rds import build_rds_describe_db_instances_response

app = FastAPI()

@app.get("/")
def read_root():
    return {"pong": "hello"}

@app.post("/")
async def read_aws(request: Request, content_type: str = Header(default="")):
    action = await get_action(request)
    if action == "PerformanceInsightsv20180227.GetResourceMetrics":
        return await build_pi_get_resource_metrics_response(request)
    elif action == "DescribeDBInstances":
        return build_rds_describe_db_instances_response()
    else:
        return build_unknown_action_response(content_type, action)

async def get_action(request: Request) -> str:
    x_amz_target = request.headers.get("X-Amz-Target", "")
    form = await request.form()
    return form.get("Action", x_amz_target)

def build_unknown_action_response(content_type: str, action: str) -> Response:
    if content_type == "application/x-www-form-urlencoded":
        content = f"""
<ErrorResponse xmlns="http://rds.amazonaws.com/doc/2014-10-31/">
  <Error>
    <Type>Sender</Type>
    UnknownAction
    <Message>Unknown Action: {action}</Message>
  </Error>
  <RequestId>799fa280-3701-465f-8b26-49edcf814748</RequestId>
</ErrorResponse>
"""
        return Response(status_code=400, content=content, media_type="text/xml")
    else:
        detail = {"__type": "UnknownTargetException","Message": f"Unknown X-Amz-Target: {action}"}
        return JSONResponse(status_code=400, content=detail)


from fastapi import APIRouter
from typing import List
from models.product import Product

from services.product_sevices import delete_all_products, get_product_by_id, get_products_by_category, save_products, get_products_with_purchases, search_products

router = APIRouter(
  prefix="/api/products",
  tags=["products"]
)

@router.post('/')
async def create_product(products: List[Product]):
  return save_products(products)

@router.get('/category/{category}')
async def view_products(category: str):
  return get_products_by_category(category)

@router.get('/popular/list')
async def popular_prodcuts():
  return get_products_with_purchases()

@router.get('/product/{id}')
async def view_product(id: str):
  return get_product_by_id(id)

@router.delete('/delete_all')
async def remove_product():
  return delete_all_products()

@router.get('/search/{query}')
async def search_products_by_query(query: str):
  return search_products(query)

# @router.delete('/delete_product')
# async def remove_product(id: str):
#   product_delete = collection.find_one_and_delete({"_id": ObjectId(id)})
#   if not product_delete:
#     return {"error": "No se pudo eliminar el usuario"}

#   return product_delete


from openmeteo_py.Utils.constants import *
from openmeteo_py.Utils.timezones import *

class ElevationOptions():
    """

    The API Options accepts a WGS4 coordinate and other  weather variables .
    Time always starts at 0:00 today and contains 168 hours.

    """
    def __init__(self, latitude, longitude):
        """
        Args:
            latitude (float): Latitude (Geographical WGS84 coordiante of the location).
            longitude (float): Longitude (Geographical WGS84 coordiante of the location).
        Raises:
            ValueError: Raises when latitude is not between -90 and 90 degrees.
            ValueError: Raises when longitude is not between -180 and 180 degrees.
        """
        if latitude < -90 or latitude > 90:
            raise ValueError("Latitude should be between -90 and 90 degrees.")
        if longitude < -180 or longitude > 180:
            raise ValueError("Longitude should be between -180 and 180 degrees.")

        self.latitude = latitude
        self.longitude = longitude


import pandas as pd

def get_TF(lst: pd.Series):
    bag = list(map(lambda x: str(x).split(), lst))

    TF = pd.Series(bag).apply(lambda x: pd.Series(x).value_counts())

    TF_ordered = TF.sort_index(axis=1)

    return TF_ordered

def get_cooccur(lst: pd.Series):
    TF = get_TF(lst)
    TF = TF.fillna(0)
    cooccurr = TF.transpose().dot(TF)

    return cooccurr


from pycocotools.coco import COCO
import numpy as np
import skimage
import skimage.io as io
import skimage.color as color
import matplotlib.pyplot as plt
import pylab

import sys

DEBUG=True

def computeMetrics(maskBase,maskCompare):
  b_maskBase = maskBase>0
  b_maskCompare = maskCompare>0

  true_positive = b_maskCompare[ b_maskBase ].sum()

  false_positive =  b_maskCompare.sum() - true_positive
  false_negative = b_maskBase.sum() - true_positive

  return {
    "precision": true_positive / (true_positive + false_positive), #b_maskCompare.sum()
    "recall": true_positive / (true_positive + false_negative) #b_maskBase.sum()
   }

def findMoreLikelyAnnotation(cocoImg,maskImg):
  annIds = coco.getAnnIds(imgIds=cocoImg['id'])
  anns = coco.loadAnns(annIds)

  aRank={}
  for i,annotation in enumerate(anns):
    cocoMask = coco.annToMask(annotation)

    M = computeMetrics(cocoMask,maskImg)
    aRank[i] = M["precision"] + M["recall"]

  highestMatch = sorted(aRank.items(),key=lambda item: item[1],reverse=True)[0]
  if DEBUG:
    print("Highest Match: ",highestMatch)

  return anns[ highestMatch[0] ]

def compareMaskWithCocoMask(cocoImgId,maskPath):
  cocoImg = coco.loadImgs(cocoImgId)[0]

  maskImg = io.imread(maskPath,as_gray=True)
  maskImg = skimage.img_as_bool(maskImg)

  ann = findMoreLikelyAnnotation(cocoImg,maskImg)
  cocoMask = coco.annToMask(ann)

  alo = color.gray2rgb(cocoMask)
  alo*=255

  return computeMetrics(cocoMask,maskImg)

COCO_DATA_DIR=sys.argv[1]
COCO_DATA_TYPE=sys.argv[2]
OUTPUT_METRICS_FILE_PATH=sys.argv[3]

annFile='{}/annotations/instances_{}.json'.format(COCO_DATA_DIR,COCO_DATA_TYPE)
coco=COCO(annFile)

with open(OUTPUT_METRICS_FILE_PATH,"w+") as f:
  f.write("Id,gc-Precision,gc-Recall,gc-Mixed,ccd-Precision,ccd-Recall,ccd-Mixed,ccnd-Precision,ccnd-Recall,ccnd-Mixed\n")

  for i in range(4,len(sys.argv)-3,4):
    cocoImgId = int(sys.argv[i])
    label = cocoImgId
    if cocoImgId == 200000:
      cocoImgId = 44699

    correctedMaskPath = sys.argv[i+1]
    correctedNoDataMaskPath = sys.argv[i+2]
    gcMaskPath = sys.argv[i+3]

    M_gc=compareMaskWithCocoMask(cocoImgId,gcMaskPath)
    M_cd=compareMaskWithCocoMask(cocoImgId,correctedMaskPath)
    M_cnd=compareMaskWithCocoMask(cocoImgId,correctedNoDataMaskPath)

    f.write(
      "{},{:.4f},{:.4f},{:.4f},{:.4f},{:.4f},{:.4f},{:.4f},{:.4f},{:.4f}\n".format(
          label,
          M_gc["precision"],M_gc["recall"],0.5*(M_gc["precision"]+M_gc["recall"]),
          M_cd["precision"],M_cd["recall"],0.5*(M_cd["precision"]+M_cd["recall"]),
          M_cnd["precision"],M_cnd["recall"],0.5*(M_cnd["precision"]+M_cnd["recall"])
      )
    )


import json
import os
paperdirectory = "/Users/fernandodurier/Desktop/sarcasm-paper-review"
data=[]

def graphPapers(papers):
    ##
    # clean-authors
    # clean-refs
    # title
    # path.split('\/')[len(path.split('\/'))]
    vertrixes=[] #set of works
    edges=[] #set of tuples, (source, target)
    jedges=[] #set of edges for jgraph lib
    jvertrixes={} #set of edges for jgraph lib
    for p in papers:
        ptitle=p['title']
        if ptitle:
            ptitle=ptitle.replace('\n', ' ')
            ptitle=ptitle.strip().lstrip().rstrip()

        vertrixes.append(ptitle)

        for r in p['clean-refs']:
            rtitle=r['title']
            if rtitle:
                rtitle=rtitle.replace('\n', ' ')
                rtitle=rtitle.strip().lstrip().rstrip()

            if rtitle not in vertrixes:
                vertrixes.append(rtitle)

            ptuple=(vertrixes.index(ptitle), vertrixes.index(rtitle))
            jgraphdict={"source":vertrixes.index(ptitle), "target":vertrixes.index(rtitle)}
            if vertrixes.index(ptitle) not in jvertrixes:
                jvertrixes[vertrixes.index(ptitle)] = {'name':ptitle,'path':p['path'],'color': 0xffaaaa, 'size': 3.0}
            if vertrixes.index(rtitle) not in jvertrixes:
                jvertrixes[vertrixes.index(rtitle)] = {'name':rtitle,'path':p['path'],'color': 0x2222ff, 'size': 1.50}
            if jgraphdict not in jedges:
                jedges.append(jgraphdict)
            if ptuple not in edges:
                edges.append(ptuple)

    print("Vertrixes: ",vertrixes)
    print("Edges: ", edges)
    print("N_Vertrixes: ", len(vertrixes))
    print("N_Edges: ", len(edges))
    print("Jvertrixes:", jvertrixes)
    print("Jedges:", jedges)
    return {"edges_array":edges, "vertrix_array":vertrixes,"nodes":jvertrixes,"edges":jedges}

for root, dirs, files in os.walk(paperdirectory, topdown=False):
   for name in files:
        if "-deep.json" in name:
            with open(paperdirectory+"/"+name) as f:
                d = json.load(f)
                data.append(d)

gf = graphPapers(data)

import jgraph
jgraph.draw(gf)

# with open(paperdirectory+'/cross-ref-jgraph.json', 'w') as outfile:
#          json.dump(gf, outfile, indent=4)


from autogpt.command_decorator import command

COMMAND_CATEGORY = "mock"

@command(
    "function_based",
    "Function-based test command",
    {
        "arg1": {"type": "int", "description": "arg 1", "required": True},
        "arg2": {"type": "str", "description": "arg 2", "required": True},
    },
)
def function_based(arg1: int, arg2: str) -> str:
    """A function-based test command that returns a string with the two arguments separated by a dash."""
    return f"{arg1} - {arg2}"


from django.shortcuts import render , redirect
from django.contrib.auth.forms import UserCreationForm
from django.contrib.auth.models import User
from django.contrib.auth.views import LoginView
from .models import Room , Attendance , Notification
from . forms import UserEditForm , AttendanceForm , Roomform , NotiicationForm
from datetime import date

from django.urls import reverse_lazy
from django.contrib.auth.decorators import login_required
from django.db.models import Q

from django.http import HttpResponse

# Create your views here.

class CustomLoginView(LoginView):
    template_name = 'base/login.html'
    fields = '__all__'
    redirect_authenticated_user = True

    def get_success_url(self):
        return reverse_lazy('home')

def RegisterPage(request):
    form = UserCreationForm
    if request.method == 'POST':
        form = UserCreationForm(request.POST)
        if form.is_valid():
            form.save()

            return redirect('home')
    context = {'form':form}
    return render(request,'base/register.html',context)

@login_required(login_url=  'login')
def EditProfile(request , pk):
    user = User.objects.get(id=pk)
    form = UserEditForm(instance=user)
    if not request.user.groups.filter(name='Admin').exists():
        return HttpResponse("YOU ARE NOT ALLOWED HERE", status=403)
    if request.method == 'POST':
        form = UserEditForm(request.POST,instance=user)
        if form.is_valid():

            form.save()
            return redirect ( 'home' )

    context = {'form' : form}
    return render (request , 'base/edit.html' , context)

def CreateAttendanceRoom(request):
    form = Roomform()
    if not request.user.groups.filter(name='Admin').exists():
        return HttpResponse("YOU ARE NOT ALLOWED HERE", status=403)
    if request.method=='POST':
        form = Roomform(request.POST)
        if form.is_valid():
            form.save()
        return redirect('home')
    context = {'form':form}
    return render (request , 'base/createroom.html' , context)

def CreateAttendance(request):
    if Attendance.objects.filter(user=request.user, date=date.today()).exists():
        return HttpResponse("You have already done the attendance for today.")

    form = AttendanceForm()
    if request.method == "POST":
        form = AttendanceForm(request.POST)
        if form.is_valid():
            form.save()
            att= form.save(commit=False)
            att.user = request.user
            att.save()
            return redirect('home')

    context = {'form':form}
    return render(request,'base/attend.html' , context )

def AttendanceList(request):
    q = request.GET.get('q', '')

    attendance = Attendance.objects.filter (

        Q(date__icontains=q) |
        Q(user__username__icontains=q)

    )

    if not request.user.groups.filter(name='Admin').exists():
        return HttpResponse("YOU ARE NOT ALLOWED HERE", status=403)
    context = {'attendance' : attendance}
    return render(request,'base/attendancelist.html' , context)

def home(request):
    user = User.objects.all()

    room = Room.objects.all()
    notification = Notification.objects.all()
    context = {'users':user , 'room':room  , 'notifications':notification}

    return render (request , 'base/home.html' , context)

def index(request , pk):
    room=Room.objects.get(id=pk)
    context = {'room':room}
    return render(request , 'base/index.html' , context)

def UserView(request):
    user = User.objects.all()
    if not request.user.groups.filter(name='Admin').exists():
        return HttpResponse("YOU ARE NOT ALLOWED HERE", status=403)
    context = {'user':user}
    return render (request , 'base/users.html' , context)

def Deleteuser(request , pk):
    user = User.objects.get(id=pk)

    if request.method=='POST':
        user.delete()

        return redirect('home')
    context={'obj':user}
    return render(request,'base/delete.html',context)

def Deleteroom(request , pk):
    room = Room.objects.get(id=pk)
    if request.method=='POST':
        room.delete()
        return redirect('home')
    context={'obj':room}
    return render(request,'base/delete.html',context)

def ViewAttendance(request):
    user = request.user
    info = user.attendance_set.all()
    context={'info':info}
    return render(request , 'base/viewatt.html' , context)

def CreateNotification(request):
    form = NotiicationForm()
    if request.method=='POST':
        form=NotiicationForm(request.POST)
        if form.is_valid():
            form.save()
        return redirect('home')
    context={'form':form}
    return render(request,'base/notification.html', context)

def DeleteNotification(request , pk):
    notification = Notification.objects.get(id=pk)
    if request.method=='POST':
        notification.delete()
        return redirect('home')
    context={'obj':notification}
    return render(request,'base/delete.html',context)


import threading
import time

def producer():
    time.sleep(1)
    print("producer is producing event 1st time!")
    event.set()

    # time.sleep(1)
    print("producer is clearing event")
    event.clear()

    # time.sleep(1)
    # print("producer is producing event 2nd time!")
    # event.set()

def consumer():
    print("concumer is waiting for event 1st time...")
    event.wait()
    print("consumer found event 1st time!")

    print("concumer is waiting for event 2nd time...")
    event.wait()
    print("consumer found event 2nd time!")

event = threading.Event()

thread_producer = threading.Thread(target=producer)
thread_consumer = threading.Thread(target=consumer)

thread_producer.start()
thread_consumer.start()

thread_producer.join()
thread_consumer.join()