import ipdb
import numpy as np
import theano
import theano.tensor as T

from cle.cle.data import Iterator
from cle.cle.cost import Gaussian
from cle.cle.models import Model
from cle.cle.layers import InitCell
from cle.cle.layers.feedforward import FullyConnectedLayer
from cle.cle.layers.layer import PriorLayer
from cle.cle.layers.recurrent import LSTM
from cle.cle.train import Training
from cle.cle.train.ext import (
    EpochCount,
    GradientClipping,
    Monitoring,
    Picklize,
    EarlyStopping,
    WeightNorm
)
from cle.cle.train.opt import Adam
from cle.cle.utils import flatten, sharedX
from cle.cle.utils.compat import OrderedDict
from sk.datasets.blizzard_h5 import Blizzard_h5_tbptt

data_path = '/raid/chungjun/data/blizzard_unseg/'
save_path = '/raid/chungjun/repos/sk/cle/models/nips2015/blizzard/pkl/'
#data_path = '/home/junyoung/data/blizzard_unseg/'
#save_path = '/home/junyoung/repos/sk/cle/models/nips2015/blizzard/pkl/'

reset_freq = 4
batch_size = 128
mn_batch_size = 1280
frame_size = 200
latent_size = 200
encoder_dim = 3100
decoder_dim = 3100
target_size = frame_size
lr = 3e-4
debug = 0

file_name = 'blizzard_unseg_tbptt'
normal_params = np.load(data_path + file_name + '_normal.npz')
X_mean = normal_params['X_mean']
X_std = normal_params['X_std']

model = Model()
train_data = Blizzard_h5_tbptt(name='train',
                               path=data_path,
                               frame_size=frame_size,
                               file_name=file_name,
                               X_mean=X_mean,
                               X_std=X_std)

valid_data = Blizzard_h5_tbptt(name='valid',
                               path=data_path,
                               frame_size=frame_size,
                               file_name=file_name,
                               X_mean=X_mean,
                               X_std=X_std)

init_W = InitCell('rand')
init_U = InitCell('ortho')
init_b = InitCell('zeros')
init_b_sig = InitCell('const', mean=0.6)

x = train_data.theano_vars()
mn_x = valid_data.theano_vars()
if debug:
    x.tag.test_value = np.zeros((15, batch_size, frame_size), dtype=theano.config.floatX)
    mn_x.tag.test_value = np.zeros((15, mn_batch_size, frame_size), dtype=theano.config.floatX)
x_tm1 = T.concatenate([T.zeros((1, x.shape[1], x.shape[2])), x[:-1]], axis=0)
x_tm1.name = 'x_tm1'
mn_x_tm1 = T.concatenate([T.zeros((1, mn_x.shape[1], mn_x.shape[2])), mn_x[:-1]], axis=0)
mn_x_tm1.name = 'mn_x_tm1'

encoder = LSTM(name='encoder',
               parent=['x_t'],
               parent_dim=[frame_size],
               batch_size=batch_size,
               nout=encoder_dim,
               unit='tanh',
               init_W=init_W,
               init_U=init_U,
               init_b=init_b)

decoder = LSTM(name='decoder',
               parent=['x_tm1', 'z_t'],
               parent_dim=[frame_size, latent_size],
               batch_size=batch_size,
               nout=decoder_dim,
               unit='tanh',
               init_W=init_W,
               init_U=init_U,
               init_b=init_b)

phi_mu = FullyConnectedLayer(name='phi_mu',
                             parent=['enc_t'],
                             parent_dim=[encoder_dim],
                             nout=latent_size,
                             unit='linear',
                             init_W=init_W,
                             init_b=init_b)

phi_sig = FullyConnectedLayer(name='phi_sig',
                              parent=['enc_t'],
                              parent_dim=[encoder_dim],
                              nout=latent_size,
                              unit='softplus',
                              cons=1e-4,
                              init_W=init_W,
                              init_b=init_b_sig)

prior = PriorLayer(name='prior',
                   parent=['phi_mu', 'phi_sig'],
                   parent_dim=[latent_size, latent_size],
                   use_sample=1,
                   num_sample=1,
                   nout=latent_size)

kl = PriorLayer(name='kl',
                parent=['phi_mu', 'phi_sig'],
                parent_dim=[latent_size, latent_size],
                use_sample=0,
                nout=latent_size)

theta_mu = FullyConnectedLayer(name='theta_mu',
                               parent=['dec_t'],
                               parent_dim=[decoder_dim],
                               nout=target_size,
                               unit='linear',
                               init_W=init_W,
                               init_b=init_b)

theta_sig = FullyConnectedLayer(name='theta_sig',
                                parent=['dec_t'],
                                parent_dim=[decoder_dim],
                                nout=target_size,
                                unit='softplus',
                                cons=1e-4,
                                init_W=init_W,
                                init_b=init_b_sig)

nodes = [encoder, decoder, prior, kl,
         phi_mu, phi_sig, theta_mu, theta_sig]

for node in nodes:
    node.initialize()

params = flatten([node.get_params().values() for node in nodes])

step_count = sharedX(0, name='step_count')
last_encoder = np.zeros((batch_size, encoder_dim*2), dtype=theano.config.floatX)
last_decoder = np.zeros((batch_size, decoder_dim*2), dtype=theano.config.floatX)
encoder_tm1 = sharedX(last_encoder, name='encoder_tm1')
decoder_tm1 = sharedX(last_decoder, name='decoder_tm1')
update_list = [step_count, encoder_tm1, decoder_tm1]

step_count = T.switch(T.le(step_count, reset_freq), step_count + 1, 0)
enc_0 = T.switch(T.or_(T.cast(T.eq(step_count, 0), 'int32'),
                       T.cast(T.eq(T.sum(encoder_tm1), 0.), 'int32')),
                 encoder.get_init_state(batch_size), encoder_tm1)
dec_0 = T.switch(T.or_(T.cast(T.eq(step_count, 0), 'int32'),
                       T.cast(T.eq(T.sum(decoder_tm1), 0.), 'int32')),
                 decoder.get_init_state(batch_size), decoder_tm1)

x_shape = x.shape
x_in = x.reshape((x_shape[0]*x_shape[1], -1))

def inner_fn(x_t, x_tm1, enc_tm1, dec_tm1):

    enc_t = encoder.fprop([[x_t], [enc_tm1]])

    phi_mu_t = phi_mu.fprop([enc_t])
    phi_sig_t = phi_sig.fprop([enc_t])

    z_t = prior.fprop([phi_mu_t, phi_sig_t])

    dec_t = decoder.fprop([[x_tm1, z_t], [dec_tm1]])

    return enc_t, dec_t, phi_mu_t, phi_sig_t

((enc_t, dec_t, phi_mu_t, phi_sig_t), updates) =\
    theano.scan(fn=inner_fn,
                sequences=[x, x_tm1],
                outputs_info=[enc_0, dec_0, None, None])

for k, v in updates.iteritems():
    k.default_update = v

encoder_tm1 = enc_t[-1]
decoder_tm1 = dec_t[-1]

dec_shape = dec_t.shape
dec_in = dec_t.reshape((dec_shape[0]*dec_shape[1], -1))
theta_mu_in = theta_mu.fprop([dec_in])
theta_sig_in = theta_sig.fprop([dec_in])

z_shape = phi_mu_t.shape
phi_mu_in = phi_mu_t.reshape((z_shape[0]*z_shape[1], -1))
phi_sig_in = phi_sig_t.reshape((z_shape[0]*z_shape[1], -1))
kl_in = kl.fprop([phi_mu_in, phi_sig_in])
kl_t = kl_in.reshape((z_shape[0], z_shape[1]))

recon = Gaussian(x_in, theta_mu_in, theta_sig_in)
recon = recon.reshape((x_shape[0], x_shape[1]))
recon_term = recon.mean()
kl_term = kl_t.mean()
nll_lower_bound = recon_term + kl_term
nll_lower_bound.name = 'nll_lower_bound'

mn_x_shape = mn_x.shape
mn_x_in = mn_x.reshape((mn_x_shape[0]*mn_x_shape[1], -1))

mn_enc_0 = encoder.get_init_state(mn_batch_size)
mn_dec_0 = decoder.get_init_state(mn_batch_size)

((mn_enc_t, mn_dec_t, mn_phi_mu_t, mn_phi_sig_t), mn_updates) =\
    theano.scan(fn=inner_fn,
                sequences=[mn_x, mn_x_tm1],
                outputs_info=[mn_enc_0, mn_dec_0, None, None])

for k, v in mn_updates.iteritems():
    k.default_update = v

mn_dec_shape = mn_dec_t.shape
mn_dec_in = mn_dec_t.reshape((mn_dec_shape[0]*mn_dec_shape[1], -1))
mn_theta_mu_in = theta_mu.fprop([mn_dec_in])
mn_theta_sig_in = theta_sig.fprop([mn_dec_in])

mn_z_shape = mn_phi_mu_t.shape
mn_phi_mu_in = mn_phi_mu_t.reshape((mn_z_shape[0]*mn_z_shape[1], -1))
mn_phi_sig_in = mn_phi_sig_t.reshape((mn_z_shape[0]*mn_z_shape[1], -1))
mn_kl_in = kl.fprop([mn_phi_mu_in, mn_phi_sig_in])
mn_kl_t = mn_kl_in.reshape((mn_z_shape[0], mn_z_shape[1]))

mn_recon = Gaussian(mn_x_in, mn_theta_mu_in, mn_theta_sig_in)
mn_recon = mn_recon.reshape((mn_x_shape[0], mn_x_shape[1]))
mn_recon_term = mn_recon.mean()
mn_kl_term = mn_kl_t.mean()
mn_nll_lower_bound = mn_recon_term + mn_kl_term
mn_nll_lower_bound.name = 'nll_lower_bound'
mn_recon_term.name = 'recon_term'
mn_kl_term.name = 'kl_term'

mn_kl_ratio = mn_kl_term / T.abs_(mn_recon_term)
mn_kl_ratio.name = 'kl_term proportion'

max_x = mn_x.max()
mean_x = mn_x.mean()
min_x = mn_x.min()
max_x.name = 'max_x'
mean_x.name = 'mean_x'
min_x.name = 'min_x'

max_theta_mu = mn_theta_mu_in.max()
mean_theta_mu = mn_theta_mu_in.mean()
min_theta_mu = mn_theta_mu_in.min()
max_theta_mu.name = 'max_theta_mu'
mean_theta_mu.name = 'mean_theta_mu'
min_theta_mu.name = 'min_theta_mu'

max_theta_sig = mn_theta_sig_in.max()
mean_theta_sig = mn_theta_sig_in.mean()
min_theta_sig = mn_theta_sig_in.min()
max_theta_sig.name = 'max_theta_sig'
mean_theta_sig.name = 'mean_theta_sig'
min_theta_sig.name = 'min_theta_sig'

max_phi_sig = mn_phi_sig_t.max()
mean_phi_sig = mn_phi_sig_t.mean()
min_phi_sig = mn_phi_sig_t.min()
max_phi_sig.name = 'max_phi_sig'
mean_phi_sig.name = 'mean_phi_sig'
min_phi_sig.name = 'min_phi_sig'

model.inputs = [x]
model._params = params
model.nodes = nodes
model.set_updates(update_list)

optimizer = Adam(
    lr=lr
)

mn_fn = theano.function(inputs=[mn_x],
                        outputs=[mn_nll_lower_bound, mn_recon_term, mn_kl_term, mn_kl_ratio,
                                 max_phi_sig, mean_phi_sig, min_phi_sig,
                                 max_theta_sig, mean_theta_sig, min_theta_sig,
                                 max_x, mean_x, min_x,
                                 max_theta_mu, mean_theta_mu, min_theta_mu],
                        on_unused_input='ignore')

extension = [
    GradientClipping(batch_size=batch_size),
    EpochCount(4),
    Monitoring(freq=1000,
               monitor_fn=mn_fn,
               ddout=[mn_nll_lower_bound, mn_recon_term, mn_kl_term, mn_kl_ratio,
                      max_phi_sig, mean_phi_sig, min_phi_sig,
                      max_theta_sig, mean_theta_sig, min_theta_sig,
                      max_x, mean_x, min_x,
                      max_theta_mu, mean_theta_mu, min_theta_mu],
               data=[Iterator(valid_data, mn_batch_size, start=2040064, end=2152704)]), #112640 is 5%
    Picklize(freq=1000, force_save_freq=10000, path=save_path),
    EarlyStopping(freq=1000, force_save_freq=10000, path=save_path),
    WeightNorm()
]

mainloop = Training(
    name='storn0_1',
    data=Iterator(train_data, batch_size, start=0, end=2040064),
    model=model,
    optimizer=optimizer,
    cost=nll_lower_bound,
    outputs=[nll_lower_bound],
    extension=extension
)
mainloop.run()


import functools
import itertools
import os
import pickle
import shutil

def batch(iter_, page_size):
    for _, batch in itertools.groupby(enumerate(iter_), lambda tuple_: tuple_[0] // page_size):
        yield [value for index, value in batch]

class PagedList:

    def __init__(self, list_, pages_dir, page_size, cache_size, exist_ok=False, open_func=open):
        self.pages_dir = pages_dir  # An empty or non-existant directory.
        self.page_size = page_size
        self.cache_size = cache_size
        self.open_func = open_func
        self._len = 0
        tmp_dir = pages_dir + ".tmp"
        if exist_ok:
            shutil.rmtree(tmp_dir, ignore_errors=True)
            shutil.rmtree(pages_dir, ignore_errors=True)
        os.makedirs(tmp_dir)
        index = 0
        for index, page in enumerate(batch(list_, page_size)):
            pickle_path = os.path.join(tmp_dir, str(index))
            with self.open_func(pickle_path, "wb") as file_:
                pickle.dump(page, file_, protocol=pickle.HIGHEST_PROTOCOL)
            self._len += len(page)
        self.page_count = index + 1
        os.rename(tmp_dir, self.pages_dir)
        self._setup_page_cache()

    def __len__(self):
        return self._len

    def _get_page_org(self, index):  # This is cached, see setup_page_cache.
        pickle_path = os.path.join(self.pages_dir, str(index))
        try:
            with self.open_func(pickle_path, "rb") as file_:
                return pickle.load(file_)
        except FileNotFoundError:
            raise IndexError

    def __getitem__(self, index):
        if isinstance(index, slice):
            start, stop, step = index.indices(self._len)
            start_page_index, start_page_offset = divmod(start, self.page_size)
            stop_page_index, stop_page_offset = divmod(stop, self.page_size)
            if stop_page_index == self.page_count:
                stop_page_index -= 1
                stop_page_offset = self.page_size
            if start_page_index == stop_page_index:
                return self._get_page(start_page_index)[start_page_offset:stop_page_offset]
            else:
                return (self._get_page(start_page_index)[start_page_offset:] +
                        [line for page_index in range(start_page_index+1, stop_page_index)
                         for line in self._get_page(page_index)] +
                        self._get_page(stop_page_index)[:stop_page_offset])
        else:
            page_index, page_offset = divmod(index, self.page_size)
            return self._get_page(page_index)[page_offset]

    def _setup_page_cache(self):
        self._get_page = functools.lru_cache(self.cache_size)(self._get_page_org)

    def __getstate__(self):  # Don't pickle the lru_cache.
        state = self.__dict__.copy()
        del state["_get_page"]
        return state

    def __setstate__(self, state):
        self.__dict__ = state
        self._setup_page_cache()


# Copyright 2021, Mycroft AI Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Representations and conversions of the data returned by the weather API."""
from datetime import timedelta
from pathlib import Path
from typing import List

from .config import MILES_PER_HOUR
from .util import convert_to_local_datetime

# Forecast timeframes
CURRENT = "current"
DAILY = "daily"
HOURLY = "hourly"

# Days of week
SATURDAY = 5
SUNDAY = 6

# Condition Icons (see https://openweathermap.org/weather-conditions)
#   Map each of the possible weather condition icon codes from OpenWeatherMap to an
#   image/animation file used by the GUI.  The icon codes contain a number and a letter.
#   A "n" after the number indicates night and a "d" indicates day.
#
#   The icon/image map is used by the Mark II, which does not use animations for
#   performance reasons.  The icon/animation map is used by the scalable QML.  The
#   icon/code map is for the Mark I, which accepts a code to determine what
#   is displayed.
ICON_IMAGE_MAP = (
    (("01d",), "sun.svg"),
    (("01n",), "moon.svg"),
    (("04d", "04n"), "clouds.svg"),
    (("50d",), "fog.svg"),
    (("02d", "03d"), "partial_clouds_day.svg"),
    (("02n", "03n"), "partial_clouds_night.svg"),
    (("09d", "10d"), "rain.svg"),
    (("13d",), "snow.svg"),
    (("11d",), "storm.svg"),
)
ICON_ANIMATION_MAP = (
    (("01d", "01n"), "sun.json"),
    (("04d", "04n"), "clouds.json"),
    (("50d",), "fog.json"),
    (("02d", "03d", "02n", "03n"), "partial_clouds.json"),
    (("09d", "10d"), "rain.json"),
    (("13d",), "snow.json"),
    (("11d",), "storm.json"),
)
ICON_CODE_MAP = (
    (("01d", "01n"), 0),
    (("04d", "04n"), 2),
    (("50d",), 7),
    (("02d", "03d", "02n", "03n"), 1),
    (("09d", "10d"), 3),
    (("13d",), 6),
    (("11d",), 5),
)

THIRTY_PERCENT = 30
WIND_DIRECTION_CONVERSION = (
    (22.5, "north"),
    (67.5, "northeast"),
    (112.5, "east"),
    (157.5, "southeast"),
    (202.5, "south"),
    (247.5, "southwest"),
    (292.5, "west"),
    (337.5, "northwest"),
)

class WeatherCondition:
    """Data representation of a weather conditions JSON object from the API"""

    def __init__(self, conditions: dict):
        self.id = conditions["id"]
        self.category = conditions["main"]
        self.description = conditions["description"]
        self.icon = conditions["icon"]

    @property
    def image(self) -> str:
        """Use the icon to image mapping to determine which image to display."""
        image_path = Path("images")
        for icons, image_file_name in ICON_IMAGE_MAP:
            if self.icon in icons:
                image_path = image_path.joinpath(image_file_name)

        return str(image_path)

    @property
    def animation(self) -> str:
        """Use the icon to animation mapping to determine which animation to display."""
        image_path = Path("animations")
        for icons, animation_file_name in ICON_ANIMATION_MAP:
            if self.icon in icons:
                image_path = image_path.joinpath(animation_file_name)

        return str(image_path)

    @property
    def code(self) -> str:
        """Use the icon to animation mapping to determine which animation to display."""
        condition_code = None
        for icons, code in ICON_CODE_MAP:
            if self.icon in icons:
                condition_code = code

        return condition_code

class Weather:
    """Abstract data representation of commonalities in forecast types."""

    def __init__(self, weather: dict, timezone: str):
        self.date_time = convert_to_local_datetime(weather["dt"], timezone)
        self.feels_like = weather["feelsLike"]
        self.pressure = weather["pressure"]
        self.humidity = weather["humidity"]
        self.dew_point = weather["dewPoint"]
        self.clouds = weather["clouds"]
        self.wind_speed = int(weather["windSpeed"])
        self.wind_direction = self._determine_wind_direction(weather["windDeg"])
        self.condition = WeatherCondition(weather["weather"][0])

    @staticmethod
    def _determine_wind_direction(degree_direction: int):
        """Convert wind direction from compass degrees to compass direction.

        Args:
            degree_direction: Degrees on a compass indicating wind direction.

        Returns:
            the wind direction in one of eight compass directions
        """
        for min_degree, compass_direction in WIND_DIRECTION_CONVERSION:
            if degree_direction < min_degree:
                wind_direction = compass_direction
                break
        else:
            wind_direction = "north"

        return wind_direction

    def determine_wind_strength(self, speed_unit: str):
        """Convert a wind speed to a wind strength.

        Args:
            speed_unit: unit of measure for speed depending on device configuration

        Returns:
            a string representation of the wind strength
        """
        if speed_unit == MILES_PER_HOUR:
            limits = dict(strong=20, moderate=11)
        else:
            limits = dict(strong=9, moderate=5)
        if self.wind_speed >= limits["strong"]:
            wind_strength = "strong"
        elif self.wind_speed >= limits["moderate"]:
            wind_strength = "moderate"
        else:
            wind_strength = "light"

        return wind_strength

class CurrentWeather(Weather):
    """Data representation of the current weather returned by the API"""

    def __init__(self, weather: dict, timezone: str):
        super().__init__(weather, timezone)
        self.sunrise = convert_to_local_datetime(weather["sunrise"], timezone)
        self.sunset = convert_to_local_datetime(weather["sunset"], timezone)
        self.temperature = round(weather["temp"])
        self.visibility = weather["visibility"]
        self.low_temperature = None
        self.high_temperature = None

class DailyFeelsLike:
    """Data representation of a "feels like" JSON object from the API"""

    def __init__(self, temperatures: dict):
        self.day = round(temperatures["day"])
        self.night = round(temperatures["night"])
        self.evening = round(temperatures["eve"])
        self.morning = round(temperatures["morn"])

class DailyTemperature(DailyFeelsLike):
    """Data representation of a temperatures JSON object from the API"""

    def __init__(self, temperatures: dict):
        super().__init__(temperatures)
        self.low = round(temperatures["min"])
        self.high = round(temperatures["max"])

class DailyWeather(Weather):
    """Data representation of a daily forecast JSON object from the API"""

    def __init__(self, weather: dict, timezone: str):
        super().__init__(weather, timezone)
        self.sunrise = convert_to_local_datetime(weather["sunrise"], timezone)
        self.sunset = convert_to_local_datetime(weather["sunset"], timezone)
        self.temperature = DailyTemperature(weather["temp"])
        self.feels_like = DailyFeelsLike(weather["feelsLike"])
        self.chance_of_precipitation = int(weather["pop"] * 100)

class HourlyWeather(Weather):
    """Data representation of a hourly forecast JSON object from the API"""

    def __init__(self, weather: dict, timezone: str):
        super().__init__(weather, timezone)
        self.temperature = round(weather["temp"])
        self.chance_of_precipitation = int(weather["pop"] * 100)

class WeatherAlert:
    """Data representation of a weather conditions JSON object from the API"""

    def __init__(self, alert: dict, timezone: str):
        self.sender = alert.get("sender_name")
        self.event = alert["event"]
        self.start = convert_to_local_datetime(alert["start"], timezone)
        self.end = convert_to_local_datetime(alert["end"], timezone)
        self.description = alert["description"]

class WeatherReport:
    """Full representation of the data returned by the Open Weather Maps One Call API"""

    def __init__(self, report):
        timezone = report["timezone"]
        self.current = CurrentWeather(report["current"], timezone)
        self.hourly = [HourlyWeather(hour, timezone) for hour in report["hourly"]]
        self.daily = [DailyWeather(day, timezone) for day in report["daily"]]
        today = self.daily[0]
        self.current.high_temperature = today.temperature.high
        self.current.low_temperature = today.temperature.low
        if "alerts" in report:
            self.alerts = [WeatherAlert(alert, timezone) for alert in report["alerts"]]
        else:
            self.alerts = None

    def get_weather_for_intent(self, intent_data):
        """Use the intent to determine which forecast satisfies the request.

        Args:
            intent_data: Parsed intent data
        """
        if intent_data.timeframe == "hourly":
            weather = self.get_forecast_for_hour(intent_data)
        elif intent_data.timeframe == "daily":
            weather = self.get_forecast_for_date(intent_data)
        else:
            weather = self.current

        return weather

    def get_forecast_for_date(self, intent_data):
        """Use the intent to determine which daily forecast(s) satisfies the request.

        Args:
            intent_data: Parsed intent data
        """
        if intent_data.intent_datetime.date() == intent_data.location_datetime.date():
            forecast = self.daily[0]
        else:
            delta = intent_data.intent_datetime - intent_data.location_datetime
            day_delta = int(delta / timedelta(days=1))
            day_index = day_delta + 1
            forecast = self.daily[day_index]

        return forecast

    def get_forecast_for_multiple_days(self, days: int) -> List[DailyWeather]:
        """Use the intent to determine which daily forecast(s) satisfies the request.

        Args:
            days: number of forecast days to return

        Returns:
            list of daily forecasts for the specified number of days

        Raises:
            IndexError when number of days is more than what is returned by the API
        """
        if days > 7:
            raise IndexError("Only seven days of forecasted weather available.")

        forecast = self.daily[1 : days + 1]

        return forecast

    def get_forecast_for_hour(self, intent_data):
        """Use the intent to determine which hourly forecast(s) satisfies the request.

        Args:
            intent_data: Parsed intent data

        Returns:
            A single hour of forecast data based on the intent data
        """
        delta = intent_data.intent_datetime - intent_data.location_datetime
        hour_delta = int(delta / timedelta(hours=1))
        hour_index = hour_delta + 1
        report = self.hourly[hour_index]

        return report

    def get_weekend_forecast(self):
        """Use the intent to determine which daily forecast(s) satisfies the request.

        Returns:
            The Saturday and Sunday forecast from the list of daily forecasts
        """
        forecast = list()
        for forecast_day in self.daily:
            report_date = forecast_day.date_time.date()
            if report_date.weekday() in (SATURDAY, SUNDAY):
                forecast.append(forecast_day)

        return forecast

    def get_next_precipitation(self, intent_data):
        """Determine when the next chance of precipitation is in the forecast.

        Args:
            intent_data: Parsed intent data

        Returns:
            The weather report containing the next chance of rain and the timeframe of
            the selected report.
        """
        report = None
        current_precipitation = True
        timeframe = HOURLY
        for hourly in self.hourly:
            if hourly.date_time.date() > intent_data.location_datetime.date():
                break
            if hourly.chance_of_precipitation > THIRTY_PERCENT:
                if not current_precipitation:
                    report = hourly
                    break
            else:
                current_precipitation = False

        if report is None:
            timeframe = DAILY
            for daily in self.daily:
                if daily.date_time.date() == intent_data.location_datetime.date():
                    continue
                if daily.chance_of_precipitation > THIRTY_PERCENT:
                    report = daily
                    break

        return report, timeframe


import sys, pygame
pygame.init()

size = width, height = 1080, 720
speed = [2, 2]
DX, DY = 0.1, 0.1
black = 0, 0, 0
screen = pygame.display.set_mode(size)

ball = pygame.image.load("intro_ball.gif")
ballrect = ball.get_rect()
x, y = width/2, height/2
w, h = ballrect.w/2, ballrect.h/2

pygame.time.set_timer(pygame.USEREVENT, 100)
pull = None

while 1:
    event = pygame.event.wait()
    if event.type == pygame.QUIT: sys.exit()
    elif event.type == pygame.MOUSEBUTTONDOWN or pull and event.type == pygame.MOUSEMOVE:
        pull = event.pos

    if x-w/2<0 or x+w/2 > width:
        speed[0] = -speed[0]
    if y-h < 0 or y+h > height:
        speed[1] = -speed[1]

    screen.fill(black)
    x += speed[0]
    y += speed[1]
    print (x, y)
    ballrect.move_ip(int(x*DX), int(y*DY))
    screen.blit(ball, ballrect)
    pygame.display.flip()


from selenium import webdriver
import time
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.support.select import Select
from selenium.webdriver.support.wait import WebDriverWait
import pytest

from PagesObject.HomePage import HomePage
from utilities.BaseClass import BaseClass

class Test_One(BaseClass):

    def test_e2e(self):

        log = self.get_Logger()
        home = HomePage(self.driver)
        time.sleep(3)
        home.shopItem().click()
        time.sleep(3)
        log.info("Getting all cards info")
        time.sleep(3)
        products = self.driver.find_elements_by_xpath("//div[@class='card h-100']")
        time.sleep(3)
        for product in products:
            productName = product.find_element_by_xpath("div/h4/a").text

            if productName == "Blackberry":
                # Add item into cart

                product.find_element_by_xpath("div/button").click()
        time.sleep(3)
        self.driver.find_element_by_css_selector("a[class*='btn-primary']").click()
        time.sleep(3)
        self.driver.find_element_by_xpath("//button[@class='btn btn-success']").click()
        time.sleep(3)
        self.driver.find_element_by_id("country").send_keys("ind")
        time.sleep(3)
        self.verifylinkPresence("India")
        time.sleep(3)
        self.driver.find_element_by_link_text("India").click()
        time.sleep(3)
        self.driver.find_element_by_xpath("//div[@class='checkbox checkbox-primary']").click()
        time.sleep(3)
        self.driver.find_element_by_css_selector("[type='submit']").click()
        time.sleep(3)
        successText = self.driver.find_element_by_class_name("alert-success").text
        time.sleep(3)

        assert "Success! Thank you!" in successText

        self.driver.get_screenshot_as_file("screen.png")


import pyshark
import matplotlib.pyplot as plt
import pickle

# override_prefs = {'ssl.keylog_file': "Default\\Default_sslkey2.log"}
#
# cap = pyshark.FileCapture("Default/Default_Selection2.pcapng",
#                           display_filter="(ssl.record.content_type==23 && tls)",
#                           override_prefs=override_prefs)

override_prefs = {'ssl.keylog_file': "Non_Default\\Trace1\\sslkey.log"}

cap = pyshark.FileCapture("Non_Default\\Trace1\\non_default1.pcapng",
                          display_filter="(ssl.record.content_type==23 && tls)",
                          override_prefs=override_prefs)

# print("Packets are loading...")
# cap.load_packets()
# packet_amount = len(cap)
# print("Total packets: ", packet_amount)

time_seq_list = []
packet_len_list = []

# # # capturing http or application layer response packet # # #

time_seq = 0
for pckt in cap:
    time_seq = float(pckt.frame_info.time_relative)

    if len(time_seq_list) == 0:
        packet_len = float(pckt.ip.len)
    else:
        if time_seq == time_seq_list[-1]:
            packet_len = float(pckt.ip.len) + packet_len_list[-1]
            continue
        else:
            packet_len = float(pckt.ip.len)

    time_seq_list.append(time_seq)
    packet_len_list.append(packet_len)

fig1 = plt.figure(1)
plt.plot(time_seq_list, packet_len_list, linewidth=0.5, color='green')
plt.title("Non Default Selection Packet Length vs Time Plot")
plt.xlabel("time")
plt.ylabel("Packet Length")
plt.show()


from django.http import request
from django.http.response import HttpResponseBadRequest, JsonResponse
from django.shortcuts import redirect, render
from App.models import Pkg_Form_Data, Influencer_Details, Account
from .Checkoutform import buyersform
import json
from .models import Checkout_Data, Buyers, Notification_Data, Project_Creation
from notifications.signals import notify

# Create your views here.

def Checkout(request,id):
    Split_Str = str(id).split("_",2)
    try:
        if Split_Str[0] == "Basic":
            Context = Pkg_Form_Data.objects.all().filter(Gig_Id = int(Split_Str[1])).values("Besic_Packages_Price","Besic_Packages_Name","Besic_Packages_Descriptions")
            Package = "Basic"
        elif Split_Str[0]== "std":
            Context = Pkg_Form_Data.objects.all().filter(Gig_Id = int(Split_Str[1])).values("Standered_Packeges_Price","Standered_Packeges_Description","Standered_Packeges_Name")
            #Context[0]["Pkg"] = "Sd"
            Package = "Standared"
        else:
            Context = Pkg_Form_Data.objects.all().filter(Gig_Id = int(Split_Str[1])).values("Premium_Packages_Price","Premium_Packages_Name","Premium_Packages_Descriptions")
            #Context[0]["Pkg"] = "Pm"
            Package = "Premium"
    except:
        Context = None

    if request.method == "POST":
        Id_User = request.user.id
        body = json.loads(request.body)
        print('Data is', body)
    #try:
        Split_string = str(body["id"]).split("_",2)
        print(Split_string)
        Pkg = Split_string[0]
        User_Receiver = Pkg_Form_Data.objects.all().filter(Gig_Id = int(Split_string[1])).values("User")[0]["User"]
        User_Recipent = Account.objects.get(id = User_Receiver)
        User_Name_Sender = Influencer_Details.objects.all().filter(User = Id_User).values("First_Name")[0]["First_Name"]
        notify.send(request.user,recipient= User_Recipent, verb= "{} Purchased Your {} Package, A project has been created Due to That!".format(User_Name_Sender,Pkg))
        Checkout_obj = Checkout_Data(User_id =Id_User,Ordered_Product = Split_string[1], Order_Product_Type = Split_string[0])
        Checkout_obj.save()
        Notify_Data = Notification_Data(Sender_id = Id_User, Receiver_id = User_Receiver, Topic = str(User_Name_Sender + "Purchased Your {} Package, A project has been created Due to That!".format(Pkg)))
        Notify_Data.save()
        Prject_obj = Project_Creation(Created_For = User_Receiver, Created_By = Id_User, Product_Id = Split_string[1],Product_Type = Split_string[0],is_Completed = False,Payment_Made =True,Payment_Released = False,Is_Delivery_Time = False, Is_Pyament_Released = False, Is_Payment_peding = False, Is_Disputed = False, Is_Realesed_Requested = False,Is_Proof_Url = False)
        Prject_obj.save()
        return redirect("/Gig")
    else:
        return render(request,"Payment.html",{"Context_Data":Context,"Pkg_Id":id,"Split1":Split_Str[0]})

def Checkout_Pay(request, id):
    pass

def Completeorder(request):
    body = json.loads(request.body)
    print('Data is', body)
    Split_string = str(body["id"]).split("_",2)
    return JsonResponse('payment completed!',safe=False)


from core import Currency
from threading import Thread
import time

def sub(name):
    while 1:
        try:
            paramDict = {'btc': ['btc', 'xl', '2',1],'eth': ['eth', 'xl', '3',1],'ltc': ['ltc', 'xl', '4',1],'etc': ['etc', 'xl', '5',1],
                         'bch': ['bch', 'xl', '6',1],'eos': ['eos', 'xl', '7',1],'xrp': ['xrp', 'xl', '8',1],}
            riskManageUrl = 'http://www.xlbex.net/api/Riskmanagement/fkMethod?cid='
            currency=Currency()
            currency.name=name
            currency.name2 = paramDict[name][0]
            currency.projectName = paramDict[name][1]
            #   currency.riskUrl = riskManageUrl+paramDict[name][2]
            # currency.sub_bian()
            # currency.sub_ok()
            currency.sub_huobi(paramDict[name][3])
        except Exception as e:
            print(e)
        time.sleep(10)

currency_list=['btc','eth','ltc','etc','bch','eos','xrp']
# currency_list=['eos']
for currency in currency_list:
    t1 = Thread(target=sub,args=(currency,))
    t1.start()
    print(currency, 'ok')
t1.join()


# coding: utf-8

from django.http import Http404
from django.contrib.auth.models import User

from zds.member.models import Profile
from django.contrib.auth import logout

def can_read_now(func):
    '''Decorator to check that the user can read now'''
    def _can_read_now(request, *args, **kwargs):
        try:
            profile = Profile.objects.get(user__pk = request.user.pk)
        except:
            # The user is a visitor
            profile = None

        if profile is not None:
            if not profile.can_read_now():
                logout(request)
                request.session.clear()
                raise Http404
        return func(request, *args, **kwargs)
    return _can_read_now

def can_write_and_read_now(func):
    '''Decorator to check that the user can read and write now'''
    def _can_write_and_read_now(request, *args, **kwargs):
        try:
            profile = Profile.objects.get(user__pk = request.user.pk)
        except:
            # The user is a visitor
            profile = None

        if profile is not None:
            if not profile.can_read_now() or not profile.can_write_now():
                raise Http404
        return func(request, *args, **kwargs)
    return _can_write_and_read_now


#PyCord Responder by RealistikDash
#From https://github.com/RealistikDash/PyCord
print("Initialising Pycord Responder")

#All the necessary imports
try:
	from Pycord import pycord
except ImportError:
	print("Could not import the Pycord module! Cannot continue!")
	exit()
try: #Attempts to import
	import discord
	import asyncio
	import random
	import time
	from Pycord import pycord
	import os
	from settings import * #Imports variables you set in the setup

except ImportError: #This is run if there is an error while importing
	pycord.errorLog("There was an error while importing! Pycord cannot continue. Make sure you ran setup.py before launching Pycord")
	print("")
	runSetup = input("Do you want to run the setup now? (y/N)")
	runSetup = runSetup.lower() #Turns runSetup into lower case letters

	if runSetup == "y":
		try:
			os.system("python setup.py")
			exit()
		except Exception: #In case the system uses "python3" for it's commands, that will happen
			try:
				os.system("python3 setup.py")
				exit()
			except Exception:
				pycord.errorLog("There was an error while attempting to run the setup. Pycord cannot proceed.")
				time.sleep(3)
				exit()

#Sets the window title
try:
	os.system("title Pycord Responder")

except Exception:
	pass
######################################

######################################

#Message list
cmdlist="""The currently available commands are:
Discord Based Commands
-/setgame		Sets your presence to a given input.
-/sendfile		Sends a file with a given name.
-/customsend		Lets you choose the variables such as the channel id, message and any other attributes.
-/changeuser		Changes your Pycord username (does not edit settings.py and therefore is temporary).
-/sendtxt		Sends the contents of a specified txt file.
-/embed 		Sends an embed with your selected variables (Usage: /embed [title] - [content])
-/changeid		Changes the channel id to which your messages are being sent to.
-/shrug			Sends a ¯\_(ツ)_/¯
-/ping			Measures the speed of your connection to discord (lower is better)
-/clear			Clears the screen
-/exit			Exits Pycord"""

exitMsg = """---------------------------------------------------------------------
Thank you for using Pycord!
Make sure to report any issues on GitHub.
The program will turn off in 3 seconds
---------------------------------------------------------------------"""
######################################

#Tips
tips = ["Is the Discord AIP blocked too? Try using repl.it and run Pycord there!"]
######################################

bot = discord.Client()

#Makes variables accessable in on_ready()
bot.pycordUsername = username
bot.channelId = channelId
#########################################

@bot.event
async def on_ready():
		try:
			channelId = bot.channelId
			username = bot.pycordUsername
		except Exception:
			pass

		###MISSING VARIABLE CHECK###
		try:
			channelId = channelId #checks if the variable can be accessed

		except Exception:
			errorForId = 1
			pycord.errorLog("Error loading channel id from file...")
			while errorForId == 1: #A loop that is run while the user is entering their new channel id
				channelId = input("Please enter the channel id: ")
				if pycord.checkNumber(channelId) == 'false': #checknumber returns true if it is a number and false if it itn't
					pycord.errorLog("Invalid channel id!")
				else:
					errorForId = 0
					break

		try:
			username = username #checks if the variable can be accessed

		except Exception:
			pycord.errorLog("Error getting username from settings.py...")
			usernameLoop = 1
			while usernameLoop == 1: #cheks for invalid usernames
				username = input("Enter your username: ")
				if username == "":
					pycord.errorLog("Invalid username!")
				if username == " ":
					pycord.errorLog("Invalid username!")
				else:
					usernameLoop = 0
					break

		channelId = discord.Object(id=channelId) #Makes the variable a discord object

		#####################################

		#Checks ping
		t1 = time.time()
		await bot.send_typing(channelId)
		t2 = time.time()
		ping = (t2-t1)*1000
		ping = round(ping, 2)
		######################################

		pycord.log("Logged into {}".format(bot.user.name)) #Sets game

		#Welcome message
		print("Welcome to Pycord {}!".format(username))
		print(cmdlist)
		print("---------------------------------------------------------------------")
		print("Connection info:")
		print("Logged into: {}".format(bot.user.name))
		print("Your ping is: {}".format(ping))
		print("---------------------------------------------------------------------")
		print(random.choice(tips))
		print("---------------------------------------------------------------------")
		######################################

		await bot.send_message(channelId, "**{}** logged into Pycord.".format(username))

		while True : #The main loop

			msg = input("Pycord> ")
			#Checks if the msg is empty
			if msg == "":
				pycord.errorLog("Cannot send empty message.")

			#Changes the presence
			elif msg.startswith("/setgame"):
				game = msg[9:]
				await bot.change_presence(game=discord.Game(name=game))
				pycord.log("Game set to {}".format(msg[9:]))

			#Send file command (Use: /sendfile file.png), all formats supported as long as it is under 8MB
			elif msg.startswith("/sendfile"):
				pycord.log("Sending file {}...".format(msg[10:]))
				try:
					await bot.send_file(channelId, msg[10:])

				except Exception:
					pycord.errorLog("An error occured while sending the file...")

			#Lets the user set the parameters for the send
			elif msg.startswith("/customsend"):
				pycord.log("Attempting custom send...")
				eval("""await bot.send_message({})""".format(msg[12:]))

			#Changes pycord username
			elif msg.startswith("/changeuser"):
				username = msg[12:]
				pycord.log("Username changed to {}".format(username))

			#Opens a text file and says it's contents in discord
			elif msg.startswith("/sendtxt"):
				try:
					file = open(msg[9:], 'r')
					pycord.log("Sending txt file...")
					try:
						await bot.send_message(channelId, """`[{}]` TXT FILE
""".format(username) + file.read())
					except Exception:
						pycord.errorLog("An error occured while sending the message...")
				except Exception:
					pycord.errorLog("An error occured opening the file...")

			#Change channel ID
			elif msg.startswith("/changeid"):
				channelIdNew = msg[10:]
				if pycord.checkNumber(channelIdNew) == false: #Checks if channelIdNew is a number or not. If it fails, the below is executed.
					pycord.errorLog("Could not change the id. Failed checkNumber check.")
				else:
					channelId = discord.Object(id=channelIdNew)
					pycord.log("Channel Id changed to {}".format(channelId))

			#Lists all the commands
			elif msg.startswith("/help"):
				print(cmdlist)

			#Exits PyCord
			elif msg.startswith("/exit"):
				print(exitMsg)
				time.sleep(3)
				await bot.send_message(channelId, "**{}** logged out of Pycord.".format(username))
				await bot.close() #Logs out
				break #Breaks the loop

			#Sends the shrug emote
			elif msg.startswith("/shrug"):
				try:
					await bot.send_message(channelId, "`[{}]` ¯\_(ツ)_/¯".format(username))
				except Exception:
					pycord.errorLog("An error occured while sending the message...")

			#A ping command
			elif msg.startswith("/ping"):
				try:
					t1 = time.time()
					await bot.send_typing(channelId)
					t2 = time.time()
					ping = (t2-t1)*1000
					ping = round(ping, 2)
					pycord.log("Your ping is {}ms".format(ping))

				except Exception:
					pycord.errorLog("An error occured while sending the message...")

			#Embed command
			elif msg.startswith("/embed"):
				args = msg[7:]
				try:
					title, content = args.split("-")
					content = content[1:]
					embed = discord.Embed(title=title, description=content)
					try:
						await bot.send_message(channelId, embed=embed)
					except Exception:
						pycord.errorLog("Failed sending the message")
				except Exception:
					pycord.errorLog("Splitting failed. Required argumends might have not been met. Make sure to split the title and content with a dash and space like '- '.")

			#Clears the screen
			elif msg.startswith("/clear"):
				try:
					os.system("cls") #windows variant
				except Exception:
					try:
						os.system("clear") #linux variant
					except Exception:
						pycord.errorLog("Clear failed! Possibly unsupported os!")

			#If none of the requirements above are met, send the message
			else:
				try:
					await bot.send_message(channelId, "`[{}]` {}".format(username, msg))
				except Exception:
					pycord.errorLog("An error occured while sending the message...")

		pycord.log("Loop broken")

try:
	bot.run(botToken) #Connects the bot.
except Exception:
	pycord.errorLog("Could not connect to Discord. Check your token. Login aborted.")
	time.sleep(3)
	exit()


import board
from adafruit_display_text import label
import terminalio

from led_status import LedStatus
from sensors import Sensors
from network_service import NetworkService
from display import Display
from plotter import Plotter

led = LedStatus()
lcd = Display(backlight_control=True, baudrate=8000000)
nws = NetworkService()
sns = Sensors(update_timeout=30.0, debug=True)

plotter = Plotter(lcd,
                  style="lines", #"dots"
                  mode="scroll", #"wrap"
                  screen_width=160, screen_height=80,
                  plot_width=112, plot_height=41)

plotter.display_on()
plotter.clear_all()
plotter.title = "Enviro+"
plotter.y_axis_lab = ""
# The range on graph will start at this value
plotter.y_range = (20, 60)
plotter.y_min_range = 1
# Sensor/data source is expected to produce data between these values
plotter.y_full_range = (0, 100)
plotter.channels = 3  # Can be between 1 and 3
plotter.channel_colidx = [0xffff00, 0x00ffff, 0xff0080]

@sns.on_update
def on_update(readings):
    led.show_air_quality(int(readings.pm2_5))
    lcd.update(readings)
    # plotter.data_add((readings.temperature, readings.pm2_5, readings.humidity))
    # nw.connect_and_send(readings)

while True:
    sns.run()


import os
import csv
import numpy as np
from .mfcc import mfcc_kaldi, mfcc_librosa
from .custom import acf, histogram, rms_energy, rms_energy_infra
from .utils import root_folder, concat_samples

CACHE_SIZE = 20

def get_available_features() -> tuple:
	feats = (mfcc_kaldi, mfcc_librosa, acf, histogram, rms_energy, rms_energy_infra)
	return tuple(f.__name__ for f in feats), feats

def read_dataset(name: str, partition: str, label: str=None) -> tuple:
	if partition.lower() not in ("train", "test"):
		raise ValueError("Partition must be 'train' or 'test'")
	files_root = os.path.join(root_folder(), "datasets", name, partition)
	if not os.path.exists(files_root):
		raise ValueError(f"Dataset does not exist: {name} ({partition})")

	with open(os.path.join(files_root, "labels.csv"), newline='') as f:
		reader = csv.reader(f)
		classes = next(reader)[1:]
		filenames, labels = zip(*((line[0], line[1:]) for line in reader))

	filenames = np.array([os.path.join(files_root, fn) for fn in filenames])
	classes = np.array(classes)
	labels = np.array(labels) != '0'

	if label is None:
		return filenames, classes, labels
	else:
		try:
			mask = labels[:, np.nonzero(classes == label)]
			return filenames[mask], classes, labels[mask]
		except ValueError as e:
			raise ValueError(f"Label {label} not found in dataset {name}") from e

def _filename(filenames: list, feature: str, feature_args: dict, vad: dict):
	from pickle import dumps
	from hashlib import sha256
	obj = (tuple(filenames), feature, tuple(sorted(feature_args.items())), tuple() if vad is None else tuple(sorted(vad.items())))
	_hash = sha256(dumps(obj)).hexdigest()
	return os.path.join(root_folder(), "features", "cache", str(_hash) + ".features")

def extract_features(filenames: list, feature: str, feature_args: dict, vad: dict=None,
		concatenate: bool=False, cache: bool=True):

	if cache:
		# Lookup features in cache and return them if found
		cache_filename = _filename(filenames, feature, feature_args, vad)

		if os.path.exists(cache_filename):
			from pickle import load
			with open(cache_filename, "rb") as f:
				cached_tuple, feature_vals, index = load(f)
			if (np.array(cached_tuple[0]) == filenames).all() and cached_tuple[1:] == (feature, feature_args, vad):
				return feature_vals, index
			else:
				del cached_tuple, feature_vals, index

	# Get feature function
	available_features = get_available_features()
	if feature not in available_features[0]:
		raise ValueError(f"Unrecognized feature {feature}")
	feature_func = available_features[1][available_features[0].index(feature)]

	if vad is None or len(vad) == 0:
		# Compute features on original files
		index = np.arange(len(filenames))
		feature_vals = feature_func(filenames, **feature_args)
	else:
		# Split files according to VAD output and perform feature extraction on each part
		from .vad import analyze_gen, smooth, splitaudio, write_tempfiles, clean_tempdir
		if "frame_length" not in vad or "aggressiveness" not in vad:
			raise ValueError("Argument 'vad' must contain 'frame_length' and 'aggressiveness'")

		gen = analyze_gen(filenames, vad["frame_length"], vad["aggressiveness"])
		if "smooth_n" in vad:
			gen = (
				(audio, smooth(activity, n=vad["smooth_n"], threshold=vad.get("smooth_treshold")))
					for audio, activity in gen)

		# TODO: this essentially copies the entire dataset to disk, which is ugly
		tempfiles = []
		tempdir = None
		for audio, activity in gen:
			segments = splitaudio(audio, vad["frame_length"], activity, vad.get("inverse", False), vad.get("min_length", 0))
			_tf, tempdir = write_tempfiles(segments, tempdir)
			tempfiles.append(_tf)

		index = np.array([i for i in range(len(tempfiles)) for j in range(len(tempfiles[i]))])

		tempfiles = [t for l in tempfiles for t in l]
		feature_vals = feature_func(tempfiles, **feature_args)
		clean_tempdir(tempdir)

	empty = sum(np.prod(v.shape) == 0 for v in feature_vals if isinstance(v, np.ndarray))
	if empty > 0:
		raise ValueError(f"{empty} feature value{'' if empty == 1 else 's'} of dimension 0")

	if cache:
		# Remove least recently used cache entries if needed
		cache_files = next(os.walk(os.path.join(root_folder(), "features", "cache")))
		cache_files = [os.path.join(cache_files[0], fn) for fn in cache_files[2] if fn.endswith(".features")]
		cache_files = sorted(cache_files, key=lambda fn: os.stat(fn).st_atime, reverse=True)
		if len(cache_files) > CACHE_SIZE - 1:
			for fn in cache_files[CACHE_SIZE-1:]:
				try:
					os.remove(fn)
				except OSError:
					pass

		from pickle import dump
		with open(cache_filename, "wb") as f:
			dump(((filenames, feature, feature_args, vad), feature_vals, index), f)

	if concatenate:
		feature_vals = concat_samples(feature_vals)

	return feature_vals, index

# For compability with older classifiers
def extract_dataset_features(name: str, partition: str, features: list, feature_args: list,
		concatenate: bool=False, cache: bool=True) -> dict:

	filenames, classes, labels = read_dataset(name, partition, None)
	feature_vals, _ = [extract_features(filenames, f, f_a, concatenate=False, cache=cache) for f, f_a in zip(features, feature_args)]
	res = dict()
	for label in classes:
		mask = labels[:, classes.index(label)]
		f_vals = []
		for i in range(len(feature_vals)):
			f_vals.append([sequence for sequence, bit in zip(feature_vals[i], mask) if bit])
		res[label] = f_vals

	del filenames, feature_vals, classes, labels

	if concatenate:
		for label in res:
			for i in range(len(res[label])):
				res[label][i] = concat_samples(res[label][i])

	return res


import pandas as pd
import numpy as np
from load_data import load_titanic_data

from sklearn.impute import SimpleImputer
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OrdinalEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline

def load_preprocessed_data(mode):
    titanic = load_titanic_data(mode)
    if mode == 'train':
        shuffled_titanic = titanic.sample(frac=1)
        shuffled_titanic = shuffled_titanic.drop('PassengerId', axis=1)
        # print(shuffled_titanic.head())
        # print(shuffled_titanic.head())
        x_data, t_data = shuffled_titanic.drop('Survived', axis=1), \
                        shuffled_titanic['Survived'].copy()

        x_data_num = x_data.drop(['Name', 'Sex', 'Ticket', 'Cabin', 'Embarked'], axis=1)
        num_attribs = list(x_data_num)
        cat_attribs = ['Sex']
        # print('attribs:', num_attribs + cat_attribs)

        num_pipeline = Pipeline([
            ('imputer', SimpleImputer(strategy='median')),
            ('std_scaler', StandardScaler()),
        ])

        full_pipeline = ColumnTransformer([
            ('num', num_pipeline, num_attribs),
            ('cat', OrdinalEncoder(), cat_attribs),
        ])

        x_data_prepared = full_pipeline.fit_transform(x_data)
        t_data = np.array(t_data, dtype=np.uint8)

        return x_data_prepared, t_data
    elif mode == 'test':
        titanic_num = titanic.drop(['PassengerId', 'Name', 'Sex', 'Ticket', 'Cabin', 'Embarked'], axis=1)
        num_attribs = list(titanic_num)
        cat_attribs = ['Sex']
        # print(titanic_num.head())

        num_pipeline = Pipeline([
            ('imputer', SimpleImputer(strategy='median')),
            ('std_scaler', StandardScaler()),
        ])

        full_pipeline = ColumnTransformer([
            ('num', num_pipeline, num_attribs),
            ('cat', OrdinalEncoder(), cat_attribs),
        ])

        titanic_prepared = full_pipeline.fit_transform(titanic)

        return titanic_prepared

# if __name__ == '__main__':
#     xdata, tdata = load_preprocessed_data('train')
#     print(xdata.shape, tdata.shape)
#     data = load_preprocessed_data('test')
#     print(data.shape)


from basicdb import *
from operator import itemgetter

def state_of(zipcode):
    load_db('geo.json')
    return select(where(db_from('geozipcodes'), 'Zip Code', zipcode), 'State')[0]

def northernmost(state):
    load_db('geo.json')
    return select(orderby(where(db_from('geozipcodes'), 'State', state), 'Latitude'), 'Zip Code')[-1]

def zipcodes_in_city(city_name):
    load_db('geo.json')
    return count(where(db_from('geozipcodes'), 'City', city_name))

def states_by_size():
    load_db('geo.json')
    state_list = distinct(select(db_from('geozipcodes'), 'State'))

    counts = []
    for state in state_list:
        # This next line should not be necessary
        # 'State' is in the list of states due to a bug in basicdb.py
        # '' is used as the state name for zipcodes not in a state
        if state != 'State' and state != '':
            counts.append({'state': state,
                           'count': count(where(db_from('geozipcodes'), 'State', state))})

    output_list = []
    for c in sorted(counts, key=itemgetter('count')):
        output_list.append(c['state'])
    return output_list


import random
import sys

def GetIntersection( A, B ):
    newSet = []

    for a in A:
        for b in B:
            if ( a == b ):
                newSet.append( a )

    newSet.sort()

    return newSet

def GetUnion( A, B ):
    newSet = []

    for a in A:
        if a not in newSet:
            newSet.append( a )

    for b in B:
        if b not in newSet:
            newSet.append( b )

    newSet.sort()

    return newSet

def GetDifference( A, B ):
    newSet = []

    for a in A:
        if a not in B:
            newSet.append( a )

    newSet.sort()

    return newSet

def OutputSet( fileHandler, key, values ):
    # So I can use it in LaTeX

    adjustKey = key.replace( "n", "\cap" ).replace( "u", "\cup" )

    fileHandler.write( "$" + adjustKey + " = \{ " )

    items = 0
    for v in values:
        if items != 0:
            fileHandler.write( ", " )

        fileHandler.write( str( v ) )
        items = items + 1

    fileHandler.write( " \}$" )
    fileHandler.write( "\n" )

fileHandler = open( "Sets.txt", "w" )

sets = {}

universe = range( 1, 11 )

letter = "A"

# Generate sets
for i in range( 1, 5 ):

    nS = random.randint( 2, 5 )
    thisSet = []

    for e in range( nS ):
        element = random.randint( 1, 10 )
        if element not in thisSet:
            thisSet.append( element )

    thisSet.sort()
    sets[ letter ] = thisSet
    letter = chr( ord( letter ) + 1 )

# Show sets
print( "U = ", universe )
OutputSet( fileHandler, "U", universe )

for key, value in sets.iteritems():
    print( key + " = ", value )
    OutputSet( fileHandler, key, value )

# Do operations
operations = {}

print( "\n\n BASIC OPERATIONS \n" )
fileHandler.write( "\n\n ------------------------------------------------------------------" )
fileHandler.write( "\n BASIC OPERATIONS \n" )

for k1, s1 in sets.iteritems():
    for k2, s2 in sets.iteritems():
        if k1 == k2:
            continue

        operationSet = {}

        operationSet[ k1 + " n " + k2 ] = GetIntersection( sets[ k1 ], sets[ k2 ] )
        operationSet[ k1 + " u " + k2 ] = GetUnion( sets[ k1 ], sets[ k2 ] )
        operationSet[ k1 + " - " + k2 ] = GetDifference( sets[ k1 ], sets[ k2 ] )
        operationSet[ k2 + " - " + k1 ] = GetDifference( sets[ k2 ], sets[ k1 ] )
        operationSet[ k1 + "'" ] = GetDifference( universe, sets[ k1 ] )
        operationSet[ k2 + "'" ] = GetDifference( universe, sets[ k2 ] )

        for k, o in operationSet.iteritems():
            sys.stdout.write( k + "\t" )
            sys.stdout.write( str( o ) + "\n" )

            OutputSet( fileHandler, k, o )

            operations[ k ] = o

        print( "\n" )
        fileHandler.write( "\n" )

print( "\n\n COMPOUND OPERATIONS \n" )

fileHandler.write( "\n\n ------------------------------------------------------------------" )
fileHandler.write( "\n COMPOUND OPERATIONS \n" )

for k1, s1 in operations.iteritems():
    for k2, s2 in operations.iteritems():
        if k1 == k2:
            continue

        operationSet = {}

        operationSet[ "(" + k1 + ") n (" + k2 + ")" ] = GetIntersection( s1, s2 )
        operationSet[ "(" + k1 + ") u (" + k2 + ")" ] = GetUnion( s1, s2 )
        operationSet[ "(" + k1 + ") - (" + k2 + ")" ] = GetDifference( s1, s2 )
        operationSet[ "(" + k2 + ") - (" + k1 + ")" ] = GetDifference( s2, s1 )
        operationSet[ "(" + k1 + ")'" ] = GetDifference( universe, s1 )
        operationSet[ "(" + k2 + ")'" ] = GetDifference( universe, s2 )

        for k, o in operationSet.iteritems():
            #sys.stdout.write( k + "\t" )
            #sys.stdout.write( str( o ) + "\n" )

            OutputSet( fileHandler, k, o )

        #print( "\n" )
        fileHandler.write( "\n" )


import os

import helpers.config_helper as config_helper
from scheduler.run_scheduler import Scheduler

def main():
    file_name = 'optimal_execution_of_portfolio_transactions'
    meta_data_file_path = os.path.join(config_helper.get_generic_dir(), 'scheduler', 'configs',
                                       file_name + '.yml')
    output_path = os.path.join(config_helper.get_output_dir(), file_name)
    example_scheduler = Scheduler(meta_data_file_path=meta_data_file_path,
                                  plot_component_list=True,
                                  plot_tree=True,
                                  run_components=True,
                                  async_flag=False)

if __name__ == '__main__':
    main()


from random import randrange

from binary_search import binary_search_ceil

def LIS(a):
    """Determines a longest increasing subsequence of the array a"""
    n = len(a)
    curr_subseq = [a[0]]
    indices = [0]
    parent = [None] * n
    for i in range(1, n):
        if a[i] > curr_subseq[-1]:
            parent[i] = indices[-1]
            curr_subseq.append(a[i])
            indices.append(i)
        else:
            j = binary_search_ceil(a[i], curr_subseq)
            curr_subseq[j] = a[i]
            indices[j] = i
            if j >= 1:
                parent[i] = indices[j - 1]
    max_len = len(curr_subseq)
    max_subseq = [None] * max_len
    i = indices[-1]
    for j in range(max_len - 1, -1, -1):
        max_subseq[j] = a[i]
        i = parent[i]
    return max_len, max_subseq

if __name__ == '__main__':
    a = [10, 22, 9, 33, 21, 50, 41, 60, 80]
    print(LIS(a))


import xml.etree.ElementTree as ET
import re

xml = ET.parse("covid_cases_xml.xml")
root = xml.getroot()

ns = re.match('{.*}', root.tag).group(0)
records = root.find("{}record".format(ns))
daterep = records.find("{}dateRep".format(ns))
countter = records.find("{}countriesAndTerritories".format(ns))
numbercases = records.find("{}cases".format(ns))
deaths = records.find("{}deaths".format(ns))

print("The date recorded contains: {}".format(daterep.text))
print("The countries and territories contains: {}".format(countter.text))
print("The number of cases are: {}".format(numbercases.text))
print("The number of deaths are: {}".format(deaths.text))


# -*- coding: utf-8 -*-

from flask import Flask, render_template
from catchutil.DB import *
from flask import request

app = Flask(__name__, static_folder='static', static_url_path='')

# @app.route('/')
def index():
    return render_template('index.html')

@app.route("/rss")
def rss():
    rss_list = TRssDao.listTRss(True)
    rss_false_list = TRssDao.listTRss(False)
    return render_template("rss.html", rss_list=rss_list, rss_false_list=rss_false_list)

@app.route("/add", methods=['GET', 'POST'])
def add_rss():
    status = {}
    if request.method == 'POST':
        t_rss = TRss(name=request.form["name"], url=request.form["url"])
        if request.form["enabled"]:
            t_rss.enabled = False
        else:
            t_rss.enabled = True
        TRssDao.saveTRss(t_rss)
        status["s"] = "Success"
    return render_template("add_rss.html", status=status)

@app.route("/news")
def read_news():
    rss_items = TRssItemDao.queryTRssItem(**request.args)
    return render_template("news.html", items=rss_items)

@app.route("/items")
def read_items():
    rss_items = TRssItemDao.queryTRssItemByRssId(**request.args)
    return render_template("items.html", items=rss_items, rss_id=request.args["rssId"])

if __name__ == '__main__':
    app.run("0.0.0.0", 8080, True)


from flask import Flask, jsonify, request
app = Flask(__name__)
information = {
    "Amount": 1000,
    "rate": 5,
    "time": 2
    }
@app.route('/whatisthis')
def find():
    return "this is for calculate simple interest by using flask..."
@app.route('/hi')
def hello():
    queryparam = request.args.get('queryparam')
    if queryparam == "yes":
        return jsonify({ "Output":(information['Amount'] * information['rate'] * information['time'])/100 })
    else:
        return "sorry"
app.run(port = 5000)


# import necessary libraries
from Machine_Learning.model import run_model, ModelInput, ModelOutput, ModelException
from flask import (
    Flask,
    render_template,
    jsonify,
    request,
    redirect)

app = Flask(__name__)

# create route that renders index.html template

@app.route("/")
def home():
    return render_template("index.html")

@app.route("/visualizations.html")
def viz():
    return render_template("visualizations.html")

@app.route("/results.html")
def results():
    return render_template("results.html")

@app.route("/send", methods=["GET", "POST"])
def send():
    if request.method == "POST":
        data = ModelInput(
            budget=request.form["budget"],
            actor=request.form["actor"],
            director=request.form["director"],
            language=request.form["lenguage"],
            production=request.form["production"],
            writer=request.form["writer"],
            runtime=request.form["runtime"],
            country=request.form["country"],
            rated=request.form["rating"],
            genre=request.form["genre"]
        )

        try:
            result: ModelOutput = run_model(data)
        except ModelException:
            print('Bad input')
        else:
            print('Is profitable?')
            print(result.profitable)
            data = {'result': result.profitable}

        return render_template("results.html", data=data['result'])

if __name__ == "__main__":
    app.run(debug=True)


from Matrix import *
def numeral(n):
    return chr(n + 65)

# 0-25 to A-Z
# 0=A
# 26=AA
# 675=YZ limit check how to manage left zeros

def int_to_letter(number, base=26):
    # if is 0 convertion is direct
    # not support for negative numbers
    if (number == 0): return numeral(0)
    elif(number<0): return '--'

    mods = []
    while number > 0:
        mods.append(number % base)
        number //= base
    if (len(mods) == 1): mods[0] = numeral(mods[0])

    else:
        for i in list(range(len(mods))):
            if(i==0):
                mods[i] = numeral(mods[i])
            else:
                mods[i] = numeral(mods[i] - 1)
            #manage left zeros

    mods.reverse()
    return ''.join(mods)

def colNameToInt( name):  # name is the letter part of the cell name
    # invert oder of letters (lowest count first)
    name = name[::-1]
    result = 0
    # loop over all letters
    for i in range(len(name)):
        # convert the letter to its value, make 1 the first value
        num = (ord(name[i]) - 65) + 1
        # add the number multiplied by its weight
        result += num * 26 ** i
    return result - 1

def rangeToListConverter(implicit_range):  # nameRange: 'A5:D11'

    implicit_range = implicit_range.split(':')

    first_col = colNameToInt(getLetters(implicit_range[0]))
    second_col = colNameToInt(getLetters(implicit_range[1]))
    start_column = min(first_col, second_col)
    end_column = max(first_col, second_col)
    first_row = int(getNumbers(implicit_range[0]))
    second_row = int(getNumbers(implicit_range[1]))
    start_row = min(first_row, second_row)
    end_row = max(first_row, second_row)

    explicit_range = []
    for i, colnumber in enumerate(range(start_column, end_column + 1)):
        localLetters = colNumberToName(colnumber + 1)  # i+1: actual to nominal correction
        for j, rownumber in enumerate(range(start_row, end_row + 1)):
            explicit_range.append(localLetters + str(rownumber))

    return explicit_range

def getLetters(cell):
    s = list(cell)
    letters = [x for x in s if (not x.isdigit())]
    result = ''.join(letters)
    return result

def getNumbers(cell):
    s = list(cell)
    numbers = [x for x in s if (x.isdigit())]
    result = ''.join(numbers)
    return result

#1=A, Z=26, AA=27 etc
def colNumberToName(number):
    if number<=0:
        print('Column number cannot be non positive')
        return

    # Base Conversion
    name_list = []

    divisible = number
    while divisible>0:
        remain = divisible%26
        if remain !=0:
            name_list.insert(0, str(remain))
            divisible = divisible // 26
        else:
            name_list.insert(0, str(26))
            divisible = divisible // 26-1

    letters=''
    for x in name_list:
        x = int(x)
        letters += chr(65+(x-1))

    return letters

def cell_to_coords(cell):
    row = int(getNumbers(cell)) - 1
    col = colNameToInt(getLetters(cell))
    return (row,col)


class Canvas:
    def __init__(self, output_file):
        self.width = None
        self.height = None
        self.statement = []
        self.exist = False
        self.output = output_file

    @staticmethod
    def right_order(a, b):
        if a > b:
            t = a
            a = b
            b = t
        return a, b

    def _check_border(self, x1, y1):
        if x1 <= 0 or x1 > self.width or y1 <= 0 or y1 > self.height:
            raise Exception("Point out of canvas")

    def _make_canvas(self, width, height):  # create canvas
        if width > 0 and height > 0:
            self.width = width
            self.height = height
            top_border = '-' * (self.width + 2) + '\n'
            canvas_line = '|' + ' ' * self.width + '|\n'
            blank = top_border + canvas_line * self.height + top_border
            for row in blank.splitlines():
                self.statement.append(list(row + '\n'))
            self.exist = True
        else:
            raise Exception("ValueError: width and height must be > 0")

    def do_command(self, command):  # do command (command is array of commands)
        if command[0] == 'C':  # command 'C' create canvas
            if len(command) == 3:  # 3 parameters if it's creation of canvas: 'C' 'width' 'height'
                try:
                    w = int(command[1])
                    h = int(command[2])
                except ValueError:
                    print("ValueError: Width and height must be integers")
                    raise
                self._make_canvas(w, h)
            else:
                raise Exception("Wrong number of arguments for Canvas")
        if self.exist:  # command 'C' should be first in list of commands, if doesn't - throw Exception
            if command[0] == 'L':  # command 'L' create line
                if len(command) == 5:  # 3 parameters if it's creation of canvas: 'C' 'width' 'height'
                    try:
                        x1 = int(command[1])
                        y1 = int(command[2])
                        x2 = int(command[3])
                        y2 = int(command[4])
                    except ValueError:
                        print("ValueError: x1,y1,x2,y2 must be integers")
                        raise
                    self._check_border(x1, y1)
                    self._check_border(x2, y2)
                    if x1 == x2 or y1 == y2:
                        self._make_line(x1, y1, x2, y2)
                    else:
                        raise Exception("Line must be vertical or horizontal")
                else:
                    raise Exception("Wrong number of arguments for Line")

            if command[0] == 'R':  # command 'R' create rect
                if len(command) == 5:  # 3 parameters if it's creation of canvas: 'C' 'width' 'height'
                    try:
                        x1 = int(command[1])
                        y1 = int(command[2])
                        x2 = int(command[3])
                        y2 = int(command[4])
                    except ValueError:
                        print("ValueError: x1,y1,x2,y2 must be integers")
                        raise
                    self._check_border(x1, y1)
                    self._check_border(x2, y2)
                    self._make_rect(x1, y1, x2, y2)
                else:
                    raise Exception("Wrong number of arguments for Rectangle")

            if command[0] == 'B':  # command 'B' fill by colour
                if len(command) == 4:  # 3 parameters if it's creation of canvas: 'C' 'width' 'height'
                    try:
                        x1 = int(command[1])
                        y1 = int(command[2])
                    except ValueError:
                        print("ValueError: x1,y1 must be integers")
                        raise
                    color = command[3]
                    if len(color) != 1:
                        raise Exception("Color must be 1-character symbol")
                    self._check_border(x1, y1)
                    self.__paint_neighbors(x1, y1, color, old_color=self.statement[y1][x1])
                else:
                    raise Exception("Wrong number of arguments for Bucket Fill")

            if command[0] == 'P':  # command 'P' create point
                if len(command) == 3:
                    try:
                        x1 = int(command[1])
                        y1 = int(command[2])
                    except ValueError:
                        print("ValueError: x1 and y1 must be integers")
                        raise
                    self._check_border(x1, y1)
                    self._make_point(x1, y1)
                else:
                    raise Exception("Wrong number of arguments for Point")
        else:
            raise Exception("Canvas doesn't exist")

    def print_statement(self, output_file):
        with open(output_file, 'w') as output:  # write canvas
            for line in self.statement:
                output.writelines(line)

    def _make_line(self, x1, y1, x2, y2, symbol='x'):  # only vertical and horizontal lines
        if x1 == x2 and y1 == y2:
            self._make_point(x1, y1)
        elif x1 == x2:
            y1, y2 = self.right_order(y1, y2)
            for y in range(y1, y2 + 1):
                self.statement[y][x1] = symbol
        elif y1 == y2:
            x1, x2 = self.right_order(x1, x2)
            for x in range(x1, x2+1):
                self.statement[y1][x] = symbol

    def _make_point(self, x1, y1, symbol='x'):  # point
        self.statement[y1][x1] = symbol

    def _make_rect(self, x1, y1, x2, y2, symbol='x'):
        self._make_line(x1, y1, x2, y1, symbol=symbol)
        self._make_line(x1, y2, x2, y2, symbol=symbol)
        self._make_line(x1, y1, x1, y2, symbol=symbol)
        self._make_line(x2, y1, x2, y2, symbol=symbol)

    def __paint_neighbors(self, x, y, new_color, old_color):
        if (0 < x <= self.width) and (0 < y <= self.height) and self.statement[y][x] == old_color and self.statement[y][x] != new_color:
            self.statement[y][x] = new_color
            for row in range(y-1, y+2):
                for col in range(x-1, x+2):
                    if not(row == y and col == x):
                        self.__paint_neighbors(col, row, new_color, old_color)

def paint(input_file, output_file):
    with open(input_file, 'r') as f:
        source = f.read()
        commands = source.splitlines()  # read all commands in array one line by one
    canvas = Canvas(output_file)  # create empty object of Canvas
    with open(output_file, 'w') as output:  # write canvas
        for com in commands:
            com = com.split()  # split command in parameters
            canvas.do_command(com)
            for line in canvas.statement:
                output.writelines(line)
    output = open(output_file, 'r').read()
    return source, output

if __name__ == '__main__':
    assert paint('tests/input/input1.txt', 'tests/output/output1.txt') == (open('tests/input/input1.txt', 'r').read(), open('tests/true_output/true_output1.txt', 'r').read())


from torch.utils.data import Dataset, DataLoader
import torch.nn as nn
from torchvision import transforms
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import pims
import pathlib
import torch.optim as optim
from torch.autograd import Variable
import torch
torch.cuda.empty_cache()
import skimage as skm
import glob
import sys

#classes/functions

class DefectDataset(Dataset):
    """custom dataset for star locator"""

    def __init__(self, label_dir, root_dir):
        """
        Args:
            csvfile (pathlib): folder containing labels
            root_dir (pathlib): root directory with all the images
        """
        self.root_dir = root_dir
        self.label_dir = label_dir
        self.labels = self.process_labels()
        self.frames = pims.ImageSequence(str(root_dir / '*.tiff'))
        self.shape = self.frames[1].shape
        ar = np.array(self.frames).reshape(-1,4)
        self.meand = ar.mean(axis=0)
        self.stdd = ar.std(axis=0)
        ar = None
        self.n = self.shape[0]
        #self.to_tensor = transforms.Compose([transforms.ToTensor(), transforms.Normalize(tuple(self.meand[0:3]),tuple(self.stdd[0:3]))])
        self.to_tensor = transforms.ToTensor()

    def process_labels(self):
        '''
        we need a function that will read in all the annotations and create a master dataframe [img_name][total_defect]
        '''
        files = glob.glob(str(self.label_dir / '*.dat'))
        df = pd.DataFrame([[f,np.count_nonzero(np.genfromtxt(f))] for f in files], columns = ['names', 'numbers'])
        df['frame'] = df['names'].str.extract('label-t_(\d*)\.dat').astype('int')
        df = df.sort_values('frame').reset_index(drop='True')
        #df.set_index('frame', inplace=True)
        return(df)

    def __len__(self):
        return len(self.frames)

    def __getitem__(self, idx):
        if torch.is_tensor(idx):
            idx.tolist()

        sample = [self.to_tensor(self.frames[idx][:,:,0:3]/255).float(), torch.tensor(self.labels.iloc[idx]['numbers']).float()/256**2]
        return sample

class Net(nn.Module):
    def __init__(self):
        super().__init__()
        cLayers1 = 16
        cLayers2 = 8

        self.conv1 = nn.Conv2d(3, 32, kernel_size = 3, padding = 1)
        self.conv2 = nn.Conv2d(32, 64, kernel_size = 3, padding = 1)
        self.conv3 = nn.Conv2d(64, 1, kernel_size = 3, padding = 1)
        self.fc1 = nn.Linear(1 * 4*4, 1)

        self.fc3 = nn.Linear(1, 1, bias = True)

        self.pool = nn.MaxPool2d(4)
        self.act1 = nn.LeakyReLU()
        self.act1 = nn.ReLU()
        self.prob = nn.Sigmoid()
    def forward(self, x):
        out = self.pool( self.act1( self.conv1(x)))
        out = self.pool(self.act1(self.conv2(out)))
        out = self.pool(self.act1(self.conv3(out)))
        out = out.view(-1, 1*4*4)
        out = self.act1(self.fc1(out))
        out = self.act1(self.fc3(out))

        return out
        #the output of this will be a 125*125 matrix.

#first, read in data into dataloader, and split into test and validation set
data_folder = pathlib.Path('.') /'..' / 'data' /'img'
csv_name = pathlib.Path('.')/'..'/'data' / 'labels'

print(data_folder)
defectD = DefectDataset(csv_name, data_folder)
train_size = int(0.8*len(defectD))
test_size = len(defectD) - train_size

import datetime  # <1>
device = (torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu'))
print(f"Training on device {device}.")

def loss_funct(output, target):
    #loss = torch.sum((torch.mm((torch.exp( (output-target)**2)-1).t(),target))[:,0])
    #rel_error = torch.sum((output.T[0]-target)**2/(target+1)**2)
    error = torch.mean((output.sum(dim=1)-target)**2)
    return error #+.5*rel_error

def training_loop(n_epochs, optimizer, model, loss_fn, train_loader):
    for epoch in range(1, n_epochs + 1):  # <2>
        loss_train = 0.0
        for imgs, labels in train_loader:  # <3>
            imgs = imgs.to(device=device)
            labels = labels.to(device=device)
            outputs = model(imgs)  # <4>

            loss = loss_fn(outputs, labels)  # <5>
            optimizer.zero_grad()  # <6>

            loss.backward()  # <7>

            optimizer.step()  # <8>

            loss_train += loss.item()  # <9>

        if epoch == 1 or epoch % 10 == 0:
            print('{} Epoch {}, Training loss {}, std{} , teststd {} '.format(
                datetime.datetime.now(), epoch, loss_train/len(train_loader), outputs.std(), labels.float().std()))

def validate(model, train_loader, val_loader):
    for name, loader in [('train', train_loader), ('val', val_loader)]:
        correct = 0
        total = 0

        with torch.no_grad():
            for imgs, labels in loader:
                imgs = imgs.to(device)
                labels = labels.to(device)
                outputs = model(imgs)
                total += labels.shape[0]
                cor = ((outputs - labels)*256**2).int()
                correct += int((cor == 0).sum())
        print("Accuracy {}: {:.2f}".format(name , correct / total))

def val2(model, train_loader, val_loader):
    for name, loader in [('train', train_loader), ('val', val_loader)]:
        ave_dist= 0
        total = 0
        right_counts = 0

        with torch.no_grad():
            for imgs, labels in loader:
                imgs = imgs.to(device)
                labels = labels.to(device)
                outputs = model(imgs)
                total += labels.shape[0]
                num= outputs.argmax(dim=1)
                cor =  (num-labels)
                right_counts += int((cor == 0).sum())
                ave_dist = torch.sqrt( ((num-labels)**2).float().sum())
        print("Right percent {}, dist to truth {}".format(right_counts/total , ave_dist))

train_dataset, test_dataset = torch.utils.data.random_split(defectD, [train_size, test_size])
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size =32 , shuffle = True)
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size =32, shuffle = False)
model = Net().to(device = device)
lr = 1e-4
optimizer = optim.SGD(model.parameters(), lr=lr, weight_decay = .0005, momentum = .9)
loss_fn = loss_funct
loss_fn = nn.CrossEntropyLoss()
loss_fn = nn.MSELoss()
weight_file = pathlib.Path('./m1.pt')
if weight_file.is_file():
    loaded_model = Net().to(device)
    loaded_model.load_state_dict(torch.load(str(weight_file), map_location=torch.device(device)))
    val2(loaded_model, train_loader, test_loader)
else:
    run_more = lr
    while(run_more != 'n'):
        try:
            optimizer = optim.SGD(model.parameters(), lr=float(lr))
            training_loop(  # <5>
                n_epochs = 1000,
                optimizer = optimizer,
                model = model,
                loss_fn = loss_fn,
                train_loader = train_loader,
            )
        except KeyboardInterrupt:
            run_more = 'n'

    weight_file = pathlib.Path('./m1.pt')
    torch.save(model.state_dict(), str(weight_file))

    val2(model, train_loader, test_loader)

def plt_frame(frame):
   f=test_dataset.__getitem__(frame)
   fig,ax = plt.subplots()
   ax.imshow(f[0].permute(1,2,0))
   outputs = model(f[0].unsqueeze(0).to(device))
   prob = torch.exp(outputs)/torch.exp(outputs).sum()
   thresh = .0015
   pred = torch.where(prob > thresh)
   ax.scatter(20,20, color = 'blue', s = 10)
   for i in pred:
       x = (i//31)*8
       y = (i % 31)*8
       print(x,y)
       ax.scatter(x.cpu(),y.cpu(),color = 'red', s = 10)

#plt_frame(10)
plt.show()


# ===================================================================== #
#                             APP : views                               #
# ===================================================================== #

from django.shortcuts   import render, redirect, get_object_or_404, get_list_or_404

from ..models           import Article, Commentaire
from ..forms            import CommentaireForm

# ===================================================================== #
# VIEW: article                                                         #
# ===================================================================== #

def article_home(request):
    return redirect(to=article_list, permanent=true)

def article_list(request):
    """retrouve les 5 derniers articles les plus récents"""
    #list_articles = Article.objects.filter(is_visible=True).order_by(('-date',))[:5]
    # last_five_articles = get_list_or_404(Article, is_visible=True)[:5]
    list_articles = get_list_or_404(Article.objects.order_by('-parution'), is_visible=True)[:5]
    return render(request,
                  template_name='blog/article/list.html',
                  context={'articles': list_articles})

def article_show(request, id):
    article = get_object_or_404(Article, id = id)

    # handle commentaire post
    commentaire_form = CommentaireForm(request.POST or None)
    commentaire_saved = False
    if commentaire_form.is_valid():
        commentaire = commentaire_form.save(commit=False)
        commentaire.article = article
        commentaire.save()
        commentaire_form = CommentaireForm()
        commentaire_saved = True

    # handle commentaire list

    commentaires = Commentaire.objects.filter(article=article)

    return render(request,
                  template_name='blog/article/show.html',
                  context=locals())


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
sw = pd.read_csv("StarWars.csv", encoding="ISO-8859-1")

#print(sw.head())
########################################################################################################################
#turn certain columns into booleans 1)Have you seen any of the 6 films in the Star Wars franchise?
#2)Do you consider yourself to be a fan of the Star Wars film franchise?
yes_no = {
    "Yes": True,
    "No": False
}

sw['Have you seen any of the 6 films in the Star Wars' \
   ' franchise?'] = sw['Have you seen any of the 6 films in the Star Wars franchise?'].map(yes_no)
sw['Do you consider yourself to be a fan of the Star' \
   ' Wars film franchise?'] = sw['Do you consider yourself to be a fan of the Star Wars film franchise?'].map(yes_no)

########################################################################################################################
# turn which mvie have you seen column into booleans

map_titles = {
    'Star Wars: Episode I  The Phantom Menace':True,
'Star Wars: Episode II  Attack of the Clones':True,
'Star Wars: Episode III  Revenge of the Sith':True,
'Star Wars: Episode IV  A New Hope' : True,
'Star Wars: Episode V The Empire Strikes Back' : True,
'Star Wars: Episode VI Return of the Jedi' : True,
    np.nan: False
}

for col in sw.columns[3:9]:
    sw[col] = sw[col].map(map_titles)

sw = sw.rename(columns={
    "Which of the following Star Wars films have you seen? Please select all that apply.": "seen_episode1",
    'Unnamed: 4': 'seen_episode2',
    'Unnamed: 5': 'seen_episode3',
    'Unnamed: 6': 'seen_episode4',
    'Unnamed: 7': 'seen_episode5',
    'Unnamed: 8': 'seen_episode6',
})
#print(sw.head())
########################################################################################################################
# change the names of the columns related to rank and then turn the ranks to floats. astype(float) did not work so I needed
#to use pandas apply
map_rank ={
    'Please rank the Star Wars films in order of preference with 1 being your favorite film in the franchise and 6 being your least favorite film.':'ranking_episode1',
    'Unnamed: 10': 'ranking_episode2',
'Unnamed: 11': 'ranking_episode3',
'Unnamed: 12': 'ranking_episode4',
'Unnamed: 13': 'ranking_episode5',
'Unnamed: 14': 'ranking_episode6',

}
ranking = ['ranking_episode1','ranking_episode2','ranking_episode3','ranking_episode4','ranking_episode5','ranking_episode6']

sw = sw.rename(columns=map_rank)
sw[sw.columns[9:15]] = (sw[sw.columns[9:15]].apply(pd.to_numeric, args=('coerce',)))
#print(sw.head())
means = sw[sw.columns[9:15]].apply(np.mean)
print(means)
########################################################################################################################
#graph out
plt.bar(range(6), means)
plt.show()
#general concensus the that episode 5 is the best
########################################################################################################################
#along gender lines
m = sw[sw['Gender']=='Male']
w = sw[sw['Gender']=='Female']
means_w = w[w.columns[9:15]].apply(np.mean)
means_m = m[m.columns[9:15]].apply(np.mean)

plt.bar(range(6),means_m, color='r',label='Male Ranking')
plt.bar(range(6), means_w,color='g', label='Female Ranking')
plt.legend()
plt.show()


import pandas as pd
import numpy as np
df = pd.read_csv('lab1.csv')
columns = list(df)

print('Критерій Вальда:')
vald = []
for i in columns:
    vald.append(df[i].min())
print(vald)
vald = np.array(vald)
print('Найкращим є: ',np.argmax(vald) + 1)

print('Максимальний:')
max_ = []
for i in columns:
    max_.append(df[i].max())
print(max_)
max_ = np.array(max_)
print('Найкращим є: ',np.argmax(max_) + 1)

print('Гурвіца:')
a = 0.5
gur = {}
for i in columns:
    gur[i] = max(df[i])*0.5+min(df[i])*(1-a)
gur = list(gur.values())
print(gur)
gur = np.array(gur)
print('Найкращим є: ',np.argmax(gur) + 1)

print('Лапласса:')
lap = {}
for i in columns:
    for j in range(len(df[i])):
        if i not in lap:
            lap[i] = 0
        lap[i] += df[i][j] / len(columns)
lap = list(lap.values())
print(lap)
lap = np.array(lap)
print('Найкращим є: ',np.argmax(lap) + 1)

print('Лапласса при p1=0.5, p2=0.35, p3=0.15:')
lap2 = {}
p = [0.5,0.35,0.15]
for i in columns:
    for j in range(len(df[i])):
        if i not in lap2:
            lap2[i] = 0
        lap2[i] += df[i][j] * p[j]
lap2 = list(lap2.values())
print(lap2)
lap2 = np.array(lap2)
print('Найкращим є: ',np.argmax(lap2) + 1)
print()

results = pd.DataFrame(list(zip(vald, max_, gur, lap, lap2)),
                       columns =['Вальда', 'Максимінний','Гурвіца','Лапласа','Лапласа 2'], index = columns)
print(results)

лаб1/лаб1.py


"""network4.py
~~~~~~~~~~~~~~

An improved version of network.py, implementing the stochastic
gradient descent learning algorithm for a feedforward neural network.
Improvements include the addition of the cross-entropy cost function,
Dropout-regularization, and better initialization of network weights.  Note
that I have focused on making the code simple, easily readable, and
easily modifiable.  It is not optimized, and omits many desirable
features.

"""

#### Libraries
# Standard library
import json
import random
import sys

# Third-party libraries
import numpy as np

#### Main Network class
class Network(object):

    def __init__(self, sizes):
        """The list ``sizes`` contains the number of neurons in the respective
        layers of the network.  For example, if the list was [2, 3, 1]
        then it would be a three-layer network, with the first layer
        containing 2 neurons, the second layer 3 neurons, and the
        third layer 1 neuron.  The biases and weights for the network
        are initialized randomly, using
        ``self.default_weight_initializer`` (see docstring for that
        method).

        """
        self.num_layers = len(sizes)
        self.sizes = sizes

    def weights_biases_initializer(self):
        self.weights = [np.random.randn(y,x) for x, y in zip(self.sizes[:-1], self.sizes[1:])]
        self.biases = [np.random.randn(y,1) for y in self.sizes[1:]]

    def dropout_mask(self, prob=0.5):
        """Using binomial distribution to generate a thinner network,
        which is inherit the weights and bias from the mother full
        network. The "prob" is the probability of the binomial
        distribution.

        """
        thin_sizes = [np.ones(self.sizes[0])]
        # To record the selected neurons.
        thin_sizes_num = [self.sizes[0]]
        # To record the numbers of the selected neurons.

        for k in self.sizes[1:-1]:
            binom_dis = np.random.binomial(n=1, p=prob, size=k)
            print(binom_dis)
            thin_sizes_num.append(np.sum(binom_dis))
            thin_sizes.append(binom_dis)
        thin_sizes.append(np.ones(self.sizes[-1]))
        thin_sizes_num.append(self.sizes[-1])

        self.weights_mask = [np.outer(y,x)==1 for x, y in zip(thin_sizes[:-1], thin_sizes[1:])]
        self.biases_mask = [thin_sizes[i]==1 for i in range(1,self.num_layers)]

        return thin_sizes_num

    def thin_weights_biases(self):
        """
        Generate the thin weights and biases using dropout method
        """
        thin_sizes_num = self.dropout_mask(prob=0.5)
        thin_weights = [self.weights[i][self.weights_mask[i]].reshape(thin_sizes_num[i+1],\
                        thin_sizes_num[i]) for i in range(self.num_layers-1)]
        thin_biases = [self.biases[i][self.biases_mask[i]] for i in range(self.num_layers-1)]

        return thin_weights, thin_biases

if  __name__ == '__main__':
    sizes = [2,4,5,2]
    net = Network(sizes)
    net.weights_biases_initializer()
    thin_weights, thin_biases = net.thin_weights_biases()
    print(net.weights)
    print("\n")
    print(thin_weights)
    print("\n")
    print(thin_biases)
    print("\n")


from argparse import ArgumentParser
from args import LightningBaseArgParser
from pytorch_lightning import Trainer, seed_everything
from pytorch_lightning import loggers as pl_loggers
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint, LearningRateLogger

from lightning.example_lightning import ExampleModel

seed_everything(6)

def main(args):
    logger = pl_loggers.WandbLogger(experiment="example", save_dir=None)
    early_stop = EarlyStopping(
        monitor="val_loss"
    )
    checkpoint_callback = ModelCheckpoint(dirpath="ckpts/", monitor="val_loss")
    model = ExampleModel(args)
    lr_logger = LearningRateLogger()
    trainer = Trainer.from_argparse_args(
        args,
        logger=logger,
        callbacks =  [early_stop, lr_logger],
        checkpoint_callback=checkpoint_callback
    )
    trainer.fit(model)

if __name__ == "__main__":
    parser = LightningBaseArgParser().get_parser()
    parser = ExampleModel.add_model_specific_args(parser)
    parser = Trainer.add_argparse_args(parser)
    args = parser.parse_args()
    main(args)


from typing import List

from drf_yasg import openapi

from winter.core import ComponentMethodArgument
from winter.web.routing import Route
from .route_parameters_inspector import RouteParametersInspector
from .type_inspection import InspectorNotFound
from .type_inspection import TypeInfo
from .type_inspection import inspect_type

class PathParametersInspector(RouteParametersInspector):

    def inspect_parameters(self, route: 'Route') -> List[openapi.Parameter]:
        parameters = []

        for argument in self._path_arguments(route):
            openapi_parameter = self._convert_argument_to_openapi_parameter(argument)
            parameters.append(openapi_parameter)

        return parameters

    def _convert_argument_to_openapi_parameter(self, argument: ComponentMethodArgument) -> openapi.Parameter:
        try:
            type_info = inspect_type(argument.type_)
            description = argument.description
        except InspectorNotFound:
            type_info = TypeInfo(openapi.TYPE_STRING)
            description = 'winter_openapi has failed to inspect the parameter'
        return openapi.Parameter(
            name=argument.name,
            description=description,
            required=True,
            in_=openapi.IN_PATH,
            default=argument.get_default(None),
            **type_info.as_dict(output=False),
        )

    def _path_arguments(self, route: 'Route') -> List[ComponentMethodArgument]:
        path_arguments = []
        for path_variable in route.get_path_variables():
            argument = route.method.get_argument(path_variable)
            if argument is not None:
                path_arguments.append(argument)
        return path_arguments


"""
Script to start server to drive your car.
"""

import model

import keras

# Load a trained keras model and use it in the KerasAngle pilot
# model_file = ''

model_file = '/home/'
model = keras.models.load_model(model_file)
pilot = model.pilots.KerasAngle(model=model, throttle=20)

# Specify where sessions data should be saved
sh = model.sessions.SessionHandler(sessions_path='~/data/sessions')
session = sh.new()

# Start server
w = model.remotes.RemoteServer(session, pilot, port=8886)
w.start()

# in a browser go to localhost:8887 to drive your car