import pyeapi
import pprint
import os
import sys
import json
import requests
from datetime import datetime
from prettytable import PrettyTable, ALL, FRAME, from_json

pp = pprint.PrettyPrinter()

#double check you have the .eapi.conf in the correct dir

# list of all leaf swiches
leafs = ['leaf1','leaf2','leaf3','leaf4']
# list of all spine switches
spines = ['spine1','spine2']
# list of end host for ping tests
hosts = ['host1','host2']

## Todo , write file to output in local Dir
## Filter the response msg from the pings, only show [-2] second to last index in list
## Use PrettyTable to present info in a nicer format.

def write_output(data, funcname):
    """ Write a file to the present working directory """
    today = datetime.now()
    if not os.path.exists('pyeAPI-output'):
        os.mkdir('pyeAPI-output')
    os.makedirs("pyeAPI-output/"+today.strftime("%d-%m-%Y"), exist_ok=True)
    day = "pyeAPI-output/"+today.strftime("%d-%m-%Y")
    with open(os.path.join(day, (today.strftime('%d-%m-%Y_%H%M%S')+funcname+'_output.json')), 'w') as outfile:
        json.dump(data, outfile)

def append_new_line(file_name, text_to_append):
    """Append given text as a new line at the end of file"""
    tday = datetime.now()
    day = "pyeAPI-output/"+tday.strftime("%d-%m-%Y")
    original = sys.stdout
    # Open the file in append & read mode ('a+')
    with open(os.path.join(day, (tday.strftime('%d-%m-%Y_%H%M')+file_name)), "a+") as file_object:
        # Move read cursor to the start of file.
        file_object.seek(0)
        # If file is not empty then append '\n'
        data = file_object.read(100)
        if len(data) > 0:
            file_object.write("\n")
        # Append text at the end of file
        file_object.write(text_to_append)
    sys.stdout = original

def ping_l2hosts():
    """ Get Devices """
    pt = PrettyTable()
    pt.field_names = ["Role", "Name", "FabricState","Adst"]
    for host in hosts:
        # Connect to the Arista switch using eAPI
        connection = pyeapi.connect_to(host)
        # Send the command to ping both loopbacks for h1
        ping_hx = connection.enable('ping 10.10.10.1')
        # Parse the response for MLAG status
        ping_response = ping_hx[0]['result']['messages']
        # Print the response from switch
        print("\n"+40*'='+' Ping 10.10.10.1 Host1, from '+host+' '+'='*40+"\n")
        pp.pprint(ping_response)
        # Send the command to ping both loopbacks for h2
        ping_hx = connection.enable('ping 10.10.10.2')
        # Parse the response for MLAG status
        ping_response = ping_hx[0]['result']['messages']
        # Print the response from switch
        print("\n"+40*'='+' Ping 10.10.10.2 Host2, from '+host+' '+'='*40+"\n")
        pp.pprint(ping_response)

ping_l2hosts()

def ping_l3hosts():
    """ Get Devices """
    pt = PrettyTable()
    pt.field_names = ["Role", "Name", "FabricState","Adst"]
    for host in hosts:
        # Connect to the Arista switch using eAPI
        connection = pyeapi.connect_to(host)
        # Send the command to ping both loopbacks for h1
        ping_hx = connection.enable('ping 9.9.9.9')
        # Parse the response for MLAG status
        ping_response = ping_hx[0]['result']['messages']
        # Print the response from switch
        print("\n"+40*'='+' Ping 9.9.9.9 Host1, from '+host+' '+'='*40+"\n")
        pp.pprint(ping_response)
        # Send the command to ping both loopbacks for h2
        ping_hx = connection.enable('ping 8.8.8.8')
        # Parse the response for MLAG status
        ping_response = ping_hx[0]['result']['messages']
        # Print the response from switch
        print("\n"+40*'='+' Ping 8.8.8.8 Host2, from '+host+' '+'='*40+"\n")
        pp.pprint(ping_response)

ping_l3hosts()

### Get the All the Fabric EVPN Route-type 2's

def get_rt2():
    """ Get Devices """
    pt = PrettyTable()
    pt.field_names = ["Role", "Name", "FabricState","Adst"]
    for leaf in leafs:
        # Connect to the Arista switch using eAPI
        connection = pyeapi.connect_to(leaf)
        # Send the command to retrieve MLAG status
        response = connection.enable('show bgp evpn route-type mac-ip')
        # Parse the response
        bgp_evpnRt2 = response[0]['result']['evpnRoutes']
        # Print the response from switchs
        print("\n"+40*'='+' GET ALL RouteType-2 '+leaf+' '+'='*40+"\n")
        for key, value in bgp_evpnRt2.items():
            print (key)
        # print(bgp_evpnRt2)
        # print("\n"+80*'='+'='*40+"\n")

get_rt2()

def get_rt5():
    """ Get Devices """
    pt = PrettyTable()
    pt.field_names = ["Role", "Name", "FabricState","Adst"]
    for leaf in leafs:
        # Connect to the Arista switch using eAPI
        connection = pyeapi.connect_to(leaf)
        # Send the command to retrieve MLAG status
        response = connection.enable('show bgp evpn route-type ip-prefix ipv4')
        # Parse the response
        bgp_evpnRt5 = response[0]['result']['evpnRoutes']
        # Print the response from switchs
        print("\n"+40*'='+' GET ALL RouteType-5 '+leaf+' '+'='*40+"\n")
        for key, value in bgp_evpnRt5.items():
            print (key)
        # print(bgp_evpnRt2)
        # print("\n"+80*'='+'='*40+"\n")

get_rt5()


import sys

from corretores.corretor_enem import correcao_enem

def corrigir(df_respostas, df_gabarito, _, tipo_correcao):

    if tipo_correcao == "simufsc":
        print("Correção de simulado ufsc não implementado")
        # dados_alunos, resultado_alunos = correcao_ufsc(
        #     gabarito, dados_alunos, resultado_alunos)
        # return gerar_lista_resultado_alunos_ufsc(resultado_alunos, redacoes)
        sys.exit()

    if tipo_correcao == "simuenem":

        df_resultado = correcao_enem(df_respostas, df_gabarito)
        return df_resultado

    if tipo_correcao == "simulinho":
        print("Correção de simulinho não implementada")
        sys.exit()


import math

"""
Tools:
- math library
- sorted function
- list slicing
- computations
"""

sale_prices = [
  100,
  83,
  220,
  40,
  100,
  400,
  10,
  1,
  3
]

#Jordan
sorted_list = sorted(sale_prices)
num_of_sales = len(sorted_list) #9

half_slice = math.floor(num_of_sales/2)
print(half_slice)

first_sales_items = sorted_list[:half_slice]
last_sales_items = sorted_list[-(half_slice):]

median = sorted_list[half_slice:(half_slice + 1)]

print(sorted_list)
print(num_of_sales)
print(first_sales_items)
print(last_sales_items)
print(median)

#Ryan example
def find_median_odd(num_list):
  sorted_list = sorted(num_list)
  list_len_mid = len(num_list)/2

  return sorted_list[math.floor(list_len_mid)]

print(find_median_odd(sale_prices))


#!/usr/bin/env python
import os
from setuptools import setup, find_packages

os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir)))
with open('README.rst') as f:
    README = f.read()
with open('pip_requirements.txt') as f:
    REQUIREMENTS = f.readlines()

setup(
    name='django-testing-base',
    version='0.4.0',
    packages=find_packages(exclude=['testsite']),
    include_package_data=True,
    install_requires=REQUIREMENTS,
    license='MIT License',
    description='Simple Django testing base classes and patterns',
    long_description=README,
    url='https://github.com/tctimmeh/django-testing-base',
    author='Tim Court',
    author_email='tctimmeh@gmail.com',
    classifiers=[
        'Development Status :: 3 - Alpha',
        'Environment :: Web Environment',
        'Framework :: Django',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: MIT License',
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.3',
        'Programming Language :: Python :: 3.4',
    ],
)


import quandl

def getdata(start, end, stock, column = ['date', 'ticker', 'adj_close']):
    quandl.ApiConfig.api_key = '_sbkgpPqyHKZTMdzkace'
    data = quandl.get_table('WIKI/PRICES', ticker = stock,
                        qopts = { 'columns': ['date', 'ticker', 'adj_close'] },
                        date = { 'gte': start, 'lte': end }, paginate=True)
    clean = data.set_index('date')
    table = clean.pivot(columns='ticker')
    return table


import matplotlib.pyplot as plt

import pandas as pd

plt.style.use("fivethirtyeight")

# Data :

data = pd.read_excel("Production Of Steel.xlsx")

# Plotting data :

plt.plot(data.Year, data.Production, marker = "o")

plt.fill_between(data.Year, data.Production, alpha = 0.25)

plt.title("Production Of Steel From Year 1990 To 1996")

plt.xlabel("Year >>>>")

plt.ylabel("Production >>>>")

plt.tight_layout()

plt.show()

''' Created By Sourin Das '''

Production Of Steel Comparison By Year.py


import difflib
import logging
import unittest

import pandas as pd

from exception.exceptions import NotAHuggingFaceUrl, NotATFHubUrl
from license_crawler import tidy_up_licenses
from scraper import HuggingFaceScraper, TFHubScraper

logging.basicConfig(level=getattr(logging, "DEBUG"))

class HuggingFaceScraperTestCase(unittest.TestCase):
    def test_single_url(self):
        url = "https://huggingface.co/facebook/detr-resnet-50-dc5-panoptic"
        license_ = HuggingFaceScraper().scrape_license(url)
        self.assertEqual("apache-2.0", license_)

    def test_not_valid_url(self):
        url = "https://stackoverflow.com/questions/31958637/beautifulsoup-search-by-text-inside-a-tag"
        self.assertRaises(NotAHuggingFaceUrl, HuggingFaceScraper().scrape_license, url)

class TFHubScraperTestCase(unittest.TestCase):
    def test_single_url(self):
        url = "https://tfhub.dev/google/compare_gan/model_14_cifar10_resnet_cifar/1"
        license_ = TFHubScraper().scrape_license(url)
        self.assertEqual("Apache-2.0", license_)

    def test_not_valid_url(self):
        url = "https://stackoverflow.com/questions/31958637/beautifulsoup-search-by-text-inside-a-tag"
        self.assertRaises(NotATFHubUrl, TFHubScraper().scrape_license, url)

class LicenseCrawlerTestCase(unittest.TestCase):
    def test_find_closest(self):
        df = pd.read_csv("output/license_output.csv")
        # print(df)
        df['License'] = df['License'].apply(
            lambda x: x.lower().replace("license", "").strip(" ") if type(x) is str else x)
        # logging.debug(df['License'].value_counts())
        words_list = [x[:10] for x in df['License'].value_counts().keys().to_list()]
        # logging.debug(words_list)
        new_word_list = []
        while len(words_list) != 0:
            word = words_list[0]

            words_list.remove(word)
            matches = difflib.get_close_matches(word, words_list, cutoff=0.6)

            words_list = [word for word in words_list if word not in matches]
            new_word_list.append(word)

        logging.debug(new_word_list)

        def func(x):
            logging.debug(difflib.get_close_matches(x, df['License'].value_counts().keys().to_list(), cutoff=0.7))
            return difflib.get_close_matches(x, df['License'].value_counts().keys().to_list(), cutoff=0.7)

        new_word_list = [func(word)[0] for word in new_word_list]
        logging.debug(new_word_list)

        def func2(x):
            # print(type(x))
            close = difflib.get_close_matches(str(x), new_word_list, cutoff=0.5)
            if len(close) < 1:
                return x
            # print("h"+str(close))
            return close[0]

        df['License'] = df['License'].apply(func2)

        logging.debug(df['License'].value_counts())

    def test_tidy_up(self):
        df = pd.read_csv("output/license_output.csv")
        df = tidy_up_licenses(df)
        print(df)

if __name__ == '__main__':
    unittest.main()


import numpy as np
import matplotlib.pyplot as plt
from iblatlas import atlas
from brainbox.atlas import plot_atlas

def combine_layers_cortex(regions, delete_duplicates=False):
    remove = ["1", "2", "3", "4", "5", "6a", "6b", "/"]
    for i, region in enumerate(regions):
        for j, char in enumerate(remove):
            regions[i] = regions[i].replace(char, "")
    if delete_duplicates:
        regions = list(set(regions))
    return regions

# Coordinates of slices in mm
ML = -0.5
AP = 1
DV = -2

# Generate some mock data
ba = atlas.AllenAtlas(25)
all_regions = ba.regions.acronym
regions = np.random.choice(all_regions, size=500, replace=False)  # pick 500 random regions
values = np.random.uniform(-1, 1, 500)  # generate 500 random values

# Plot atlas
f, axs = plt.subplots(2, 3, figsize=(20, 10))
plot_atlas(regions, values, ML, AP, DV, color_palette="RdBu_r", minmax=[-1, 1], axs=axs[0])

# Now combine all layers of cortex
plot_regions = combine_layers_cortex(regions)
combined_cortex = combine_layers_cortex(all_regions)

# Plot atlas
plot_atlas(
    plot_regions,
    values,
    ML,
    AP,
    DV,
    color_palette="RdBu_r",
    minmax=[-1, 1],
    axs=axs[1],
    custom_region_list=combined_cortex,
)


import numpy as np
import sys

sys.path.insert(0, '../src')

from core import Sequential
from dsp.modem import get_alphabet, Modulator, Demodulator
from dsp.optical import FIR_CD_Compensator, LS_FIR_CD_Compensator
from dsp.frontend import Upsampler, Downsampler, SRRC_filter
from channels.optical import CD
from channels.noise import awgn_theo, AWGN
from metrics.functional import compute_ber
import matplotlib.pyplot as plt

# This script reproduces the figures 3 and 4 in the paper Optimal Least-Squares FIR Digital Filters
#for Compensation of Chromatic Dispersion in Digital Coherent Optical Receivers
# see: https://www.researchgate.net/profile/Amir-Eghbali-2/publication/260642085_Optimal_Least-Squares_FIR_Digital_Filters_for_Compensation_of_Chromatic_Dispersion_in_Digital_Coherent_Optical_Receivers/links/02e7e534adbd8b0d77000000/Optimal-Least-Squares-FIR-Digital-Filters-for-Compensation-of-Chromatic-Dispersion-in-Digital-Coherent-Optical-Receivers.pdf

system = 2
type = "QAM"
k_vect = [4, 6, 8]
N = 200000  # increase number for smoothing MC curves
N_h = 1000  # number of delay samples for the SRRC filter
oversampling = 2  # oversampling factor
rolloff = 0.25  # rolloff factor
D = 17*(10**-6)
lamb=1.553*(10**-6)
SNR_vect = range(10, 31, 1)

# system selection (see Table 1)
if system == 1:
    z = 4000*(10**3)
    F_s = 21.4 * (10**9)

if system == 2:
    z = 1000*(10**3)
    F_s = 80 * (10**9)

comp_ref = FIR_CD_Compensator(z, D=D, lamb=lamb, F_s=F_s)
N_filter = comp_ref.get_N()  # compute length of filter
print("filter length: {}".format(N_filter))
total_delay = int(2*N_h*oversampling + np.floor(N_filter/2)) # total delay of the filters (need to be compensated before sampling)

plt.figure()

noise = AWGN(0)

for k in k_vect:

    M = 2**k

    print("Modulation: {}{}".format(type,M))
    alphabet = get_alphabet(type, M)
    epsilon_s = np.sum(np.abs(alphabet)**2)/len(alphabet)
    epsilon_b = epsilon_s/k

    # create your chain
    channel = Sequential([
                Modulator(alphabet),
                Upsampler(oversampling),
                SRRC_filter(rolloff, oversampling, N_h=N_h),
                CD(z, D=D, lamb=lamb, F_s=F_s),
                noise
                ])

    full_compensator1 = Sequential([
                FIR_CD_Compensator(z, D=D, lamb=lamb, F_s=F_s),
                SRRC_filter(rolloff, oversampling, N_h=N_h),
                Downsampler(oversampling, pre_delay=total_delay),
                Demodulator(alphabet)
                ])

    full_compensator2 = Sequential([
                LS_FIR_CD_Compensator(z, N_filter, D=D, lamb=lamb, F_s=F_s, w_vect=[-np.pi, np.pi]),
                SRRC_filter(rolloff, oversampling, N_h=N_h),
                Downsampler(oversampling, pre_delay=total_delay),
                Demodulator(alphabet)
                ])

    compensator_list = [full_compensator1, full_compensator2]

    # monte carlo simulations
    ber_theo = []
    ber_exp = []
    ber_list_names = ["theoretical", "Savory", "LS"]
    ber_list = np.zeros((len(SNR_vect),3))

    for index_SNR, SNR_bitdB in enumerate(SNR_vect):
        snr_per_bit = 10 ** (SNR_bitdB/10)

        # compute theoretical ber
        ber_theo = awgn_theo(type, M, snr_per_bit, "bin")
        ber_list[index_SNR, 0] = ber_theo

        # perform MC simulations
        N0 = epsilon_b/snr_per_bit  #snr_bit = epsilon_b/N0 = (epsilon_s/log2(order))/N0
        noise.sigma2 = N0  # change noise variance
        s = np.random.randint(0, high=M, size=N)
        y = channel(s)

        # get compensated symbols
        for index_comp, full_compensator in enumerate(compensator_list):

            s_est = full_compensator(y)
            s_trunc = s_est[:N]
            ber_exp_temp = compute_ber(s_trunc, s, width=k)
            ber_list[index_SNR, index_comp+1] = ber_exp_temp

        print("SNR={}dB/bit".format(SNR_bitdB))
        print("BER={}".format(ber_list[index_SNR, :]))

    # plot curves
    color_list = ["k-","r-","b-"]
    for index in range(3):
        name = "{} ({}{})".format(ber_list_names[index],type,M)
        plt.semilogy(SNR_vect, ber_list[:, index], color_list[index], label=name)

plt.ylabel("BER")
plt.xlabel("SNR per bit (dB)")
plt.ylim([10**-6,0.2])
plt.xlim([SNR_vect[0], SNR_vect[-1]])
plt.legend()
plt.show()


from django.db.models.signals import post_save, post_delete
from django.dispatch import receiver

from my_auth.models import MyUser
from .models import Profile, Team

@receiver(post_save, sender = MyUser)
def user_created(sender, instance, created, **kwargs):
    Profile.objects.create(user=instance)
    print("OKOKOK")

@receiver(post_delete, sender = Profile)
def profile_deleted(sender, instance, **kwargs):
    team_id = instance.team
    team = Team.objects.get(id=team_id)
    if team.profile_set.count() == 0:
        team.delete()
        print("ok")


def palindrome(pal):
    ar = []
    palList = list(str(pal))
    for i in range(0,int(len(palList)/2)):
        if palList[int(len(palList))-i-1] != palList[i]:
            return False
    return True

max =[]
for i in range(1000,100,-1):
    for x in range(1000,100,-1):
        if palindrome(i*x):
            max.append(i*x)

tmp = sorted(max,reverse=True)

print(tmp)


import numpy as np
from rover import rover_obj
import sys

"""
.. module:: example_expected_improvement
  :synopsis: Example Expected Improvement
.. moduleauthor:: David Eriksson <dme65@cornell.edu>
"""

###############################################################################
# Copyright (c) 2019 Uber Technologies, Inc.                                  #
#                                                                             #
# Licensed under the Uber Non-Commercial License (the "License");             #
# you may not use this file except in compliance with the License.            #
# You may obtain a copy of the License at the root directory of this project. #
#                                                                             #
# See the License for the specific language governing permissions and         #
# limitations under the License.                                              #
###############################################################################

import math
import sys
from copy import deepcopy

import gpytorch
import numpy as np
import torch
from torch.quasirandom import SobolEngine

from exact_gp import train_gp
from turbo_utils import from_unit_cube, latin_hypercube, to_unit_cube

class Turbo1:
    """The TuRBO-1 algorithm.

    Parameters
    ----------
    f : function handle
    lb : Lower variable bounds, numpy.array, shape (d,).
    ub : Upper variable bounds, numpy.array, shape (d,).
    n_init : Number of initial points (2*dim is recommended), int.
    max_evals : Total evaluation budget, int.
    batch_size : Number of points in each batch, int.
    verbose : If you want to print information about the optimization progress, bool.
    use_ard : If you want to use ARD for the GP kernel.
    max_cholesky_size : Largest number of training points where we use Cholesky, int
    n_training_steps : Number of training steps for learning the GP hypers, int
    min_cuda : We use float64 on the CPU if we have this or fewer datapoints
    device : Device to use for GP fitting ("cpu" or "cuda")
    dtype : Dtype to use for GP fitting ("float32" or "float64")

    Example usage:
        turbo1 = Turbo1(f=f, lb=lb, ub=ub, n_init=n_init, max_evals=max_evals)
        turbo1.optimize()  # Run optimization
        X, fX = turbo1.X, turbo1.fX  # Evaluated points
    """

    def __init__(
        self,
        f,
        lb,
        ub,
        n_init,
        max_evals,
        batch_size=1,
        verbose=True,
        use_ard=True,
        max_cholesky_size=2000,
        n_training_steps=50,
        min_cuda=1024,
        device="cpu",
        dtype="float64",
    ):

        # Very basic input checks
        assert lb.ndim == 1 and ub.ndim == 1
        assert len(lb) == len(ub)
        assert np.all(ub > lb)
        assert max_evals > 0 and isinstance(max_evals, int)
        assert n_init > 0 and isinstance(n_init, int)
        assert batch_size > 0 and isinstance(batch_size, int)
        assert isinstance(verbose, bool) and isinstance(use_ard, bool)
        assert max_cholesky_size >= 0 and isinstance(batch_size, int)
        assert n_training_steps >= 30 and isinstance(n_training_steps, int)
        assert max_evals > n_init and max_evals > batch_size
        assert device == "cpu" or device == "cuda"
        assert dtype == "float32" or dtype == "float64"
        if device == "cuda":
            assert torch.cuda.is_available(), "can't use cuda if it's not available"

        # Save function information
        self.f = f
        self.dim = len(lb)
        self.lb = lb
        self.ub = ub

        # LCB beta
        self.lcb_beta = 10.0

        # Settings
        self.n_init = n_init
        self.max_evals = max_evals
        self.batch_size = batch_size
        self.verbose = verbose
        self.use_ard = use_ard
        self.max_cholesky_size = max_cholesky_size
        self.n_training_steps = n_training_steps

        # Hyperparameters
        self.mean = np.zeros((0, 1))
        self.signal_var = np.zeros((0, 1))
        self.noise_var = np.zeros((0, 1))
        self.lengthscales = np.zeros((0, self.dim)) if self.use_ard else np.zeros((0, 1))

        # Tolerances and counters
        self.n_cand = min(100 * self.dim, 5000)
        self.failtol = np.ceil(np.max([4.0 / batch_size, self.dim / batch_size]))
        self.succtol = 3
        self.n_evals = 0

        # Trust region sizes
        self.length_min = 0.5 ** 7
        self.length_max = 1.6
        self.length_init = 0.8

        # Save the full history
        self.X = np.zeros((0, self.dim))
        self.fX = np.zeros((0, 1))

        # Device and dtype for GPyTorch
        self.min_cuda = min_cuda
        self.dtype = torch.float32 if dtype == "float32" else torch.float64
        self.device = torch.device("cuda") if device == "cuda" else torch.device("cpu")
        if self.verbose:
            print("Using dtype = %s \nUsing device = %s" % (self.dtype, self.device))
            sys.stdout.flush()

        # Initialize parameters
        self._restart()

    def _restart(self):
        self._X = []
        self._fX = []
        self.failcount = 0
        self.succcount = 0
        self.length = self.length_init

    def _adjust_length(self, fX_next):
        if np.min(fX_next) < np.min(self._fX) - 1e-3 * math.fabs(np.min(self._fX)):
            self.succcount += 1
            self.failcount = 0
        else:
            self.succcount = 0
            self.failcount += 1

        if self.succcount == self.succtol:  # Expand trust region
            self.length = min([2.0 * self.length, self.length_max])
            self.succcount = 0
        elif self.failcount == self.failtol:  # Shrink trust region
            self.length /= 2.0
            self.failcount = 0

    def _create_candidates(self, X, fX, length, n_training_steps, hypers):
        """Generate candidates assuming X has been scaled to [0,1]^d."""
        # Pick the center as the point with the smallest function values
        # NOTE: This may not be robust to noise, in which case the posterior mean of the GP can be used instead
        assert X.min() >= 0.0 and X.max() <= 1.0

        # Standardize function values.
        mu, sigma = np.median(fX), fX.std()
        sigma = 1.0 if sigma < 1e-6 else sigma
        fX = (deepcopy(fX) - mu) / sigma

        # Figure out what device we are running on
        if len(X) < self.min_cuda:
            device, dtype = torch.device("cpu"), torch.float64
        else:
            device, dtype = self.device, self.dtype

        # We use CG + Lanczos for training if we have enough data
        with gpytorch.settings.max_cholesky_size(self.max_cholesky_size):
            X_torch = torch.tensor(X).to(device=device, dtype=dtype)
            y_torch = torch.tensor(fX).to(device=device, dtype=dtype)
            gp = train_gp(
                train_x=X_torch, train_y=y_torch, use_ard=self.use_ard, num_steps=n_training_steps, hypers=hypers
            )

            # Save state dict
            hypers = gp.state_dict()

        # Create the trust region boundaries
        x_center = X[fX.argmin().item(), :][None, :]
        lb = np.zeros(self.dim)
        ub = np.ones(self.dim)

        # Create candidate points
        X_cand = np.random.uniform(lb,ub,(self.n_cand,self.dim))

        # Figure out what device we are running on
        if len(X_cand) < self.min_cuda:
            device, dtype = torch.device("cpu"), torch.float64
        else:
            device, dtype = self.device, self.dtype

        # We may have to move the GP to a new device
        gp = gp.to(dtype=dtype, device=device)

        # We use Lanczos for sampling if we have enough data
        with torch.no_grad(), gpytorch.settings.max_cholesky_size(self.max_cholesky_size):
            X_cand_torch = torch.tensor(X_cand).to(device=device, dtype=dtype)
            preds = gp.likelihood(gp(X_cand_torch))
            y_cand = preds.mean.cpu().detach().numpy() - self.lcb_beta*preds.variance.cpu().detach().numpy()

        # Remove the torch variables
        del X_torch, y_torch, X_cand_torch, gp

        # De-standardize the sampled values
        y_cand = mu + sigma * y_cand

        return X_cand, y_cand, hypers

    def _select_candidates(self, X_cand, y_cand):
        """Select candidates."""
        idx_sort = np.argsort(y_cand)[:batch_size]
        X_next = X_cand[idx_sort,:]
        return X_next

    def optimize(self):
        """Run the full optimization process."""
        while self.n_evals < self.max_evals:

            if len(self._fX) > 0 and self.verbose:
                n_evals, fbest = self.n_evals, self._fX.min()
                print(f"{n_evals}) Restarting with fbest = {fbest:.4}")
                sys.stdout.flush()

            # Initialize parameters
            self._restart()

            # Generate and evalute initial design points
            X_init = latin_hypercube(self.n_init, self.dim)
            X_init = from_unit_cube(X_init, self.lb, self.ub)
            fX_init = np.array([[self.f(x)] for x in X_init])

            # Update budget and set as initial data for this TR
            self.n_evals += self.n_init
            self._X = deepcopy(X_init)
            self._fX = deepcopy(fX_init)

            # Append data to the global history
            self.X = np.vstack((self.X, deepcopy(X_init)))
            self.fX = np.vstack((self.fX, deepcopy(fX_init)))

            if self.verbose:
                fbest = self._fX.min()
                print(f"Starting from fbest = {fbest:.4}")
                sys.stdout.flush()

            # Thompson sample to get next suggestions
            while self.n_evals < self.max_evals and self.length >= self.length_min:

                # Warp inputs
                X = to_unit_cube(deepcopy(self._X), self.lb, self.ub)

                # Standardize values
                fX = deepcopy(self._fX).ravel()

                # Create th next batch
                X_cand, y_cand, _ = self._create_candidates(
                    X, fX, length=self.length, n_training_steps=self.n_training_steps, hypers={}
                )
                X_next = self._select_candidates(X_cand, y_cand)

                # Undo the warping
                X_next = from_unit_cube(X_next, self.lb, self.ub)

                # Evaluate batch
                fX_next = np.array([[self.f(x)] for x in X_next])

                # Update trust region
                # self._adjust_length(fX_next)

                # Update budget and append data
                self.n_evals += self.batch_size
                self._X = np.vstack((self._X, X_next))
                self._fX = np.vstack((self._fX, fX_next))

                if self.verbose and fX_next.min() < self.fX.min():
                    n_evals, fbest = self.n_evals, fX_next.min()
                    print(f"{n_evals}) New best: {fbest:.4}")
                    sys.stdout.flush()

                # Append data to the global history
                self.X = np.vstack((self.X, deepcopy(X_next)))
                self.fX = np.vstack((self.fX, deepcopy(fX_next)))

if __name__ == '__main__':

  dim = 200
  n_init = 100
  max_evals = 2000
  lb = -5 * np.ones(dim)
  ub = 5 * np.ones(dim)
  batch_size = 5
  num_epochs = 100

  from datetime import datetime
  now     = datetime.now()
  seed    = int("%d%.2d%.2d%.2d%.2d"%(now.month,now.day,now.hour,now.minute,now.second))
  barcode = "%d%.2d%.2d%.2d%.2d%.2d"%(now.year,now.month,now.day,now.hour,now.minute,now.second)
  np.random.seed(seed)

  if torch.cuda.is_available():
    turbo_device = 'cuda'
  else:
    turbo_device = 'cpu'

  # initialize TuRBO
  problem = Turbo1(
        rover_obj,
        lb=lb,ub=ub,
        n_init=n_init,
        max_evals=max_evals,
        batch_size=batch_size,
        verbose=True,
        use_ard=True,
        max_cholesky_size=2000,
        n_training_steps=num_epochs,
        min_cuda=0,
        device=turbo_device,
        dtype="float64")

  # optimize
  problem.optimize()
  X, fX = problem.X, problem.fX.flatten()  # Evaluated points

  d ={}
  d['X']  = X
  d['fX'] = fX
  d['mode'] = "BO-LCB"
  outfilename = f"./output/data_rover_BO_{max_evals}_evals_{barcode}.pickle"
  import pickle
  pickle.dump(d,open(outfilename,"wb"))


n = int(input("Enter the size of the square matrix: "))
matrix1 = []
matrix2 = []
result = []

print("Enter the values for the first matrix:")
for i in range(n):
    row = []
    for j in range(n):
        value = int(input("Enter the value for row {} and column {}: ".format(i+1, j+1)))
        row.append(value)
    matrix1.append(row)

print("Enter the values for the second matrix:")
for i in range(n):
    row = []
    for j in range(n):
        value = int(input("Enter the value for row {} and column {}: ".format(i+1, j+1)))
        row.append(value)
    matrix2.append(row)

for i in range(n):
    row = []
    for j in range(n):
        value = matrix1[i][j] + matrix2[i][j]
        row.append(value)
    result.append(row)

print("The sum of the matrices is:")
for row in result:
    print(row)


"""
Create time-stacked NetCDF files

"""
from __future__ import absolute_import, print_function, division

import copy
import datetime
import itertools
import logging
import os
import socket
from functools import partial

import click
import dask.array as da
import dask
from dateutil import tz
from pathlib import Path

import datacube
from datacube.model import Dataset
from datacube.model.utils import xr_apply, datasets_to_doc
from datacube.utils import mk_part_uri
from datacube.storage import netcdf_writer
from datacube.storage.storage import create_netcdf_storage_unit
from datacube.ui import task_app

_LOG = logging.getLogger(__name__)

APP_NAME = 'datacube-stacker'

def get_filename(config, cell_index, year):
    file_path_template = str(Path(config['location'], config['file_path_template']))
    return file_path_template.format(tile_index=cell_index, start_time=year, version=config['taskfile_version'])

def get_temp_file(final_output_path):
    """
    Get a temp file path
    Changes "/path/file.nc" to "/path/.tmp/file.nc.host.pid.tmp"
    :param Path final_output_path:
    :return: Path to temporarily write output
    :rtype: Path
    """

    tmp_folder = final_output_path.parent / '.tmp'
    id_file = '{host}.{pid}'.format(host=socket.gethostname(), pid=os.getpid())
    tmp_path = (tmp_folder / final_output_path.stem).with_suffix(final_output_path.suffix + id_file + '.tmp')
    try:
        tmp_folder.mkdir(parents=True)
    except OSError:
        pass
    if tmp_path.exists():
        tmp_path.unlink()
    return tmp_path

def make_stacker_tasks(index, config, cell_index=None, time=None, **kwargs):
    gw = datacube.api.GridWorkflow(index=index, product=config['product'].name)

    for query in task_app.break_query_into_years(time):
        cells = gw.list_cells(product=config['product'].name, cell_index=cell_index, **query)
        for cell_index_key, tile in cells.items():
            for year, year_tile in tile.split_by_time(freq='A'):
                storage_files = set(ds.local_path for ds in itertools.chain(*year_tile.sources.values))
                if len(storage_files) > 1:
                    year_tile = gw.update_tile_lineage(year_tile)
                    output_filename = get_filename(config, cell_index_key, year)
                    _LOG.info('Stacking required for: year=%s, cell=%s. Output=%s',
                              year, cell_index_key, output_filename)
                    yield dict(year=year,
                               tile=year_tile,
                               cell_index=cell_index_key,
                               output_filename=output_filename)
                elif len(storage_files) == 1:
                    [only_filename] = storage_files
                    _LOG.info('Stacking not required for: year=%s, cell=%s. existing=%s',
                              year, cell_index_key, only_filename)

def make_stacker_config(index, config, export_path=None, check_data=None, **query):
    config['product'] = index.products.get_by_name(config['output_type'])

    if export_path is not None:
        config['location'] = export_path
        config['index_datasets'] = False
    else:
        config['index_datasets'] = True

    if check_data is not None:
        config['check_data_identical'] = check_data

    if not os.access(config['location'], os.W_OK):
        _LOG.warning('Current user appears not have write access output location: %s', config['location'])

    chunking = config['storage']['chunking']
    chunking = [chunking[dim] for dim in config['storage']['dimension_order']]

    var_param_keys = {'zlib', 'complevel', 'shuffle', 'fletcher32', 'contiguous', 'attrs'}
    variable_params = {}
    for mapping in config['measurements']:
        varname = mapping['name']
        variable_params[varname] = {k: v for k, v in mapping.items() if k in var_param_keys}
        variable_params[varname]['chunksizes'] = chunking

    config['variable_params'] = variable_params

    config['taskfile_version'] = int(datacube.utils.datetime_to_seconds_since_1970(datetime.datetime.now()))

    return config

def get_history_attribute(config, task):
    return '{dt} {user} {app} ({ver}) {args}  # {comment}'.format(
        dt=datetime.datetime.now(tz.tzlocal()).isoformat(),
        user=os.environ.get('USERNAME') or os.environ.get('USER'),
        app=APP_NAME,
        ver=datacube.__version__,
        args=', '.join([config['app_config_file'],
                        str(config['version']),
                        str(config['taskfile_version']),
                        task['output_filename'],
                        str(task['year']),
                        str(task['cell_index'])
                       ]),
        comment='Stacking datasets for a year into a single NetCDF file'
    )

def _unwrap_dataset_list(labels, dataset_list):
    return dataset_list[0]

def do_stack_task(config, task):
    global_attributes = config['global_attributes']
    global_attributes['history'] = get_history_attribute(config, task)

    variable_params = config['variable_params']

    variable_params['dataset'] = {
        'chunksizes': (1,),
        'zlib': True,
        'complevel': 9,
    }

    output_filename = Path(task['output_filename'])
    output_uri = output_filename.absolute().as_uri()
    temp_filename = get_temp_file(output_filename)
    tile = task['tile']

    # Only use the time chunk size (eg 5), but not spatial chunks
    # This means the file only gets opened once per band, and all data is available when compressing on write
    # 5 * 4000 * 4000 * 2bytes == 152MB, so mem usage is not an issue
    chunk_profile = {'time': config['storage']['chunking']['time']}

    data = datacube.api.GridWorkflow.load(tile, dask_chunks=chunk_profile)

    unwrapped_datasets = xr_apply(tile.sources, _unwrap_dataset_list, dtype='O')
    data['dataset'] = datasets_to_doc(unwrapped_datasets)

    try:
        nco = create_netcdf_storage_unit(temp_filename,
                                         data.crs,
                                         data.coords,
                                         data.data_vars,
                                         variable_params,
                                         global_attributes)
        write_data_variables(data.data_vars, nco)
        nco.close()

        temp_filename.rename(output_filename)

        if config.get('check_data_identical', False):
            new_tile = make_updated_tile(unwrapped_datasets, output_uri, tile.geobox)
            new_data = datacube.api.GridWorkflow.load(new_tile, dask_chunks=chunk_profile)
            check_identical(data, new_data, output_filename)

    except Exception as e:
        if temp_filename.exists():
            temp_filename.unlink()
        raise e

    return unwrapped_datasets, output_uri

def write_data_variables(data_vars, nco):
    for name, variable in data_vars.items():
        try:
            with dask.set_options(get=dask.local.get_sync):
                da.store(variable.data, nco[name], lock=True)
        except ValueError:
            nco[name][:] = netcdf_writer.netcdfy_data(variable.values)
        nco.sync()

def check_identical(data1, data2, output_filename):
    _LOG.debug('Verifying file: "%s"', output_filename)
    with dask.set_options(get=dask.local.get_sync):
        if not all((data1 == data2).all().values()):
            _LOG.error("Mismatch found for %s, not indexing", output_filename)
            raise ValueError("Mismatch found for %s, not indexing" % output_filename)
    return True

def make_updated_tile(old_datasets, new_uri, geobox):
    def update_dataset_location(idx, labels, dataset):
        # type: ((int,), object, Dataset) -> list
        idx, = idx
        new_dataset = copy.copy(dataset)
        new_dataset.uris = [mk_part_uri(new_uri, idx)]
        return [new_dataset]

    updated_datasets = xr_apply(old_datasets, update_dataset_location, with_numeric_index=True)
    return datacube.api.Tile(sources=updated_datasets, geobox=geobox)

def process_result(index, result):
    datasets, new_common_uri = result
    for idx, dataset in enumerate(datasets.values):
        new_uri = mk_part_uri(new_common_uri, idx)
        _LOG.info('Updating dataset location: %s', dataset.local_path)
        old_uri = dataset.local_uri
        index.datasets.add_location(dataset.id, new_uri)
        index.datasets.archive_location(dataset.id, old_uri)

@click.command(name=APP_NAME)
@datacube.ui.click.pass_index(app_name=APP_NAME)
@datacube.ui.click.global_cli_options
@click.option('--cell-index', 'cell_index', help='Limit the process to a particular cell (e.g. 14,-11)',
              callback=task_app.validate_cell_index, default=None)
@click.option('--year', 'time', callback=task_app.validate_year, help='Limit the process to a particular year')
@click.option('--export-path', 'export_path',
              help='Write the stacked files to an external location without updating the index',
              default=None,
              type=click.Path(exists=True, writable=True, file_okay=False))
@click.option('--check-data/--no-check-data', is_flag=True, default=None,
              help="Overrides config option: check_data_identical")
@task_app.queue_size_option
@task_app.task_app_options
@task_app.task_app(make_config=make_stacker_config, make_tasks=make_stacker_tasks)
def main(index, config, tasks, executor, queue_size, **kwargs):
    """Store datasets into NetCDF files containing an entire year in the same file.

    - Uses the same configuration format as the `ingest` tool.
    - However, does not create new datasets, but instead updates dataset locations then archives the original location.
    """
    click.echo('Starting stacking utility...')

    task_func = partial(do_stack_task, config)
    process_func = partial(process_result, index) if config['index_datasets'] else None
    task_app.run_tasks(tasks, executor, task_func, process_func, queue_size)

if __name__ == '__main__':
    main()


#!/usr/bin/python
import socket
import string
import time
import requests
import config
import subprocess

while True:
	sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
	sock.bind((config.UDP_IP, config.UDP_PORT))

	data, addr = sock.recvfrom(4096)
	#print data

	gpsInfoNorth = data.split("+")
	gpsInfoWest = gpsInfoNorth[1].split("-")
	gpsInfoTruck = gpsInfoWest[1].split("=")
	gpsInfoTruck = gpsInfoTruck[1].split(";")

	gpsNorth = gpsInfoNorth[1][:2] + "." + gpsInfoNorth[1][2:7]
	gpsWest = "-" + gpsInfoWest[1][:3] + "." + gpsInfoWest[1][3:8]
	gpsTruck = gpsInfoTruck[0]

	print (gpsNorth + ", " + gpsWest + " - " + gpsTruck)
	print (config.urlServer + "?gpsN=" + gpsNorth + "&gpsW=" + gpsWest + "&truckNbr=" + gpsTruck)

	# Time to pass the GPS info to the internal server
	r = requests.get(config.urlServer + "?gpsN=" + gpsNorth + "&gpsW=" + gpsWest + "&truckNbr=" + gpsTruck)
	#print (r.status_code)
	#print (r.headers)
	#print (r.content)

	clientsocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	clientsocket.connect(('localhost', 5830))
	clientsocket.send("gpsN=" + gpsNorth + "&gpsW=" + gpsWest + "&truckNbr=" + gpsTruck)

	# Sleep for 5 seconds
	time.sleep(5)
else:
	rebootCommand = "sudo /home/pi/Documents/scripts/reboot.sh"
	output = subprocess.check_output(['bash','-c', rebootCommand])

sock.close()


import unittest

from ecotricity.client.dto import AccountsResult, AccountsData

class TestAccountsResult(unittest.TestCase):

    def test_deserialisation(self):
        json = '{"data": {"results": []}, "status": "success"}'
        entity = AccountsResult.from_json(json)
        self.assertEqual(entity.status, "success")
        self.assertIsInstance(entity.data, AccountsData)


import os
import logging
import sys
import matplotlib.pyplot as plt
#os.environ['PROJ_LIB'] = r'C:\ProgramData\Anaconda3\pkgs\proj4-5.2.0-hfa6e2cd_1001\Library\share'
from mpl_toolkits.basemap import Basemap
import pickle
import numpy as np
import ModuloGeneralReadOptions as MGRO

class Basemaps(object):

    def get_ESRIBasemap(lon_min,lon_max,lat_min,lat_max,self):
        _m   = 'm.pickle'
        _fig = 'fig.pickle'

        try:
            if os.path.isfile(_m) and os.path.isfile(_fig) :
                try:
                    logging.info(': Looks like that i already have BasemapConfig. Using old configuration.')

                except Exception as ex:
                    logging.info(': Modulo Maps Basemaps : Error 001 : Failed to load old configuration')
                    logging.shutdown()
                    sys.exit()
            else:
                fig=plt.figure()

                logging.info(': Accessing WMS World Image for the first time')
                m = Basemap(
                        ellps='WGS84',
                        llcrnrlat=lat_min,
                        llcrnrlon=lon_min,
                        urcrnrlat=lat_max,
                        urcrnrlon=lon_max,
                        resolution=None)

                attempts=0
                success=False

                while attempts < 3 and not success:
                    try:
                        logging.info(': Modulo Basemaps : wms WorldImage access try #' + str(attempts+1))
                        m.arcgisimage(server='http://server.arcgisonline.com/ArcGIS', service=self.options.worldImage_esri, xpixels=1920, ypixels=None, dpi=96, verbose=False)
                        success = True
                        m.drawparallels(np.arange(self.lat_minf,self.lat_maxf,self.options.dy),
                                        labels=[1,0,0,1],
                                        labelstyle='+/-',
                                        color='None',
                                        fontsize=self.options.fontsize-1)

                        m.drawmeridians(np.arange(self.lon_minf,self.lon_maxf,self.options.dx),
                                        labels=[1,0,0,1],
                                        labelstyle='+/-',
                                        color='None',
                                        fontsize=self.options.fontsize-1)

                        with open(_m, 'wb') as fin:
                            m = pickle.dump(m,fin)
                        with open(_fig, 'wb') as fin1:
                            fig = pickle.dump(fig,fin1)

                    except Exception as ex:
                        logging.info(': Modulo Maps Basemaps : Error 002 : wms WorldImage access try #'+
                                     str(attempts+1) + ' failed. Trying again in 60s. ')
                        time.sleep(60)
                        # after fail will w8 for 60 seconds before retry to connect with wms
                        attempts += 1
                return m
            del _m, _fig
        except Exception as ex:
            logging.info(': Modulo Maps Basemaps : Error 003 : Fail to generate')
            logging.shutdown()
            sys.exit()

    def get_ESRIBasemap_Validation(self):
        try:
            m= Basemap(
                    ellps='WGS84',
                    llcrnrlat=self.options.Ymin,
                    llcrnrlon=self.options.Xmin,
                    urcrnrlat=self.options.Ymax,
                    urcrnrlon=self.options.Xmax,
                    resolution=None)
            attempts=0
            success=False

            while attempts < 3 and not success:
                try:
                    logging.info(': Modulo Maps Basemaps : wms WorldImage access try #' + str(attempts+1))
                    m.arcgisimage(server='http://server.arcgisonline.com/ArcGIS', service=self.options.worldImage_esri, xpixels=1920, ypixels=None, dpi=96, verbose=False)
                    success = True

                    lala='%.0f'
                    if self.options.dx >=.1 and self.options.dx <1:
                        lala = '%.1f'
                    elif self.options.dx >=.01 and self.options.dx <.1:
                        lala = '%.2f'

                    paralels = m.drawparallels(np.arange(self.lat_minf,self.lat_maxf,self.options.dy),labels=[1,0,0,1],
                                               labelstyle='+/-',
                                               color='None',
                                               fontsize=self.options.fontsize-1,
                                               fmt=lala)
                    meridians  = m.drawmeridians(np.arange(self.lon_minf,self.lon_maxf,self.options.dx),
                                                 labels=[1,0,0,1],
                                                 labelstyle='+/-',
                                                 color='None',
                                                 fontsize=self.options.fontsize-1,
                                                 fmt=lala)

                except Exception as ex:
                    logging.info(': Modulo Maps Basemaps : Error 004 : wms WorldImage access try #'+
                                    str(attempts+1) + ' failed. Trying again in 60s. ')
                    time.sleep(60)
                    # after fail will w8 for 60 seconds before retry to connect with wms
                    attempts += 1
        except Exception as ex:
            logging.info(': Modulo Maps Basemaps : Error 005 : Failed to get ESRImaps data ')
            logging.shutdown()
            sys.exit()

    def get_GSSH(self):
        try:
            logging.info(': Modulo Maps Basemaps : Accessing GSSH coastline')

            lala='%.0f'
            if self.options.dx >=.1 and self.options.dx <1:
                lala = '%.1f'
            elif self.options.dx >=.01 and self.options.dx <.1:
                lala = '%.2f'
            elif self.options.dx >=.001 and self.options.dx <.01:
                lala = '%.3f'
            elif self.options.dx >=.0001 and self.options.dx <.001:
                lala = '%.4f'

            m = Basemap(llcrnrlon=self.options.Xmin,llcrnrlat=self.options.Ymin,urcrnrlon=self.options.Xmax,urcrnrlat=self.options.Ymax,resolution=self.options.resolution)
            m.drawcoastlines(linewidth=0.4, linestyle='solid', color='k')
            m.drawcountries(linewidth=0.2, linestyle='solid', color='k')
            if self.options.fillContinents == 1:
                m.fillcontinents(color='white')
                if self.options.fillContinents_color is not 'white':
                    m.fillcontinents(color=self.options.fillContinents_color)
            paralels = m.drawparallels(np.arange(self.lat_minf,self.lat_maxf,self.options.dy),labels=[1,0,0,1],
                                        labelstyle='+/-',
                                        color='None',
                                        fontsize=self.options.fontsize-1,
                                        fmt=lala)
            meridians  = m.drawmeridians(np.arange(self.lon_minf,self.lon_maxf,self.options.dx),
                                            labels=[1,0,0,1],
                                            labelstyle='+/-',
                                            color='None',
                                            fontsize=self.options.fontsize-1,
                                            fmt=lala)
        except Exception as ex:
            logging.info(': Modulo Maps Basemaps : Error 006 : Failed to get Coastline data ')
            logging.shutdown()
            sys.exit()


'''

-Hard-
*Math*

You want to build n new buildings in a city. The new buildings will be built in a line
and are labeled from 1 to n.

However, there are city restrictions on the heights of the new buildings:

The height of each building must be a non-negative integer.
The height of the first building must be 0.
The height difference between any two adjacent buildings cannot exceed 1.
Additionally, there are city restrictions on the maximum height of specific buildings.
These restrictions are given as a 2D integer array restrictions where
restrictions[i] = [idi, maxHeighti] indicates that building idi must have a height
less than or equal to maxHeighti.

It is guaranteed that each building will appear at most once in restrictions, and
building 1 will not be in restrictions.

Return the maximum possible height of the tallest building.

Example 1:

Input: n = 5, restrictions = [[2,1],[4,1]]
Output: 2
Explanation: The green area in the image indicates the maximum allowed height for each building.
We can build the buildings with heights [0,1,2,1,2], and the tallest building has a height of 2.
Example 2:

Input: n = 6, restrictions = []
Output: 5
Explanation: The green area in the image indicates the maximum allowed height for each building.
We can build the buildings with heights [0,1,2,3,4,5], and the tallest building has a height of 5.
Example 3:

Input: n = 10, restrictions = [[5,3],[2,5],[7,4],[10,3]]
Output: 5
Explanation: The green area in the image indicates the maximum allowed height for each building.
We can build the buildings with heights [0,1,2,3,3,4,4,5,4,3], and the tallest building has a height of 5.

Constraints:

2 <= n <= 10^9
0 <= restrictions.length <= min(n - 1, 10^5)
2 <= idi <= n
idi is unique.
0 <= maxHeighti <= 10^9

'''

from typing import List

class Solution:
    def maxBuilding(self, n: int, restrictions: List[List[int]]) -> int:
        arr = restrictions
        arr.extend([[1, 0], [n, n - 1]])
        arr.sort()
       # print(arr)
        m = len(arr)
        # there can be lots of restrictions on both sides and sometimes the adjacent
        # restrictions are not the tightest bound. We solve this issue by first "propagating"
        # restrictions from left to right and then from right to right.
        for i in range(1, m):
            arr[i][1] = min(arr[i][1], arr[i-1][1] + arr[i][0] - arr[i-1][0])
        for i in range(m - 2, -1, -1):
            arr[i][1] = min(arr[i][1], arr[i+1][1] + arr[i+1][0] - arr[i][0])
        #print(arr)
        ans = 0
        for i in range(1, m):
            l, h1 = arr[i-1]
            r, h2 = arr[i]
            ans = max(ans, max(h1, h2) + (r - l - abs(h1 - h2)) // 2)
         #   print('ans:',ans)
        return ans

if __name__ == "__main__":
    print(Solution().maxBuilding(n = 10, restrictions = [[5,3],[2,5],[7,4],[10,3]]))


#!/usr/bin/env python
"""Setup script for building prosail's python bindings"""
import os
import codecs
import re
from os import path
from setuptools import setup

# Global variables for this extension:
name = "prosail"  # name of the generated python extension (.so)
description = "PROSPECT, SAIL and PROSAIL Python wrappers"
long_description = "The PROSPECT + SAILh radiative transfer models from Python"

this_directory = path.abspath(path.dirname(__file__))

def read(filename):
    with open(os.path.join(this_directory, filename), "rb") as f:
        return f.read().decode("utf-8")

if os.path.exists("README.md"):
    long_description = read("README.md")

def read(*parts):
    with codecs.open(os.path.join(this_directory, *parts), 'r') as fp:
        return fp.read()

def find_version(*file_paths):
    version_file = read(*file_paths)
    version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]",
                              version_file, re.M)
    if version_match:
        return version_match.group(1)
    raise RuntimeError("Unable to find version string.")

author = "J Gomez-Dans/NCEO & University College London"
author_email = "j.gomez-dans@ucl.ac.uk"
url = "http://github.com/jgomezdans/prosail"
classifiers = [
    'Development Status :: 5 - Production/Stable',
    'Natural Language :: English',
    'Operating System :: OS Independent',
    'Programming Language :: Python :: 2',
    'Topic :: Scientific/Engineering',
    'Topic :: Software Development :: Libraries :: Python Modules',
    "Topic :: Scientific/Engineering :: Atmospheric Science",
    "Topic :: Scientific/Engineering :: Physics",
    "Topic :: Scientific/Engineering :: GIS",
    'Intended Audience :: Science/Research',
    'Intended Audience :: End Users/Desktop',
    'Intended Audience :: Developers',
    'Environment :: Console'
]

setup(
    name=name,
    description=description,
    long_description=long_description,
    long_description_content_type='text/markdown',
    author=author,
    url=url,
    author_email=author_email,
    classifiers=classifiers,
    package_data={"prosail": ["*.txt"]},
    install_requires=[
        "numpy",
        "numba",
        "scipy",
        "pytest",
    ],
    version=find_version("prosail", "__init__.py"),
    packages=["prosail"],
    zip_safe=False # Apparently needed for conda
)


from plotResults import plotResults
import subprocess
import yaml
import os

params = yaml.load(file("../params/salsa.yaml"))

# bagfile = "/home/superjax/rosbag/mynt_mocap/mocap2.bag"
# bagfile = "/home/superjax/rosbag/mynt_mocap/mocap2_adjust.bag"
directory = "../results/GNSSHardware"
prefix = "G/"

if not os.path.exists(os.path.join(directory, prefix)):
    os.makedirs(os.path.join(directory, prefix))
params["label"] = "G"

# params['bag_name'] = "/home/superjax/rosbag/gps_carry2_uc/small1.bag"
# params['start_time'] = 80 # small1
# params['duration'] = 80  # small 1
# params['bag_name'] = "/home/superjax/rosbag/gps_carry2_uc/small2.bag"
# params['start_time'] = 10 # small2
# params['duration'] = 300 cuda klt vs cpu klt C++# small2
# params['bag_name'] = "/home/superjax/rosbag/gps_carry2_uc/small3.bag"
# params['start_time'] = 10 # small 3
# params['duration'] = 80  # small 3
# params['bag_name'] = "/home/superjax/rosbag/gps_flight_eve/uc/flight1.bag"
# params['start_time'] = 36
params['start_time'] = 60 # small 3
params['duration'] =  220 # small 3
params['bag_name'] = "/home/superjax/rosbag/gps_flight_eve/uc/flight5.bag"
params['log_prefix'] = os.path.join(directory, prefix)
params['update_on_mocap'] = False
params['disable_mocap'] = True
params['disable_vision'] = True
params['update_on_vision'] = True
params['disable_gnss'] = False
params['update_on_gnss'] = True
params['enable_switching_factors'] = False
params['static_start_imu_thresh'] = 10.5
param_filename = os.path.join(directory, "tmp.yaml")
yaml.dump(params, file(param_filename, 'w'))

process = subprocess.call(("cmake", "..", "-DCMAKE_BUILD_TYPE=RelWithDebInfo", "-GNinja", "-DBUILD_ROS=ON"), cwd="../build")
process = subprocess.call(("ninja", "salsa_rosbag"), cwd="../build")
process = subprocess.call(("./salsa_rosbag", "-f", param_filename), cwd="../build")

plotResults(directory, False)


class Residual_Block(nn.Module):
    def __init__(self, in_channels, out_channels, stride = 1, downsample = None):
        super(Residual_Block, self).__init__()

        self.conv1 = nn.Sequential(
                        nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1),
                        nn.BatchNorm2d(out_channels),
                        nn.ReLU())

        self.conv2 = nn.Sequential(
                        nn.Conv2d(out_channels, out_channels, kernel_size=3, stride=1, padding=1),
                        nn.BatchNorm2d(out_channels))

        self.downsample = downsample
        self.relu = nn.ReLU()

    def forward(self, x):
        residual = x
        out = self.conv1(x)
        out = self.conv2(out)
        if self.downsample:
            residual = self.downsample(x)
        out += residual
        out = self.relu(out)
        return out

class Resnet_Model_Torch(nn.Module):
    def __init__(self, block, num_layers_arr, num_classes):
        super(Resnet_Model_Torch, self).__init__()

        #self.expected_input_shape = (1, 3, 224, 224)
        self.residual_channels=64
        self.conv1 = nn.Sequential(
                        nn.Conv2d(in_channels=3, out_channels=64, kernel_size=7, stride=2, padding=3),
                        nn.BatchNorm2d(64),
                        nn.ReLU())

        self.maxpool = nn.MaxPool2d(kernel_size=2)

        #self.test = self.downsample_block(in_channels, out_channels, stride)
        self.layer1 = self.make_layer(block, 64, 64, num_layers_arr[0], stride=1)
        self.layer2 = self.make_layer(block, 64, 128, num_layers_arr[1], stride=2)
        self.layer3 = self.make_layer(block, 128, 256, num_layers_arr[2], stride=2)
        self.layer4 = self.make_layer(block, 256, 512, num_layers_arr[3], stride=2)

        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
        self.Flatten = nn.Flatten()
        self.fc = nn.Linear(512, num_classes)

    def downsample_block(self, in_channels, out_channels, stride):

        return nn.Sequential(
            nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=stride),                    # 1x1 convolution with stride=2 to halve size
            nn.BatchNorm2d(out_channels)
        )

    def make_layer(self, block, in_channels, out_channels, num_layers, stride):
        downsample = None
        if stride != 1:
            downsample = self.downsample_block(in_channels, out_channels, stride)

        layers = []
        layers.append(block(self.residual_channels, out_channels, stride, downsample))
        self.residual_channels=out_channels
        for i in range(1, num_layers):
            layers.append(block(self.residual_channels, out_channels))

        return nn.Sequential(*layers)

    def forward(self, x):
        out = self.conv1(x)
        out = self.maxpool(out)

        out = self.layer1(out)
        out = self.layer2(out)
        out = self.layer3(out)
        out = self.layer4(out)

        out = self.avgpool(out)
        out = self.Flatten(out)
        out = self.fc(out)

        return out

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = Resnet_Model_Torch(Residual_Block, [2, 2, 2, 2], 75).to(device)                  ### ResNet18 Architecture
model = model.to(device)
loss_fn = nn.CrossEntropyLoss()
optim = torch.optim.Adam(model.parameters(), lr=0.001)

### PRINT MODEL LAYERS

print(model)

from torchsummary import summary
#summary(model, (3, 224, 224))
summary(model, input_image_tensor_shape)


import torch
import numpy as np
from ARC import ARC
from torch.utils.data import DataLoader
from torch.optim import lr_scheduler
import argparse
import os
import random
from meta import Meta
import pickle
from ARCDataset import ARCTrain
import matplotlib.pyplot as plt

seed = 42
print(f'setting everything to seed {seed}')
random.seed(seed)
os.environ['PYTHONHASHSEED'] = str(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
torch.backends.cudnn.deterministic = True

def main():

    torch.manual_seed(222)
    torch.cuda.manual_seed_all(222)
    np.random.seed(222)

    print(args)

    config = [
        ('bn', [1]),
        ('conv2d', [64, 1, 3, 3, 1, 1]),
        ('relu', [True]),
        ('bn', [64]),
        ('conv2d', [32, 64, 3, 3, 1, 1]),
        ('relu', [True]),
        ('bn', [32]),
        ('conv2d', [16, 32, 3, 3, 1, 1]),
        ('relu', [True]),
        ('bn', [16]),
        ('conv2d', [16, 16, 3, 3, 1, 1]),
        ('relu', [True]),
        ('max_pool2d', [args.imgsz, args.imgsz, 0]),
        ('flatten', []),
        ('linear', [args.imgsz*args.imgsz*11, 16])
    ]

    device = torch.device('cuda')
    maml = Meta(args, config).to(device)

    tmp = filter(lambda x: x.requires_grad, maml.parameters())
    num = sum(map(lambda x: np.prod(x.shape), tmp))
    print(maml)
    print('Total trainable tensors:', num)

    # batchsz here means total episode number
    arc = ARCTrain(root='/home/sid/Desktop/hertzwell/github/arc/data/',
               mode='train', n_way=1,
               k_shot=args.k_spt, k_query=1, batchsz=400, imgsz=args.imgsz)

    for epoch in range(args.epoch):
        # # fetch meta_batchsz num of episode each time
        # db = DataLoader(arc, args.task_num, shuffle=True,
        #                 num_workers=0, pin_memory=True)

        # for step, (x_spt, y_spt, x_qry, y_qry) in enumerate(arc):
        all_accs, all_loss = [], []
        for step in range(len(arc)):
        # for step in range(2):

            x_spt, y_spt, x_qry, y_qry = arc[step]
            x_spt, y_spt, x_qry, y_qry = x_spt.to(device), y_spt.to(
                device), x_qry.to(device), y_qry.to(device)

            accs, loss = maml(x_spt, y_spt, x_qry, y_qry)
            all_accs.append(accs)
            all_loss.append(loss)

            # if step % len(db)//5 == 0:
            print('epoch:', epoch, 'step:', step, '\ttraining acc:', accs)
        print()

        plt.subplot(1, 2, 1)
        plt.plot(np.mean(all_accs, 1), label=f'acc_epoch_{epoch}')
        plt.legend(loc="upper right")

        plt.subplot(1, 2, 2)
        plt.plot(all_loss, label=f'loss_epoch_{epoch}')
        plt.legend(loc="upper right")
        plt.show(block=False)
        plt.pause(0.1)
        print('epoch:', epoch, 'acc:', np.mean(all_accs, 0), 'loss:', np.mean(loss))
        print()
        if (epoch % 1 == 0) & (epoch > 0):  # evaluation
            # if (epoch % 10 == 0):  # evaluation
            with open(f'./model_weights/maml_sz30_epoch_{epoch}_accs_{np.mean(all_accs):.3}_loss_{np.mean(loss):.3}.pkl', 'wb') as f:
                pickle.dump(maml, f)

if __name__ == '__main__':

    argparser = argparse.ArgumentParser()
    argparser.add_argument('--epoch', type=int,
                           help='epoch number', default=91)
    argparser.add_argument('--n_way', type=int, help='n way', default=1)
    argparser.add_argument('--k_spt', type=int,
                           help='k shot for support set', default=2)
    argparser.add_argument('--k_qry', type=int,
                           help='k shot for query set', default=1)
    argparser.add_argument('--imgsz', type=int, help='imgsz', default=15)
    argparser.add_argument('--imgc', type=int, help='imgc', default=1)
    argparser.add_argument('--task_num', type=int,
                           help='meta batch size, namely task num', default=4)
    argparser.add_argument('--meta_lr', type=float,
                           help='meta-level outer learning rate', default=1e-3)
    argparser.add_argument('--update_lr', type=float,
                           help='task-level inner update learning rate', default=1)
    argparser.add_argument('--update_step', type=int,
                           help='task-level inner update steps', default=2)
    argparser.add_argument('--update_step_test', type=int,
                           help='update steps for finetunning', default=20)

    args = argparser.parse_args()

    main()


#Item 12
#Be defensive when iterating over arguments

data = [15,80,35]

def normalize(numbers):
    total = sum(numbers) # iternation 1
    result = []

    for value in numbers: # iternation 2
        perc = 100*(value/total)
        result.append(perc)
    return result

output = normalize(data)
print(output)
print(sum(output))

#scale the data : read the input from a file.
#leverage generator

path = './2-Comprehensions-Generators/item12.txt'
with open(path, 'w') as f:
    for i in data:
        f.write('%d\n' % i)

def read_visits_Gen(data_path):
    with open(data_path) as f:
        for line in f:
            yield int(line)

result = read_visits_Gen(path)
print(list(result))
print(list(result)) #list will not iterate again

get_iter = lambda: read_visits_Gen(path)
print(get_iter())
print(sum(get_iter()))

#pythonic way
print('pythonic way\n-----------')
a = [1,2,3,4,5]
for x in a:
    print(x)

it = iter(a)
it = a.__iter__() #equivalent to above
value = next(it)
value = it.__next___()


from pyspark.sql.session import SparkSession
from pyspark.streaming import StreamingContext
from pyspark.sql.session import SparkSession
from pyspark import SparkConf, SparkContext
import json
import numpy as np
import pandas as pd
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
from sklearn.metrics import classification_report
from sklearn.metrics import roc_curve
from pyspark.ml.classification import RandomForestClassifier
from pyspark.ml.feature import VectorAssembler
from pyspark.ml.feature import StringIndexer

sc = SparkContext()#conf=SparkConf().setAppName("MyApp").setMaster("local"))
spark = SparkSession(sc)
ssc = StreamingContext(sc, batchDuration= 10)

def f(x):
    global counter
    global features
    global all_records
    data = json.loads(x)
    for key,values in data.items():
        temp = list(values.values())
        all_records.append(temp)
        counter+=1
        if len(features) == 0:
            features = list(values.keys())
    all_records = np.asarray(all_records)
    '''
    df = pd.DataFrame(all_records, columns=features)
    mydf = spark.createDataFrame(df)
    print(mydf.show())
    '''

def process_stream(record,spark):
    if not record.isEmpty():
        #record.foreach(f)
        #print(str(counter)+" ==== "+str(all_records))
        x = record.take(100)
        x = x[0]
        all_records = []
        features = []
        data = json.loads(x)
        for key,values in data.items():
            temp = list(values.values())
            all_records.append(temp)
            if len(features) == 0:
                features = list(values.keys())
        all_records = np.asarray(all_records)
        df = pd.DataFrame(all_records, columns=features)
        mydf = spark.createDataFrame(df)
        assembler = VectorAssembler(inputCols=features, outputCol='features',handleInvalid="skip")
        transformed_data = assembler.transform(mydf)

        indexer = StringIndexer(inputCol="label",outputCol="indexlabel",handleInvalid="skip")
        transformed_data = indexer.fit(transformed_data).transform(transformed_data)

        (training_data, test_data) = transformed_data.randomSplit([0.8, 0.2])
        #creating random forest ML object and training on train data
        decision_tree = RandomForestClassifier(featuresCol = 'features',labelCol = 'indexlabel', numTrees=500)
        decision_tree_model = decision_tree.fit(training_data)
        #predicting on test data
        predictions = decision_tree_model.transform(test_data)
        #calculating accuracy
        evaluator = MulticlassClassificationEvaluator(labelCol='indexlabel',metricName="accuracy")
        accuracy = evaluator.evaluate(predictions)
        print("\nRandom Forest Classification Test Error = %g" % (1.0 - accuracy))
        print("\nRandom Forest Accuracy = %g" % (accuracy))

def getSparkSessionInstance(sparkConf):
    if ('sparkSessionSingletonInstance' not in globals()):
        globals()['sparkSessionSingletonInstance'] = SparkSession\
            .builder\
            .config(conf=sparkConf)\
            .getOrCreate()
    return globals()['sparkSessionSingletonInstance']

def process(time, rdd):
    print("========= %s =========" % str(time))
    spark = getSparkSessionInstance(rdd.context.getConf())
    rowRdd = rdd.map(lambda w: Row(word=w))

# Create a DStream that will connect to hostname:port, like localhost:9991
lines = ssc.socketTextStream("localhost", 6100)
lines.foreachRDD(lambda rdd: process_stream(rdd,spark))
#print(str(all_records.shape)+" ===== "+str(len(features)))
ssc.start()
ssc.awaitTermination()


from collections import *

"""
    1st: sliding window + hashtable
    - counter the characters outside the sliding window

    Time    O(N)
    Space   O(N)
    875 ms, faster than 100.00%
"""

class Solution:
    def shareCandies(self, candies: List[int], k: int) -> int:
        counter = Counter(candies)
        res = 0
        for i in range(len(candies)):
            c = candies[i]

            counter[c] -= 1
            if counter[c] == 0:
                del counter[c]

            if i >= k:
                p = candies[i-k]
                if p not in counter:
                    counter[p] = 0
                counter[p] += 1

            if i >= k-1:
                res = max(res, len(counter))
        return res


import sys
input = sys.stdin.readline
n = int(input())
stack = []
for i in range(n):
    a = input().strip().split()
    if len(a) == 2:
        stack.append(int(a[1]))
    else:
        if a[0] == 'pop':
            if stack:
                s = stack.pop()
                print(s)
            else:
                print(-1)
        if a[0] == 'size':
            print(len(stack))
        if a[0] == 'empty':
            if stack:
                print(0)
            else:
                print(1)
        if a[0] == 'top':
            if stack:
                print(stack[-1])
            else:
                print(-1)


import geocoder
import threading
import requests
from TweetObj import Tweet

#The point of this class is to make geocoder calls for tweet locations in a separate thread from the main in order to make location.py more efficient
class TweetThread(threading.Thread):
   def __init__(self, tweetList):
      threading.Thread.__init__(self)
      self.tweetList = tweetList
      self.done = False
      self.running = False

#Threaded method
   def run(self):
      self.running = True
      i = 1
      for tweet in self.tweetList: #Goes through list of tweets and finds locations
         tweet.coordinates = str(geocoder.arcgis(tweet.location).latlng)
         print("\n\n\n\n*********" + str(i) + "\\" + str(len(self.tweetList)) + "//////////" + tweet.file_name + ": " + tweet.coordinates + "*********\n\n\n\n\n")
         for ref_key in tweet.ref_locations: #Goes through dictionary of Multipoints and finds loc
            if tweet.ref_locations[ref_key] == "":
               tweet.ref_locations[ref_key] = str(geocoder.arcgis(ref_key).latlng)
               print("\n\nTweetlist: " + ref_key + " -- " + tweet.ref_locations[ref_key] + " " + str(len(tweet.ref_locations)) + "\n\n")
         i += 1
      self.done = True


def find_set(num):
    if num != make_sets[num]:
        make_sets[num] = find_set(make_sets[num])
    return make_sets[num]

def kruskal(n, edge):

    mst = 0
    values = 0
    while mst != n-1:
        w, v, u = edge.pop()
        if find_set(v) != find_set(u):
            make_sets[find_set(u)] = find_set(v)
            mst += 1
            values += w
    return values

for t in range(int(input())):
    n = int(input())
    x_list = list(map(int, input().split()))
    y_list = list(map(int, input().split()))
    e = float(input())
    # adj = [[0] * n for _ in range(n)]
    # adjList = [[] * n for _ in range(n)]
    adjList = []
    # 각 간선 별 가중치 계산
    # 인접 행렬 버전
    # for i in range(n):
    #     for j in range(n):
    #         if adj[i][j] == 0:
    #             w = abs(x_list[i]-x_list[j])**2 + abs(y_list[i]-y_list[j])**2
    #             adj[i][j] = w
    #             adj[j][i] = w
    # 인접 리스트 버전
    for i in range(n):
        for j in range(i+1, n):
            w = abs(x_list[i]-x_list[j])**2 + abs(y_list[i]-y_list[j])**2
            # adjList[i].append((j, w))
            # adjList[j].append((i, w))
            adjList.append((w, i, j))
    adjList.sort(reverse=True)
    # 상호 배타 집합
    make_sets = [i for i in range(n)]
    ans = kruskal(n, adjList)
    print('#{} {}'.format(t+1, round(ans*e)))

ssafy_python/210423/1251. [SW 문제해결 응용] 4일차 - 하나로.py


""" This is custom bot class """

import json

import aiohttp
from discord.ext import commands
import motor.motor_asyncio

#check discord.py rewrite documentation for why AutoShardedBot was used.
class Bot(commands.AutoShardedBot):

    """
    This is a custom object which extends default commands.Bot class and provides
    a configuration handler and a common aiohttp ClientSession.
    """

    def __init__(self, *args, **kwargs):
        """`config_file`: A string representing name of .json file which contains
        configurations for bot. Can be anything but will default to `config.json`

        Instance variables:

        `session` - An `aiohttp.ClientSession` used for performing API hooks.
        `config` - A `dictionary` containing key value pairs represnting bot configs.
        """

        #pass arguments to constructor of parent class
        super().__init__(*args, **kwargs)
        self.config = {}

        #in case I ever forget why kwargs.get method is used(god help me), just google it
        self.config_file = kwargs.get("config_file","config.json")

        #also apparently you can do Bot.loop, did not knew this.
        self.session = aiohttp.ClientSession(loop = self.loop)

        #create mongoDB client. MongoDB database server must be running in background
        self.DBClient = motor.motor_asyncio.AsyncIOMotorClient()

        #create Bot databsase
        self.db = self.DBClient.RoboX86

        #get settings of all of which bot is part. it's basically a collection
        #self.guildSettings = self.db.guilds

        #get collection containing details of all users subscribed to one piece updates
        self.onePieceCollection = self.db.onePieceCollection

        #get collection containing details of all users subscribed to unOrdinary updates
        self.unOrdinaryCollection = self.db.unOrdinaryCollection

    def load_config(self, filename: str=None):
        """
        Load congig from a .JSON file. If not specified will default to
        `Bot.config_file`.
        """

        if not filename:
            filename = self.config_file

        #pro tip: google difference between json `load,dump` and `loads,dumps`.
        with open(filename) as file_object:
            config = json.load(file_object)

        #also google `isinstance` vs `type`. Hint: `isinstance` is better.
        if isinstance(config,dict):
            for key,value in config.items():
                self.config[key] = value

    def save_config(self, filename:str=None):
        """save config to a .JSON file. Defaults to `Bot.config_file`."""

        if not filename:
            filename = self.config_file

        with open(filename,'w') as file_object:
            json.dump(self.config_file, file_object, indent=4, sort_keys=True)

    #add database connection..


from io import StringIO
from textwrap import dedent

import astropy.units as u
import astropy.utils.iers
import numpy as np
from astropy.time import Time

def get_leap_sec():
    ls = astropy.utils.iers.LeapSeconds.auto_open()
    # Annoyingly it's year/month/amount
    # Annoyingly tempo's leap.sec starts later than integer leap seconds
    leap_sec = []
    leap_sec_values = []
    for r in ls:
        t = Time(f"{r['year']}-{r['month']}-01", format="iso", scale="utc")
        leap_sec.append(int(np.round(t.mjd)))
        leap_sec_values.append(r["tai_utc"])

    return np.array(leap_sec), np.array(leap_sec_values)

def make_leap_sec():
    """Construct leap.sec file."""
    leap_sec, leap_sec_values = get_leap_sec()

    tstart = np.inf
    tend = -np.inf

    r = StringIO()
    for i, m in enumerate(leap_sec):
        if m < 41499:
            # not in leap.sec
            continue
        tstart = min(tstart, m)
        tend = max(tend, m)
        print(m, file=r)
        if len(leap_sec_values) > 1 and leap_sec_values[i] <= leap_sec_values[i - 1]:
            # This is a problem for TEMPO
            raise ValueError(f"Leap seconds not all positive: {ls}")
    return r.getvalue(), Time(tstart, format="mjd"), Time(tend, format="mjd")

def make_ut1_dat():
    r = StringIO()
    leap_sec, leap_sec_values = get_leap_sec()
    iers_a = astropy.utils.iers.IERS_Auto.open()
    tai_utc = (
        np.array(leap_sec_values)[
            np.searchsorted(leap_sec, iers_a["MJD"], side="right") - 1
        ]
        * u.s
    )
    print(
        dedent(
            """\
    TAI-UT1: IERS A/B via Astropy including forecast for about a year
    (A3,2X,I5,6(I7,1X),1X,I2)        2                  6  5           1.E-4"""
        ),
        file=r,
    )
    ut1_utc = iers_a["UT1_UTC"]
    tai_ut1 = tai_utc - ut1_utc
    i = 0
    while i < len(iers_a):
        mjd = int(iers_a[i]["MJD"].to_value(u.d))
        ut1mutcs = [tai_ut1[i + 5 * j] for j in range(6) if i + 5 * j < len(iers_a)]
        e = f"     {mjd:5d}"
        e += "".join([f"{int(np.round(uv.to_value(u.us)/100)):7d} " for uv in ut1mutcs])
        if len(ut1mutcs) < 6:
            e += (8 * " ") * (6 - len(ut1mutcs)) + f" {len(ut1mutcs):2d}"
        print(e, file=r)
        i += 30
    print("END", file=r)

    return (
        r.getvalue(),
        Time(iers_a[0]["MJD"], format="mjd"),
        Time(iers_a[-1]["MJD"], format="mjd"),
    )


import collections
import logging

import numpy as np
import pandas as pd

from scipy.sparse import vstack

from sklearn.feature_selection import SelectKBest
from sklearn.preprocessing import Binarizer

from load_csr_matrix import load_csr_matrix
from transformers import FrequencyTransformer

FILE_ITEM_PREDICATES = '../../data/item-properties/item-properties.bz2'

PATH_FEATURES = '../../data/features/'
PATH_TRAIN = PATH_FEATURES + 'training/'    # noqa
PATH_VAL   = PATH_FEATURES + 'validation/'  # noqa
PATH_TEST  = PATH_FEATURES + 'test/'        # noqa

def load_matrices():
    matrices = collections.OrderedDict()

    path = PATH_TRAIN + 'embeddings/'
    matrices['X_S_train'] = path + 'subjectOut'
    matrices['X_P_train'] = path + 'predicate'
    matrices['X_O_train'] = path + 'objectIn'
    matrices['X_OO_train'] = path + 'objectOut'

    path = PATH_VAL + 'embeddings/'
    matrices['X_S_val'] = path + 'subjectOut'
    matrices['X_P_val'] = path + 'predicate'
    matrices['X_O_val'] = path + 'objectIn'
    matrices['X_OO_val'] = path + 'objectOut'

    path = PATH_TEST + 'embeddings/'
    matrices['X_S_test'] = path + 'subjectOut'
    matrices['X_P_test'] = path + 'predicate'
    matrices['X_O_test'] = path + 'objectIn'
    matrices['X_OO_test'] = path + 'objectOut'

    for key, X in matrices.items():
        logging.debug('load {}...'.format(key))
        matrices[key] = load_csr_matrix(X)

    meta = collections.OrderedDict()
    meta['n_train'] = matrices['X_O_train'].shape[0]
    meta['n_val'] = matrices['X_O_val'].shape[0]
    meta['n_test'] = matrices['X_O_test'].shape[0]

    data = collections.OrderedDict()

    data['X_S_all'] = vstack([
        matrices['X_S_train'],
        matrices['X_S_val'],
        matrices['X_S_test']
    ])

    data['X_P_all'] = vstack([
        matrices['X_P_train'],
        matrices['X_P_val'],
        matrices['X_P_test']
    ])

    data['X_O_all'] = vstack([
        matrices['X_O_train'],
        matrices['X_O_val'],
        matrices['X_O_test']
    ])

    data['X_OO_all'] = vstack([
        matrices['X_OO_train'],
        matrices['X_OO_val'],
        matrices['X_OO_test']
    ])

    meta['X_S_all'] = np.array(
        ['S' + str(p) for p in range(data['X_S_all'].shape[1])])
    meta['X_P_all'] = np.array(
        ['P' + str(p) for p in range(data['X_P_all'].shape[1])])
    meta['X_O_all'] = np.array(
        ['O' + str(p) for p in range(data['X_O_all'].shape[1])])
    meta['X_OO_all'] = np.array(
        ['OO' + str(p) for p in range(data['X_OO_all'].shape[1])])

    return data, meta

def binarize_features(data):
    encoder = Binarizer(threshold=0.5, copy=False)

    for key, X in data.items():
        data[key] = encoder.fit_transform(X)

def select_item_predicates_at_end_of_training_set(data, meta):
    item_predicates = pd.read_csv(FILE_ITEM_PREDICATES, header=None)
    item_predicates = item_predicates.values.flatten()

    def _remove_attribute_predicates_from_X(X):
        # mask = np.zeros((1, data['X_object_pred_all'].shape[1]))
        # mask[0, item_predicates] = 1
        # return X.multiply(mask).tocsr()
        return X.tocsc()[:, item_predicates].tocsr()

    for key, X in data.items():
        logging.debug(key)
        data[key] = _remove_attribute_predicates_from_X(X)
        meta[key] = meta[key][item_predicates]

def count_nonzero(X, _):
    return np.asarray((X != 0).sum(axis=0)).ravel()

def select_features(
        data, meta, y, slice_fit, score_func=count_nonzero, k=100):
    if y is None:
        rand_X = next(iter(data.values()))
        y = np.zeros(rand_X[slice_fit].shape[0])

    logging.debug(slice_fit)

    for key in data:
        logging.debug(data[key].shape)
        selector = SelectKBest(score_func=score_func, k=k)
        selector = selector.fit(data[key][slice_fit], y[slice_fit])
        data[key] = selector.transform(data[key])
        meta[key] = meta[key][selector.get_support()]

def frequency_encoding(data, slice_fit):
    # slice_fit = slice(0, n_train + n_val)

    # convert to DataFrame
    df_freq = pd.DataFrame()
    df_freq['subjectPredEmbedFrequency'] = rows_to_str(data['X_S_all'])
    df_freq['objectPredEmbedFrequency'] = rows_to_str(data['X_O_all'])
    df_freq['objectOutPredEmbedFrequency'] = rows_to_str(data['X_OO_all'])

    transformer = FrequencyTransformer()
    transformer = transformer.fit(
        df_freq[['subjectPredEmbedFrequency']][slice_fit])
    df_freq[['subjectPredEmbedFrequency']] = transformer.transform(
        df_freq[['subjectPredEmbedFrequency']])

    transformer = FrequencyTransformer()
    transformer = transformer.fit(
        df_freq[['objectPredEmbedFrequency']][slice_fit])
    df_freq[['objectPredEmbedFrequency']] = transformer.transform(
        df_freq[['objectPredEmbedFrequency']])

    transformer = FrequencyTransformer()
    transformer = transformer.fit(
        df_freq[['objectOutPredEmbedFrequency']][slice_fit])
    df_freq[['objectOutPredEmbedFrequency']] = transformer.transform(
        df_freq[['objectOutPredEmbedFrequency']])

    return df_freq

def rows_to_str(array):
    rows = array.tolil().rows
    for i in range(len(rows)):
        rows[i] = ','.join(str(elem) for elem in rows[i])
    return rows

# ---------------------------------------------------------
# Internal Functions
# ---------------------------------------------------------

def _get_slice_fit(meta, use_test_set):
    if use_test_set:
        return slice(0, meta['n_train'] + meta['n_val'])
    else:
        return slice(0, meta['n_train'])


from selenium import webdriver
from bs4 import BeautifulSoup
import time
import dload
from openpyxl import Workbook

driver = webdriver.Chrome('D:/temp/chromedriver.exe')
driver.get("https://search.naver.com/search.naver?where=news&sm=tab_jum&query=%EC%B6%94%EC%84%9D")
time.sleep(3)

req = driver.page_source

soup = BeautifulSoup(req, 'html.parser')

articles = soup.select("#main_pack > div.news.mynews.section._prs_nws > ul > li")

wb = Workbook()
ws1 = wb.active
ws1.title = "articles"

for article in articles:
    title = article.select_one('dl > dt > a').text
    url = article.select_one('dl > dt > a')['href']
    comp = article.select_one('span._sp_each_source').text.split(' ')[0].replace('언론사','')
    if article.select_one('div > a > img') is not None:
        img = article.select_one('div > a > img')['src']
    else: img = ''
    ws1.append([title, url, comp, img])

driver.quit()

wb.save(filename='articles.xlsx')


""" discover and run doctests in modules and test files."""

import pytest, py
from _pytest.python import FixtureRequest, FuncFixtureInfo
from py._code.code import TerminalRepr, ReprFileLocation

def pytest_addoption(parser):
    group = parser.getgroup("collect")
    group.addoption("--doctest-modules",
        action="store_true", default=False,
        help="run doctests in all .py modules",
        dest="doctestmodules")
    group.addoption("--doctest-glob",
        action="store", default="test*.txt", metavar="pat",
        help="doctests file matching pattern, default: test*.txt",
        dest="doctestglob")

def pytest_collect_file(path, parent):
    config = parent.config
    if path.ext == ".py":
        if config.option.doctestmodules:
            return DoctestModule(path, parent)
    elif (path.ext in ('.txt', '.rst') and parent.session.isinitpath(path)) or \
        path.check(fnmatch=config.getvalue("doctestglob")):
        return DoctestTextfile(path, parent)

class ReprFailDoctest(TerminalRepr):
    def __init__(self, reprlocation, lines):
        self.reprlocation = reprlocation
        self.lines = lines
    def toterminal(self, tw):
        for line in self.lines:
            tw.line(line)
        self.reprlocation.toterminal(tw)

class DoctestItem(pytest.Item):
    def __init__(self, name, parent, runner=None, dtest=None):
        super(DoctestItem, self).__init__(name, parent)
        self.runner = runner
        self.dtest = dtest

    def runtest(self):
        self.runner.run(self.dtest)

    def repr_failure(self, excinfo):
        doctest = py.std.doctest
        if excinfo.errisinstance((doctest.DocTestFailure,
                                  doctest.UnexpectedException)):
            doctestfailure = excinfo.value
            example = doctestfailure.example
            test = doctestfailure.test
            filename = test.filename
            if test.lineno is None:
                lineno = None
            else:
                lineno = test.lineno + example.lineno + 1
            message = excinfo.type.__name__
            reprlocation = ReprFileLocation(filename, lineno, message)
            checker = py.std.doctest.OutputChecker()
            REPORT_UDIFF = py.std.doctest.REPORT_UDIFF
            filelines = py.path.local(filename).readlines(cr=0)
            lines = []
            if lineno is not None:
                i = max(test.lineno, max(0, lineno - 10)) # XXX?
                for line in filelines[i:lineno]:
                    lines.append("%03d %s" % (i+1, line))
                    i += 1
            else:
                lines.append('EXAMPLE LOCATION UNKNOWN, not showing all tests of that example')
                indent = '>>>'
                for line in example.source.splitlines():
                    lines.append('??? %s %s' % (indent, line))
                    indent = '...'
            if excinfo.errisinstance(doctest.DocTestFailure):
                lines += checker.output_difference(example,
                        doctestfailure.got, REPORT_UDIFF).split("\n")
            else:
                inner_excinfo = py.code.ExceptionInfo(excinfo.value.exc_info)
                lines += ["UNEXPECTED EXCEPTION: %s" %
                            repr(inner_excinfo.value)]
                lines += py.std.traceback.format_exception(*excinfo.value.exc_info)
            return ReprFailDoctest(reprlocation, lines)
        else:
            return super(DoctestItem, self).repr_failure(excinfo)

    def reportinfo(self):
        return self.fspath, None, "[doctest] %s" % self.name

class DoctestTextfile(DoctestItem, pytest.File):
    def runtest(self):
        doctest = py.std.doctest
        # satisfy `FixtureRequest` constructor...
        self.funcargs = {}
        fm = self.session._fixturemanager
        def func():
            pass
        self._fixtureinfo = fm.getfixtureinfo(node=self, func=func,
                                              cls=None, funcargs=False)
        fixture_request = FixtureRequest(self)
        fixture_request._fillfixtures()
        failed, tot = doctest.testfile(
            str(self.fspath), module_relative=False,
            optionflags=doctest.ELLIPSIS,
            extraglobs=dict(getfixture=fixture_request.getfuncargvalue),
            raise_on_error=True, verbose=0)

class DoctestModule(pytest.File):
    def collect(self):
        doctest = py.std.doctest
        if self.fspath.basename == "conftest.py":
            module = self.config._conftest.importconftest(self.fspath)
        else:
            module = self.fspath.pyimport()
        # satisfy `FixtureRequest` constructor...
        self.funcargs = {}
        self._fixtureinfo = FuncFixtureInfo((), [], {})
        fixture_request = FixtureRequest(self)
        doctest_globals = dict(getfixture=fixture_request.getfuncargvalue)
        # uses internal doctest module parsing mechanism
        finder = doctest.DocTestFinder()
        runner = doctest.DebugRunner(verbose=0, optionflags=doctest.ELLIPSIS)
        for test in finder.find(module, module.__name__,
                                extraglobs=doctest_globals):
            if test.examples: # skip empty doctests
                yield DoctestItem(test.name, self, runner, test)