#!/usr/bin/python
# coding=UTF-8
'''
@Author: recar
@Date: 2019-09-27 11:37:22
@LastEditTime: 2019-09-27 15:51:48
'''

from datetime import datetime
from threading import Thread
import redis
import time
import sys
reload(sys)
sys.setdefaultencoding('utf-8')

redis_server = "127.0.0.1"
redis_port = 6379

class Scheduler(object):
    # 基于redis实现一个定时调度功能

    def  __init__(self, schedule=False):
        self.is_schedule = schedule
        self.conn = self.get_connection()
        self.zset_delay_key = "zset_delay:"
        self.zset_schedule_key = "zset_schedule:"
        if self.is_schedule:
            self.schedule()

    def get_connection(self):
        connect = redis.Redis(host=redis_server, port=redis_port,db=0)
        return connect

    def add_task(self, task_id, delay):
        # 添加时间间隔
        self.conn.zadd(self.zset_delay_key, {task_id: delay})
        # 添加调度任务 这个会要先生成一个立即调度的
        self.conn.zadd(self.zset_schedule_key, {task_id: time.time()})

    def schedule(self):
        while True:
            next = self.conn.zrange(self.zset_schedule_key, 0, 0, withscores=True)
            now = time.time()
            # print(next)
            if not next or next[0][1] > now:
                # print('next: {0} -> now: {1}'.format(next[0][1], now))
                time.sleep(0.5)
                continue
            task_id = next[0][0]
            # 获取时间间隔
            delay = self.conn.zscore(self.zset_delay_key, task_id)
            # 如果更新了时间间隔 为0 那么都删除掉
            if delay <=0:
                self.conn.zrem(self.zset_delay_key, task_id)
                self.conn.zrem(self.zset_schedule_key, task_id)
                continue
            print('exec task: {0} time: {1}'.format(task_id, now))
            # 否则更新调度任务
            self.conn.zrem(self.zset_schedule_key, task_id)
            self.conn.zadd(self.zset_schedule_key, {task_id: now+delay})

def main():
    try:
        # 启动一个定时调度监控
        def statr_schedule():
            schedule = Scheduler(schedule=True)
        t = Thread(target=statr_schedule, args=())
        t.start()
        print('start schedule')
        schedule = Scheduler()
        schedule.add_task(1, 5)
        print('add task: {0}'.format(1))
        schedule.add_task(2, 10)
        print('add task: {}'.format(2))
    except Exception as e:
        print(e)
        exit(1)

if __name__ == "__main__":
    main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Jul 24 23:37:15 2018

@author: enriquelazcorretapuigmarti
"""

import os, psutil, sys, hashlib, time

def tiempo_transcurrido(inicio, mensaje='', fin=None):
    if fin is None:
        fin = time.time()
    segundos = fin - inicio

    dias = 0
    horas = 0
    minutos = 0
    ms = int((segundos % 1) * 1000)
    segundos = int(segundos)

    resultado = mensaje

    if segundos >= 86400:
        dias = segundos // 86400
        segundos -= dias * 86400

    if segundos >= 3600:
        horas = segundos // 3600
        segundos -= horas * 3600

    if segundos >= 60:
        minutos = segundos // 60
        segundos -= minutos * 60

    if dias:
        resultado += '{:,}'.format(dias) + ' días '

    if horas:
        resultado += str(horas) + 'h '

    if minutos:
        resultado += str(minutos) + 'm '

    if segundos:
        resultado += str(segundos) + 's '

    if len(resultado):
        if ms:
            resultado += str(ms) + 'ms'
    else:
        resultado += str(ms) + 'ms'

    return resultado

def tamanyo_legible(tamanyo_en_bytes, si=False):
    if si:
        if tamanyo_en_bytes < 1024:
            return '{:}'.format(tamanyo_en_bytes) + 'B'

        unidad = ('KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB')
        base = 10
        exponente = (3, 6, 9, 12, 15, 18, 21, 24, 27)
    else:
        if tamanyo_en_bytes < 1024:
            return '{:}'.format(tamanyo_en_bytes) + 'B'

        unidad = ('KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB')
        base = 2
        exponente = (10, 20, 30, 40, 50, 60, 70, 80)

    for i in range(7):
        if tamanyo_en_bytes < base ** exponente[i+1]:
            return '{:.1f}'.format(tamanyo_en_bytes / (base ** exponente[i])) \
                    + unidad[i]

    return '{:.1f}'.format(tamanyo_en_bytes / (base ** exponente[7])) + 'YB'

def memoria_objeto(objeto, mensaje=''):
    return tamanyo_legible(sys.getsizeof(objeto)) + mensaje

#TODO Con memoria_objeto() no debería ser necesaria memoria_dataset()
#TODO Aunque memoria_objeto() es mucho más lento (22sg frente a 18sg al usar
#     datasetclasificacion-inplace con todos los archivos), HACER MÁS PRUEBAS.
def memoria_dataset(pandas_dataframe, mensaje=''):
    ram = 0
    for columna in pandas_dataframe.memory_usage():
        ram += columna
    return tamanyo_legible(ram) + mensaje

def memoria_proceso(mensaje=' RAM usada por el proceso'):
    pid = os.getpid()
    py = psutil.Process(pid)
    return tamanyo_legible(py.memory_info()[0]) + mensaje

def sha1_archivo(archivo):
    try:
        with open(archivo, 'rb') as file:
            return hashlib.sha1(file.read()).hexdigest()
    except:
        return 'No he podido abrir el archivo ' + archivo

def md5_archivo(archivo):
    try:
        with open(archivo, 'rb') as file:
            return hashlib.md5(file.read()).hexdigest()
    except:
        return 'No he podido abrir el archivo ' + archivo

if __name__ == '__main__':
    print(tiempo_transcurrido(10000.1236))
    t0 = time.time()
    time.sleep(2)
    print(tiempo_transcurrido(inicio=t0))
    t1 = time.time()
    print(tiempo_transcurrido(inicio=t0, fin=t1))


#Chaitali Patel
#Lecture B2

def printsection1(animals, station1, station2):
    print("Number of times each animal visited each station : ")
    print("Animal Id             Station 1             Station 2")
    for a in animals:
        print(a," "*(20-len(a)),station1[a]," "*(20-len(str(station1[a]))),station2[a])
    print("="*60)

def printsection2(animals, station1, station2):
    print("Animals that visited both stations at least 4 times")
    for item in animals:
        if (station1[item]>=4 and station2[item]>=4):
            print(item)
    print("="*60)

def printsection3(animals,station1,station2):
    print("Total Number of visits for each animal")
    for a in animals:
        totalVisits=station1[a]+station2[a]
        print(a," "*20,totalVisits)
    print("="*60)

def printsection4(items):
    print("Print the month that has the highest number of visits to the stations")

    months=[]
    for i in items:
        dates=i[1].split("-")
        months.append(dates[0])

    mostVisits=0
    for most in months:
        visits=months.count(most)
        if visits>mostVisits:
            mostVisits=visits
            highestmonth=most
    print(highestmonth)

def main():
    #input
    file1=input("Enter name of input file >")
    try:
        infile=open(file1,"r")

    except IOError:
        print("File does not exist")
        return

    #required variables
    station1={}
    station2={}
    record=()
    items=[]
    animals=[]

    #remove blank and comment lines
    for line in infile:
        if len(line)==0 or line[0]=="#" or line[0]=="\n":
            continue
        (animal,time,station)=line.strip("\n").split(":")
        record=(animal,time,station)
        items.append(record)

        #Create list of animals
        if animal not in animals:
            animals.append(animal)
            animals=sorted(animals)

    #create dictionary for station 1 and station 2
    for x,y,z in items:
        if z=="s1":
            station1[x] = station1.get(x, 0) + 1
        else:
            station2[x]= station2.get(x,0)+1
    #if animal doesn't visit station, visits=0
    for a in animals:
        if a not in station1:
            station1[a]=0
        elif a not in station2:
            station2[a]=0

    #print all sections
    printsection1(animals,station1,station2)
    printsection2(animals,station1,station2)
    printsection3(animals,station1,station2)
    printsection4(items)

    #close file
    infile.close()

main()


import torch

from TCP_Manage import *
import numpy as np
class plant_manager():
    def __init__(self):
        self.Matdatapointer,self.Matadatamanager=Tcpcreator.create_proxy('127.0.0.1',8001)
        self.Matdatapointer1,self.Matadatamanager1=Tcpcreator.create_proxy('127.0.0.1',8002)
        self.matlist=np.mat("0.0,0.0,0.0,0.0,0.0")  # [x,y,phi,vel,omega]其中vel和omega不太必要
    def set_member(self):
        pass
    def add_state_control(self,state,control):
        # self.matlist.append(item)
        # mat = numpy.array(mylist)
        #考虑到这两个都是tensor，建议复制然后转化然后赋值
        state_copy=np.mat("0.0,0.0,0.0,0.0,0.0")
        control_copy=control.detach().numpy()

        state_copy[0,0]=state[0,0]
        state_copy[0,1]=state[1,0]
        state_copy[0,2]=state[2,0]
        state_copy[0,3]=control_copy[0,0]
        state_copy[0,4]=control_copy[0,1]

        self.matlist=np.vstack((self.matlist,state_copy))
        # np.mat(self.matlist)
        # self.stateMat_input= np.vstack((self.stateMat_input, mid_mat.transpose()))
        # pass
    def step_in_plant(self):
        self.stateMat_input=self.matlist[1:,:]  # [x,y,phi,vel,omega]*n
        #首先，这里要发送储存的state--act数据
        self.Matadatamanager.sendMat(self.stateMat_input) #这里没关系，输入的不是tensor或者array就行
        self.Matdatapointer.is_pudated()
        #无需作返回确认，数据会存在缓存区，直到客户端去处理
        cmd = np.mat("1")
        self.Matadatamanager.sendMat(cmd)
        if(self.Matdatapointer.is_pudated()):
            self.stateMat_plant=self.Matdatapointer.getdata()
        return self.stateMat_plant
    def step_in_healthy(self):
        #首先，这里要发送储存的state--act数据
        self.stateMat_input=self.matlist[1:,:]  # [x,y,phi,vel,omega]*n
        self.Matadatamanager1.sendMat(self.stateMat_input)
        self.Matdatapointer1.is_pudated()
        #无需作返回确认，数据会存在缓存区，直到客户端去处理
        cmd = np.mat("0")
        self.Matadatamanager1.sendMat(cmd)
        if(self.Matdatapointer1.is_pudated()):
            self.stateMat_healthy_plant=self.Matdatapointer1.getdata()
        return self.stateMat_healthy_plant
    def clear_mat(self):
        self.matlist=np.mat("0,0,0,0,0")  # [x,y,phi,vel,omega]其中vel和omega不太必要

# 从py到cpp: 3*5矩阵
# [x,y,phi,vel,omega]*3
#其中vel和omega都是

# cpp向pyhton:
# [x,y,phi,vel,omega]*3
# 补上邻居矩阵信息,应该只要index就够了
# [x,x,-1,-1,-1]
# [x,x,-1,-1,-1]
# [x,x,-1,-1,-1]
# 前面两个就是按照从近到远顺序排列的neibor了
# 最后三个-1是补位用的
# 当然，也有可能，x就是-1，那就代表邻居不存在了
#
# n*5矩阵，
#n为agent数目，可以读维度读出来

#stepin指令：1X1矩阵，1表示步进plant,0表示步进healthy
#回波信号直接使用is_updated即可

#不行，需要重订通讯协议，neibor的判断应该使用cpp的既有代码


""" test get/set & misc """

from datetime import timedelta

import numpy as np
import pytest

from pandas.core.dtypes.common import is_scalar

import pandas as pd
from pandas import (
    Categorical,
    DataFrame,
    IndexSlice,
    MultiIndex,
    Series,
    Timedelta,
    Timestamp,
    date_range,
    period_range,
    timedelta_range,
)
import pandas._testing as tm

from pandas.tseries.offsets import BDay

def test_basic_indexing():
    s = Series(np.random.randn(5), index=["a", "b", "a", "a", "b"])

    msg = "index 5 is out of bounds for axis 0 with size 5"
    with pytest.raises(IndexError, match=msg):
        s[5]
    with pytest.raises(IndexError, match=msg):
        s[5] = 0

    with pytest.raises(KeyError, match=r"^'c'$"):
        s["c"]

    s = s.sort_index()

    with pytest.raises(IndexError, match=msg):
        s[5]
    msg = r"index 5 is out of bounds for axis (0|1) with size 5|^5$"
    with pytest.raises(IndexError, match=msg):
        s[5] = 0

def test_basic_getitem_with_labels(datetime_series):
    indices = datetime_series.index[[5, 10, 15]]

    result = datetime_series[indices]
    expected = datetime_series.reindex(indices)
    tm.assert_series_equal(result, expected)

    result = datetime_series[indices[0] : indices[2]]
    expected = datetime_series.loc[indices[0] : indices[2]]
    tm.assert_series_equal(result, expected)

    # integer indexes, be careful
    s = Series(np.random.randn(10), index=list(range(0, 20, 2)))
    inds = [0, 2, 5, 7, 8]
    arr_inds = np.array([0, 2, 5, 7, 8])
    with pytest.raises(KeyError, match="with any missing labels"):
        s[inds]

    with pytest.raises(KeyError, match="with any missing labels"):
        s[arr_inds]

    # GH12089
    # with tz for values
    s = Series(
        pd.date_range("2011-01-01", periods=3, tz="US/Eastern"), index=["a", "b", "c"]
    )
    expected = Timestamp("2011-01-01", tz="US/Eastern")
    result = s.loc["a"]
    assert result == expected
    result = s.iloc[0]
    assert result == expected
    result = s["a"]
    assert result == expected

def test_getitem_setitem_ellipsis():
    s = Series(np.random.randn(10))

    np.fix(s)

    result = s[...]
    tm.assert_series_equal(result, s)

    s[...] = 5
    assert (result == 5).all()

def test_getitem_get(datetime_series, string_series, object_series):
    idx1 = string_series.index[5]
    idx2 = object_series.index[5]

    assert string_series[idx1] == string_series.get(idx1)
    assert object_series[idx2] == object_series.get(idx2)

    assert string_series[idx1] == string_series[5]
    assert object_series[idx2] == object_series[5]

    assert string_series.get(-1) == string_series.get(string_series.index[-1])
    assert string_series[5] == string_series.get(string_series.index[5])

    # missing
    d = datetime_series.index[0] - BDay()
    msg = r"Timestamp\('1999-12-31 00:00:00', freq='B'\)"
    with pytest.raises(KeyError, match=msg):
        datetime_series[d]

    # None
    # GH 5652
    s1 = Series(dtype=object)
    s2 = Series(dtype=object, index=list("abc"))
    for s in [s1, s2]:
        result = s.get(None)
        assert result is None

def test_getitem_fancy(string_series, object_series):
    slice1 = string_series[[1, 2, 3]]
    slice2 = object_series[[1, 2, 3]]
    assert string_series.index[2] == slice1.index[1]
    assert object_series.index[2] == slice2.index[1]
    assert string_series[2] == slice1[1]
    assert object_series[2] == slice2[1]

def test_type_promotion():
    # GH12599
    s = Series(dtype=object)
    s["a"] = Timestamp("2016-01-01")
    s["b"] = 3.0
    s["c"] = "foo"
    expected = Series([Timestamp("2016-01-01"), 3.0, "foo"], index=["a", "b", "c"])
    tm.assert_series_equal(s, expected)

@pytest.mark.parametrize(
    "result_1, duplicate_item, expected_1",
    [
        [
            Series({1: 12, 2: [1, 2, 2, 3]}),
            Series({1: 313}),
            Series({1: 12}, dtype=object),
        ],
        [
            Series({1: [1, 2, 3], 2: [1, 2, 2, 3]}),
            Series({1: [1, 2, 3]}),
            Series({1: [1, 2, 3]}),
        ],
    ],
)
def test_getitem_with_duplicates_indices(result_1, duplicate_item, expected_1):
    # GH 17610
    result = result_1.append(duplicate_item)
    expected = expected_1.append(duplicate_item)
    tm.assert_series_equal(result[1], expected)
    assert result[2] == result_1[2]

def test_getitem_setitem_integers():
    # caused bug without test
    s = Series([1, 2, 3], ["a", "b", "c"])

    assert s.iloc[0] == s["a"]
    s.iloc[0] = 5
    tm.assert_almost_equal(s["a"], 5)

def test_getitem_box_float64(datetime_series):
    value = datetime_series[5]
    assert isinstance(value, np.float64)

def test_series_box_timestamp():
    rng = pd.date_range("20090415", "20090519", freq="B")
    ser = Series(rng)

    assert isinstance(ser[5], Timestamp)

    rng = pd.date_range("20090415", "20090519", freq="B")
    ser = Series(rng, index=rng)
    assert isinstance(ser[5], Timestamp)

    assert isinstance(ser.iat[5], Timestamp)

def test_series_box_timedelta():
    rng = pd.timedelta_range("1 day 1 s", periods=5, freq="h")
    ser = Series(rng)
    assert isinstance(ser[0], Timedelta)
    assert isinstance(ser.at[1], Timedelta)
    assert isinstance(ser.iat[2], Timedelta)
    assert isinstance(ser.loc[3], Timedelta)
    assert isinstance(ser.iloc[4], Timedelta)

def test_getitem_ambiguous_keyerror():
    s = Series(range(10), index=list(range(0, 20, 2)))
    with pytest.raises(KeyError, match=r"^1$"):
        s[1]
    with pytest.raises(KeyError, match=r"^1$"):
        s.loc[1]

def test_getitem_unordered_dup():
    obj = Series(range(5), index=["c", "a", "a", "b", "b"])
    assert is_scalar(obj["c"])
    assert obj["c"] == 0

def test_getitem_dups_with_missing():
    # breaks reindex, so need to use .loc internally
    # GH 4246
    s = Series([1, 2, 3, 4], ["foo", "bar", "foo", "bah"])
    with pytest.raises(KeyError, match="with any missing labels"):
        s.loc[["foo", "bar", "bah", "bam"]]

    with pytest.raises(KeyError, match="with any missing labels"):
        s[["foo", "bar", "bah", "bam"]]

def test_getitem_dups():
    s = Series(range(5), index=["A", "A", "B", "C", "C"], dtype=np.int64)
    expected = Series([3, 4], index=["C", "C"], dtype=np.int64)
    result = s["C"]
    tm.assert_series_equal(result, expected)

def test_setitem_ambiguous_keyerror():
    s = Series(range(10), index=list(range(0, 20, 2)))

    # equivalent of an append
    s2 = s.copy()
    s2[1] = 5
    expected = s.append(Series([5], index=[1]))
    tm.assert_series_equal(s2, expected)

    s2 = s.copy()
    s2.loc[1] = 5
    expected = s.append(Series([5], index=[1]))
    tm.assert_series_equal(s2, expected)

def test_setitem(datetime_series, string_series):
    datetime_series[datetime_series.index[5]] = np.NaN
    datetime_series[[1, 2, 17]] = np.NaN
    datetime_series[6] = np.NaN
    assert np.isnan(datetime_series[6])
    assert np.isnan(datetime_series[2])
    datetime_series[np.isnan(datetime_series)] = 5
    assert not np.isnan(datetime_series[2])

    # caught this bug when writing tests
    series = Series(tm.makeIntIndex(20).astype(float), index=tm.makeIntIndex(20))

    series[::2] = 0
    assert (series[::2] == 0).all()

    # set item that's not contained
    s = string_series.copy()
    s["foobar"] = 1

    app = Series([1], index=["foobar"], name="series")
    expected = string_series.append(app)
    tm.assert_series_equal(s, expected)

def test_setitem_dtypes():
    # change dtypes
    # GH 4463
    expected = Series([np.nan, 2, 3])

    s = Series([1, 2, 3])
    s.iloc[0] = np.nan
    tm.assert_series_equal(s, expected)

    s = Series([1, 2, 3])
    s.loc[0] = np.nan
    tm.assert_series_equal(s, expected)

    s = Series([1, 2, 3])
    s[0] = np.nan
    tm.assert_series_equal(s, expected)

    s = Series([False])
    s.loc[0] = np.nan
    tm.assert_series_equal(s, Series([np.nan]))

    s = Series([False, True])
    s.loc[0] = np.nan
    tm.assert_series_equal(s, Series([np.nan, 1.0]))

def test_setslice(datetime_series):
    sl = datetime_series[5:20]
    assert len(sl) == len(sl.index)
    assert sl.index.is_unique is True

def test_loc_setitem_2d_to_1d_raises():
    x = np.random.randn(2, 2)
    y = Series(range(2))

    msg = "|".join(
        [
            r"shape mismatch: value array of shape \(2,2\)",
            r"cannot reshape array of size 4 into shape \(2,\)",
        ]
    )
    with pytest.raises(ValueError, match=msg):
        y.loc[range(2)] = x

    msg = r"could not broadcast input array from shape \(2,2\) into shape \(2,?\)"
    with pytest.raises(ValueError, match=msg):
        y.loc[:] = x

# FutureWarning from NumPy about [slice(None, 5).
@pytest.mark.filterwarnings("ignore:Using a non-tuple:FutureWarning")
def test_basic_getitem_setitem_corner(datetime_series):
    # invalid tuples, e.g. td.ts[:, None] vs. td.ts[:, 2]
    msg = "key of type tuple not found and not a MultiIndex"
    with pytest.raises(KeyError, match=msg):
        datetime_series[:, 2]
    with pytest.raises(KeyError, match=msg):
        datetime_series[:, 2] = 2

    # weird lists. [slice(0, 5)] will work but not two slices
    with tm.assert_produces_warning(FutureWarning):
        # GH#31299
        result = datetime_series[[slice(None, 5)]]
    expected = datetime_series[:5]
    tm.assert_series_equal(result, expected)

    # OK
    msg = r"unhashable type(: 'slice')?"
    with pytest.raises(TypeError, match=msg):
        datetime_series[[5, slice(None, None)]]
    with pytest.raises(TypeError, match=msg):
        datetime_series[[5, slice(None, None)]] = 2

@pytest.mark.parametrize("tz", ["US/Eastern", "UTC", "Asia/Tokyo"])
def test_setitem_with_tz(tz):
    orig = Series(pd.date_range("2016-01-01", freq="H", periods=3, tz=tz))
    assert orig.dtype == f"datetime64[ns, {tz}]"

    # scalar
    s = orig.copy()
    s[1] = Timestamp("2011-01-01", tz=tz)
    exp = Series(
        [
            Timestamp("2016-01-01 00:00", tz=tz),
            Timestamp("2011-01-01 00:00", tz=tz),
            Timestamp("2016-01-01 02:00", tz=tz),
        ]
    )
    tm.assert_series_equal(s, exp)

    s = orig.copy()
    s.loc[1] = Timestamp("2011-01-01", tz=tz)
    tm.assert_series_equal(s, exp)

    s = orig.copy()
    s.iloc[1] = Timestamp("2011-01-01", tz=tz)
    tm.assert_series_equal(s, exp)

    # vector
    vals = Series(
        [Timestamp("2011-01-01", tz=tz), Timestamp("2012-01-01", tz=tz)],
        index=[1, 2],
    )
    assert vals.dtype == f"datetime64[ns, {tz}]"

    s[[1, 2]] = vals
    exp = Series(
        [
            Timestamp("2016-01-01 00:00", tz=tz),
            Timestamp("2011-01-01 00:00", tz=tz),
            Timestamp("2012-01-01 00:00", tz=tz),
        ]
    )
    tm.assert_series_equal(s, exp)

    s = orig.copy()
    s.loc[[1, 2]] = vals
    tm.assert_series_equal(s, exp)

    s = orig.copy()
    s.iloc[[1, 2]] = vals
    tm.assert_series_equal(s, exp)

def test_setitem_with_tz_dst():
    # GH XXX TODO: fill in GH ref
    tz = "US/Eastern"
    orig = Series(pd.date_range("2016-11-06", freq="H", periods=3, tz=tz))
    assert orig.dtype == f"datetime64[ns, {tz}]"

    # scalar
    s = orig.copy()
    s[1] = Timestamp("2011-01-01", tz=tz)
    exp = Series(
        [
            Timestamp("2016-11-06 00:00-04:00", tz=tz),
            Timestamp("2011-01-01 00:00-05:00", tz=tz),
            Timestamp("2016-11-06 01:00-05:00", tz=tz),
        ]
    )
    tm.assert_series_equal(s, exp)

    s = orig.copy()
    s.loc[1] = Timestamp("2011-01-01", tz=tz)
    tm.assert_series_equal(s, exp)

    s = orig.copy()
    s.iloc[1] = Timestamp("2011-01-01", tz=tz)
    tm.assert_series_equal(s, exp)

    # vector
    vals = Series(
        [Timestamp("2011-01-01", tz=tz), Timestamp("2012-01-01", tz=tz)],
        index=[1, 2],
    )
    assert vals.dtype == f"datetime64[ns, {tz}]"

    s[[1, 2]] = vals
    exp = Series(
        [
            Timestamp("2016-11-06 00:00", tz=tz),
            Timestamp("2011-01-01 00:00", tz=tz),
            Timestamp("2012-01-01 00:00", tz=tz),
        ]
    )
    tm.assert_series_equal(s, exp)

    s = orig.copy()
    s.loc[[1, 2]] = vals
    tm.assert_series_equal(s, exp)

    s = orig.copy()
    s.iloc[[1, 2]] = vals
    tm.assert_series_equal(s, exp)

def test_categorical_assigning_ops():
    orig = Series(Categorical(["b", "b"], categories=["a", "b"]))
    s = orig.copy()
    s[:] = "a"
    exp = Series(Categorical(["a", "a"], categories=["a", "b"]))
    tm.assert_series_equal(s, exp)

    s = orig.copy()
    s[1] = "a"
    exp = Series(Categorical(["b", "a"], categories=["a", "b"]))
    tm.assert_series_equal(s, exp)

    s = orig.copy()
    s[s.index > 0] = "a"
    exp = Series(Categorical(["b", "a"], categories=["a", "b"]))
    tm.assert_series_equal(s, exp)

    s = orig.copy()
    s[[False, True]] = "a"
    exp = Series(Categorical(["b", "a"], categories=["a", "b"]))
    tm.assert_series_equal(s, exp)

    s = orig.copy()
    s.index = ["x", "y"]
    s["y"] = "a"
    exp = Series(Categorical(["b", "a"], categories=["a", "b"]), index=["x", "y"])
    tm.assert_series_equal(s, exp)

    # ensure that one can set something to np.nan
    s = Series(Categorical([1, 2, 3]))
    exp = Series(Categorical([1, np.nan, 3], categories=[1, 2, 3]))
    s[1] = np.nan
    tm.assert_series_equal(s, exp)

def test_getitem_categorical_str():
    # GH#31765
    ser = Series(range(5), index=Categorical(["a", "b", "c", "a", "b"]))
    result = ser["a"]
    expected = ser.iloc[[0, 3]]
    tm.assert_series_equal(result, expected)

    # Check the intermediate steps work as expected
    with tm.assert_produces_warning(FutureWarning):
        result = ser.index.get_value(ser, "a")
    tm.assert_series_equal(result, expected)

def test_slice(string_series, object_series):
    numSlice = string_series[10:20]
    numSliceEnd = string_series[-10:]
    objSlice = object_series[10:20]

    assert string_series.index[9] not in numSlice.index
    assert object_series.index[9] not in objSlice.index

    assert len(numSlice) == len(numSlice.index)
    assert string_series[numSlice.index[0]] == numSlice[numSlice.index[0]]

    assert numSlice.index[1] == string_series.index[11]
    assert tm.equalContents(numSliceEnd, np.array(string_series)[-10:])

    # Test return view.
    sl = string_series[10:20]
    sl[:] = 0

    assert (string_series[10:20] == 0).all()

def test_slice_can_reorder_not_uniquely_indexed():
    s = Series(1, index=["a", "a", "b", "b", "c"])
    s[::-1]  # it works!

def test_loc_setitem(string_series):
    inds = string_series.index[[3, 4, 7]]

    result = string_series.copy()
    result.loc[inds] = 5

    expected = string_series.copy()
    expected[[3, 4, 7]] = 5
    tm.assert_series_equal(result, expected)

    result.iloc[5:10] = 10
    expected[5:10] = 10
    tm.assert_series_equal(result, expected)

    # set slice with indices
    d1, d2 = string_series.index[[5, 15]]
    result.loc[d1:d2] = 6
    expected[5:16] = 6  # because it's inclusive
    tm.assert_series_equal(result, expected)

    # set index value
    string_series.loc[d1] = 4
    string_series.loc[d2] = 6
    assert string_series[d1] == 4
    assert string_series[d2] == 6

def test_timedelta_assignment():
    # GH 8209
    s = Series([], dtype=object)
    s.loc["B"] = timedelta(1)
    tm.assert_series_equal(s, Series(Timedelta("1 days"), index=["B"]))

    s = s.reindex(s.index.insert(0, "A"))
    tm.assert_series_equal(s, Series([np.nan, Timedelta("1 days")], index=["A", "B"]))

    s.loc["A"] = timedelta(1)
    expected = Series(Timedelta("1 days"), index=["A", "B"])
    tm.assert_series_equal(s, expected)

    # GH 14155
    s = Series(10 * [np.timedelta64(10, "m")])
    s.loc[[1, 2, 3]] = np.timedelta64(20, "m")
    expected = Series(10 * [np.timedelta64(10, "m")])
    expected.loc[[1, 2, 3]] = Timedelta(np.timedelta64(20, "m"))
    tm.assert_series_equal(s, expected)

@pytest.mark.parametrize(
    "nat_val,should_cast",
    [
        (pd.NaT, True),
        (np.timedelta64("NaT", "ns"), False),
        (np.datetime64("NaT", "ns"), True),
    ],
)
@pytest.mark.parametrize("tz", [None, "UTC"])
def test_dt64_series_assign_nat(nat_val, should_cast, tz):
    # some nat-like values should be cast to datetime64 when inserting
    #  into a datetime64 series.  Others should coerce to object
    #  and retain their dtypes.
    dti = pd.date_range("2016-01-01", periods=3, tz=tz)
    base = Series(dti)
    expected = Series([pd.NaT] + list(dti[1:]), dtype=dti.dtype)
    if not should_cast:
        expected = expected.astype(object)

    ser = base.copy(deep=True)
    ser[0] = nat_val
    tm.assert_series_equal(ser, expected)

    ser = base.copy(deep=True)
    ser.loc[0] = nat_val
    tm.assert_series_equal(ser, expected)

    ser = base.copy(deep=True)
    ser.iloc[0] = nat_val
    tm.assert_series_equal(ser, expected)

@pytest.mark.parametrize(
    "nat_val,should_cast",
    [
        (pd.NaT, True),
        (np.timedelta64("NaT", "ns"), True),
        (np.datetime64("NaT", "ns"), False),
    ],
)
def test_td64_series_assign_nat(nat_val, should_cast):
    # some nat-like values should be cast to timedelta64 when inserting
    #  into a timedelta64 series.  Others should coerce to object
    #  and retain their dtypes.
    base = Series([0, 1, 2], dtype="m8[ns]")
    expected = Series([pd.NaT, 1, 2], dtype="m8[ns]")
    if not should_cast:
        expected = expected.astype(object)

    ser = base.copy(deep=True)
    ser[0] = nat_val
    tm.assert_series_equal(ser, expected)

    ser = base.copy(deep=True)
    ser.loc[0] = nat_val
    tm.assert_series_equal(ser, expected)

    ser = base.copy(deep=True)
    ser.iloc[0] = nat_val
    tm.assert_series_equal(ser, expected)

@pytest.mark.parametrize(
    "td",
    [
        Timedelta("9 days"),
        Timedelta("9 days").to_timedelta64(),
        Timedelta("9 days").to_pytimedelta(),
    ],
)
def test_append_timedelta_does_not_cast(td):
    # GH#22717 inserting a Timedelta should _not_ cast to int64
    expected = Series(["x", td], index=[0, "td"], dtype=object)

    ser = Series(["x"])
    ser["td"] = td
    tm.assert_series_equal(ser, expected)
    assert isinstance(ser["td"], Timedelta)

    ser = Series(["x"])
    ser.loc["td"] = Timedelta("9 days")
    tm.assert_series_equal(ser, expected)
    assert isinstance(ser["td"], Timedelta)

def test_underlying_data_conversion():
    # GH 4080
    df = DataFrame({c: [1, 2, 3] for c in ["a", "b", "c"]})
    return_value = df.set_index(["a", "b", "c"], inplace=True)
    assert return_value is None
    s = Series([1], index=[(2, 2, 2)])
    df["val"] = 0
    df
    df["val"].update(s)

    expected = DataFrame(
        {"a": [1, 2, 3], "b": [1, 2, 3], "c": [1, 2, 3], "val": [0, 1, 0]}
    )
    return_value = expected.set_index(["a", "b", "c"], inplace=True)
    assert return_value is None
    tm.assert_frame_equal(df, expected)

    # GH 3970
    # these are chained assignments as well
    pd.set_option("chained_assignment", None)
    df = DataFrame({"aa": range(5), "bb": [2.2] * 5})
    df["cc"] = 0.0

    ck = [True] * len(df)

    df["bb"].iloc[0] = 0.13

    # TODO: unused
    df_tmp = df.iloc[ck]  # noqa

    df["bb"].iloc[0] = 0.15
    assert df["bb"].iloc[0] == 0.15
    pd.set_option("chained_assignment", "raise")

    # GH 3217
    df = DataFrame({"a": [1, 3], "b": [np.nan, 2]})
    df["c"] = np.nan
    df["c"].update(Series(["foo"], index=[0]))

    expected = DataFrame({"a": [1, 3], "b": [np.nan, 2], "c": ["foo", np.nan]})
    tm.assert_frame_equal(df, expected)

def test_preserve_refs(datetime_series):
    seq = datetime_series[[5, 10, 15]]
    seq[1] = np.NaN
    assert not np.isnan(datetime_series[10])

def test_cast_on_putmask():
    # GH 2746

    # need to upcast
    s = Series([1, 2], index=[1, 2], dtype="int64")
    s[[True, False]] = Series([0], index=[1], dtype="int64")
    expected = Series([0, 2], index=[1, 2], dtype="int64")

    tm.assert_series_equal(s, expected)

def test_type_promote_putmask():
    # GH8387: test that changing types does not break alignment
    ts = Series(np.random.randn(100), index=np.arange(100, 0, -1)).round(5)
    left, mask = ts.copy(), ts > 0
    right = ts[mask].copy().map(str)
    left[mask] = right
    tm.assert_series_equal(left, ts.map(lambda t: str(t) if t > 0 else t))

    s = Series([0, 1, 2, 0])
    mask = s > 0
    s2 = s[mask].map(str)
    s[mask] = s2
    tm.assert_series_equal(s, Series([0, "1", "2", 0]))

    s = Series([0, "foo", "bar", 0])
    mask = Series([False, True, True, False])
    s2 = s[mask]
    s[mask] = s2
    tm.assert_series_equal(s, Series([0, "foo", "bar", 0]))

def test_multilevel_preserve_name():
    index = MultiIndex(
        levels=[["foo", "bar", "baz", "qux"], ["one", "two", "three"]],
        codes=[[0, 0, 0, 1, 1, 2, 2, 3, 3, 3], [0, 1, 2, 0, 1, 1, 2, 0, 1, 2]],
        names=["first", "second"],
    )
    s = Series(np.random.randn(len(index)), index=index, name="sth")

    result = s["foo"]
    result2 = s.loc["foo"]
    assert result.name == s.name
    assert result2.name == s.name

"""
miscellaneous methods
"""

def test_uint_drop(any_int_dtype):
    # see GH18311
    # assigning series.loc[0] = 4 changed series.dtype to int
    series = Series([1, 2, 3], dtype=any_int_dtype)
    series.loc[0] = 4
    expected = Series([4, 2, 3], dtype=any_int_dtype)
    tm.assert_series_equal(series, expected)

def test_getitem_unrecognized_scalar():
    # GH#32684 a scalar key that is not recognized by lib.is_scalar

    # a series that might be produced via `frame.dtypes`
    ser = Series([1, 2], index=[np.dtype("O"), np.dtype("i8")])

    key = ser.index[1]

    result = ser[key]
    assert result == 2

@pytest.mark.parametrize(
    "index",
    [
        date_range("2014-01-01", periods=20, freq="MS"),
        period_range("2014-01", periods=20, freq="M"),
        timedelta_range("0", periods=20, freq="H"),
    ],
)
def test_slice_with_zero_step_raises(index):
    ts = Series(np.arange(20), index)

    with pytest.raises(ValueError, match="slice step cannot be zero"):
        ts[::0]
    with pytest.raises(ValueError, match="slice step cannot be zero"):
        ts.loc[::0]
    with pytest.raises(ValueError, match="slice step cannot be zero"):
        ts.iloc[::0]

@pytest.mark.parametrize(
    "index",
    [
        date_range("2014-01-01", periods=20, freq="MS"),
        period_range("2014-01", periods=20, freq="M"),
        timedelta_range("0", periods=20, freq="H"),
    ],
)
def test_slice_with_negative_step(index):
    def assert_slices_equivalent(l_slc, i_slc):
        expected = ts.iloc[i_slc]

        tm.assert_series_equal(ts[l_slc], expected)
        tm.assert_series_equal(ts.loc[l_slc], expected)
        tm.assert_series_equal(ts.loc[l_slc], expected)

    keystr1 = str(index[9])
    keystr2 = str(index[13])
    box = type(index[0])

    ts = Series(np.arange(20), index)
    SLC = IndexSlice

    for key in [keystr1, box(keystr1)]:
        assert_slices_equivalent(SLC[key::-1], SLC[9::-1])
        assert_slices_equivalent(SLC[:key:-1], SLC[:8:-1])

        for key2 in [keystr2, box(keystr2)]:
            assert_slices_equivalent(SLC[key2:key:-1], SLC[13:8:-1])
            assert_slices_equivalent(SLC[key:key2:-1], SLC[0:0:-1])

def test_tuple_index():
    # GH 35534 - Selecting values when a Series has an Index of tuples
    s = Series([1, 2], index=[("a",), ("b",)])
    assert s[("a",)] == 1
    assert s[("b",)] == 2
    s[("b",)] = 3
    assert s[("b",)] == 3

def test_frozenset_index():
    # GH35747 - Selecting values when a Series has an Index of frozenset
    idx0, idx1 = frozenset("a"), frozenset("b")
    s = Series([1, 2], index=[idx0, idx1])
    assert s[idx0] == 1
    assert s[idx1] == 2
    s[idx1] = 3
    assert s[idx1] == 3


#!/usr/bin/env python

import sys
# add current directory to path
sys.path.insert(0, '')

from sa import _version, repo as rp, versioning, docker_api
import os, argparse, git

def main():
    # initialize argparser
    parser = argparse.ArgumentParser()
    # initialize sub argument parsers
    subparsers = parser.add_subparsers(dest='command')

    parser.add_argument('--version', action='version',
                    version='%(prog)s {version}'.format(version=_version.__version__))

    parser.add_argument('-gu','--gitusername',
                    dest='git_username',
                    type=str,
                    help='git username to use during commits, defaults to \'Jenkins\'')
    parser.add_argument('-ge','--gituseremail',
                    dest='git_useremail',
                    type=str,
                    help='git email to use during commits, defaults to \'jenkins@noreply.com\'')

    # git
    parser_git = subparsers.add_parser('git')
    subparser_git = parser_git.add_subparsers(dest='git_subcommand')
    # git push
    git_subparser_push = subparser_git.add_parser('push')
    add_required_creds_args(git_subparser_push)
    add_required_repo_args(git_subparser_push)
    # git checkprevcommit
    git_subparser_checkprevcommit = subparsers.add_parser('checkprevcommit')
    add_required_repo_args(git_subparser_checkprevcommit)

    # version
    parser_version = subparsers.add_parser('version')
    subparser_version = parser_version.add_subparsers(dest='version_subcommand')
    # version get
    version_subparser_get = subparser_version.add_parser('get')
    add_optional_docker_arg(version_subparser_get)
    add_required_files_args(version_subparser_get)
    # version bump
    version_subparser_bump = subparser_version.add_parser('bump')
    add_required_repo_args(version_subparser_bump)
    add_optional_docker_arg(version_subparser_bump)
    add_required_files_args(version_subparser_bump)
    # TODO add flag to allow for custom provided versioning script

    # docker
    parser_docker = subparsers.add_parser('docker')
    subparser_docker = parser_docker.add_subparsers(dest='docker_subcommand')
    # docker build
    docker_subparser_build = subparser_docker.add_parser('build')
    add_required_files_args(docker_subparser_build)
    # docker geninfo
    #TODO: generate sample docker info json file

    # Process CLI arguments
    options = parser.parse_args()

    if options.command == 'git':

        if options.git_subcommand == 'push':
            rp.push(get_git_repo(options.repo_dir), options.username, options.password)
            exit()
        elif options.git_subcommand == 'checkprevcommit':
            #TODO: implement function
            rp.check_prev_commit(get_git_repo(options.repo_dir))

    elif options.command == 'version':

        if options.version_subcommand == 'bump':
            # Process all file paths provided and only include the valid files
            rp.version_bump(get_git_repo(options.repo_dir), get_files(options.files), options)
            exit()
        elif options.version_subcommand == 'get':
            # Process all files paths provided and only include the valid files
            version_info = versioning.version_get(get_files(options.files), options)
            for file,version in version_info:
                print(''.join([file,' ', version]))
            exit()

    elif options.command == 'docker':

        if options.docker_subcommand == 'build':
            docker_api.build(get_files(options.files))
        elif options.docker_subcommand == 'push':
            docker_api.push()

def get_git_repo(repo_dir):
    # get/validate absolute path of provided repository directory
    abs_repo_path = ""
    if repo_dir != None:
        if os.path.isdir(repo_dir):
            abs_repo_path = os.path.abspath(repo_dir)
        else:
            # raise argparse.ArgumentTypeError(f"readable_dir:{repo_dir} is not a valid path")
            print(f'readable_dir:{abs_repo_path} is not a valid path')
            exit(1)

    # validate provided path is a git repository and setup for use
    if (rp.is_git_repo(abs_repo_path)):
        repo = git.Repo(abs_repo_path)
        rp.reattach_head(repo)
    else:
        # raise Exception(f"{repo_dir} is not a valid git repository")
        print(f'{abs_repo_path} is not a valid git repository')
        exit(1)
    return repo

def get_files(infiles):
    files = []
    if infiles != None:
        for f in infiles:
            if os.path.isfile(f):
                files.append(os.path.abspath(f))

    if (len(files) == 0):
        print('no files provided, use \'-h|--help\' for more details'.format())
        exit(1)
    return files

def add_required_files_args(parser):
    parser.add_argument(
                    dest='files',
                    type=str,
                    nargs='*',
                    help='list of all files to be to be operated on')

def add_optional_docker_arg(parser):
    parser.add_argument('-d','--docker',
                    action='store_true',
                    help='indicate if provided files are docker-info.json files for versioning')

def add_required_creds_args(parser):
    parser.add_argument('-u','--username',
                    dest='username',
                    required=True,
                    help='username for logging into service')
    parser.add_argument('-p','--password',
                    dest='password',
                    required=True,
                    help='password for logging into service')

def add_required_repo_args(parser):
    parser.add_argument('-r','--repo',
                    dest='repo_dir',
                    type=str,
                    required=True,
                    help='specific path to git project files')

if __name__ == '__main__':
    sys.exit(main())


# %%
import numpy as np
import random

# namedtuple:   factory function for creating tuple subclasses with named fields
# deque:        list-like container with fast appends and pops on either end
from collections import namedtuple, deque

# Custom DQN model set up in model.py
from model import QNetwork

# Import torch framework
import torch
import torch.nn.functional as F
import torch.optim as optim

# %%
# Set hyperparameters
BUFFER_SIZE = int(100000)   # Replay memory size
BATCH_SIZE = 64             # Mini-batch size
GAMMA = 0.99                # Discount factor
TAU = 0.001                 # For soft update of target parameters
LR = 0.0001                 # Learning rate alpha
UPDATE_EVERY = 4            # How often to update the network

# Check if we have CUDA enabled device i.e. a GPU, else use the CPU
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

# Define DQN agent
class Agent():
    '''Agent to interact with the environment and learn
    '''

    def __init__(self, state_size, action_size, seed):
        self.state_size = state_size
        self.action_size = action_size
        self.seed = random.seed(seed)

        ### Make Q-network, both the local and the target network ###
        self.qnetwork_local = QNetwork(state_size, action_size, seed)
        self.qnetwork_target = QNetwork(state_size, action_size, seed)

        ### Set optimizer for gradient descent ###
        # Select the parameters of the local qnetwork we are training on.
        # Weights will later be copied to the target qnetwork
        self.optimizer = optim.Adam(params=self.qnetwork_local.parameters(), lr=LR)

        ### Initialize replay memory ###
        self.replay_memory = ReplayMemory(buffer_size=BUFFER_SIZE, batch_size=BATCH_SIZE)

        ### Initialize time step (for updating every UPDATE_EVERY steps)###
        self.t_step = 0

    def step(self, state, action, reward, next_state, done):
        ### Save experience in memory ###
        self.replay_memory.add(state, action, reward, next_state, done)

        ### Learn every UPDATE_EVERY time steps ###
        self.t_step += 1
        # If remainder of division is 0, then learn from experience replay memory
        if self.t_step % UPDATE_EVERY == 0:
            # If we have collected enough experience in our memory i.e. more
            # than the mini-batch size, then call the self.learn() function
            if len(self.replay_memory) > BATCH_SIZE:
                experiences = self.replay_memory.sample()
                self.learn(experiences, GAMMA)

    def act(self, state, eps=0.0):
        """Return actions for the given state and policy (e.g. ε-greedy)

        Arguments:
            state {[numpy array]} -- Current state
            eps {float} -- Epsilon for ε-greedy policy (default: {0.0})
        """
        # Convert 'state' numpy array to pytorch tensor using the current device
        # i.e. GPU or CPU.
        # unsqueeze() returns a new tensor with a dimension of size one
        # inserted at the specified position.
        state = torch.from_numpy(state).float().unsqueeze(0).to(device)

        # Set the module in evaluation mode.
        self.qnetwork_local.eval()
        with torch.no_grad():
            # Evaluate the Q-network with the current state
            # Call a forward pass with the current state to get q-values for
            # that state.
            action_values = self.qnetwork_local(state)

        # Set the module in training mode.
        self.qnetwork_local.train()

        # ε-greedy action selection
        if random.random() > eps:
            # Get action with the max q-value
            # .cpu(): Move all model parameters and buffers to the CPU
            return np.argmax(action_values.cpu().data.numpy())
        else:
            return random.choice(np.arange(self.action_size))

    def learn(self, experiences, gamma):
        """Update value parameters using given batch of experience tuples

        Arguments:
            experiences {Tupple(torch.Tensor)} -- Tuple of (s, a, r, s', done) tuples
            gamma {float} -- Discount factor
        """
        # Experiences, mini-batch of 64
        states, actions, rewards, next_states, dones = experiences

        ### Get max predicted Q-values (for next states) from target model ###
        # .detach(): returns a new Variable that does not back-propagate to
        # whatever detach() was called on. Creates a tensor that shares storage
        # with tensor that does not require grad.
        # .max(): return the max value of a tensor on a given dimension (dim)
        Q_targets_next = self.qnetwork_target(next_states).detach().max(1)[0].unsqueeze(1)
        # size([4, 1])
        # the tensors below are [64, x]
        ### Computer Q-targets for current states ###
        Q_targets = rewards + (gamma * Q_targets_next * (1 - dones))

        ### Get expected Q-values from local model ###
        # .gather(): Gathers values along an axis specified by dim.
        Q_expected = self.qnetwork_local(states).gather(1, actions)

        ### Compute loss ###
        loss = F.mse_loss(Q_expected, Q_targets)
        # Minimize loss
        # Clear the gradients so they are not accumulated
        self.optimizer.zero_grad()
        loss.backward()
        # Perform a single optimization step.
        self.optimizer.step()

        ### Update target network ###
        self.soft_update(self.qnetwork_local, self.qnetwork_target, TAU)

    def soft_update(self, local_model, target_model, tau):
        """Soft update model parameters

        Arguments:
            local_model {PyTorch model} -- Copy weights from
            target_model {PyTorch model} -- Copy weights to
            tau {float} -- Interpolation parameter
        """
        for target_param, local_param in zip(target_model.parameters(), local_model.parameters()):
            target_param.data.copy_(tau*local_param.data + (1.0-tau) * target_param.data)

# Define replay memory class
class ReplayMemory:
    def __init__(self, buffer_size, batch_size):
        """Initialise a replay memory buffer
        """
        self.memory = deque(maxlen=buffer_size)
        self.batch_size = batch_size
        self.experience = namedtuple("Experience", field_names=["state", "action", "reward", "next_state", "done"])

    def add(self, state, action, reward, next_state, done):
        """Add new experience to the replay memory buffer

        Arguments:
            state {[type]} -- Current state
            action {[type]} -- Current action
            reward {[type]} -- Reward for taking action in current state
            next_state {[type]} -- Next state after taking action
            done {bool} -- Whether episode is done (terminal state)
        """
        new_experience = self.experience(state, action, reward, next_state, done)
        self.memory.append(new_experience)

    def sample(self):
        """Sample a random mini-batch from the experience replay buffer
        """
        sampled_experiences = random.sample(self.memory, k=self.batch_size)

        # Convert the sampled experiences to torch tensors on the GPU or CPU,
        # depending on what 'device' is. 'device' is set at the beggining of
        # the script.
        # The '.to()' method of Tensors and Modules can be used to easily move
        # objects to different devices (instead of having to call cpu() or
        # cuda() based on the context).
        states = torch.from_numpy(np.vstack([e.state for e in sampled_experiences if e is not None])).float().to(device)
        actions = torch.from_numpy(np.vstack([e.action for e in sampled_experiences if e is not None])).long().to(device)
        rewards = torch.from_numpy(np.vstack([e.reward for e in sampled_experiences if e is not None])).float().to(device)
        next_states = torch.from_numpy(np.vstack([e.next_state for e in sampled_experiences if e is not None])).float().to(device)
        dones = torch.from_numpy(np.vstack([e.done for e in sampled_experiences if e is not None]).astype(np.uint8)).float().to(device)

        # Return tuple of random sampled experiences
        return (states, actions, rewards, next_states, dones)

    def __len__(self):
        """Return the length of the replay memory

        Returns:
            [int] -- Length of the current replay memory
        """
        return len(self.memory)

Project 1 - Navigation with DQN Agent/agent.py


import os

from setuptools import find_namespace_packages, setup

here = os.path.abspath(os.path.dirname(__file__))
with open(os.path.join(here, 'VERSION')) as f:
    VERSION = f.read().strip()

with open(os.path.join(here, 'README.md')) as f:
    README = f.read()

requires = [
    'bcrypt',
    'fastapi',
    'uvicorn',
    'passlib',
    'python-dotenv',

    'pyjwt',
    'cryptography',

    # http client stuff
    # see: https://docs.aiohttp.org/en/v2.3.4/
    'aiohttp',
    'cchardet',
    'aiodns',
    # another one
    'httpx',
]

standalone_require = [
]

tests_require = [
    'mypy>=0.760',
    'pytest',
    'requests',
    'vcrpy',
]

setup(name='pansen-castlabs',
      version=VERSION,
      description='castlabs challenge',
      long_description=README,
      classifiers=[
          "Programming Language :: Python",
          "Programming Language :: Python :: 3 :: Only",
          "Programming Language :: Python :: 3.7",
          "Programming Language :: Python :: 3.8",
          "Framework :: FastAPI",
          "Topic :: Internet :: WWW/HTTP",
          "Topic :: Internet :: WWW/HTTP :: ASGI :: Application"
      ],
      keywords="web services",
      author='pansen',
      author_email='andi@zerotired.com',
      url='',
      packages=find_namespace_packages(include=['pansen.*']),
      include_package_data=True,
      # http://pythonhosted.org/distribute/setuptools.html#namespace-packages
      namespace_packages=['pansen'],
      zip_safe=False,
      extras_require={
          'testing': tests_require + standalone_require,
          'standalone': standalone_require,
      },
      install_requires=requires,
      entry_points={
          'console_scripts': [
              'pansen_castlabs = pansen.castlabs.main:run'
          ],
      },
      )


__author__ = 'Karl'

import pandas as pd
import numpy as np
from sklearn import cross_validation
from time import process_time
from sklearn.cross_validation import train_test_split
import xgboost as xgb

# Thanks to Chenglong Chen for providing this in the forum
def ToWeight(y):
    w = np.zeros(y.shape, dtype=float)
    ind = y != 0
    w[ind] = 1./(y[ind]**2)
    return w

def rmspe(yhat, y):
    w = ToWeight(y)
    rmspe = np.sqrt(np.mean( w * (y - yhat)**2 ))
    return rmspe

def rmspe_xg(yhat, y):
    # y = y.values
    y = y.get_label()
    y = np.exp(y) - 1
    yhat = np.exp(yhat) - 1
    w = ToWeight(y)
    rmspe = np.sqrt(np.mean(w * (y - yhat)**2))
    return "rmspe", rmspe

# Gather some features
def merge(df):
    df = df.merge(store, on='Store', how="left")
    # Some Store have empty open value on test set
    df.loc[df.Open.isnull(), 'Open'] = 1
    # Break down date column
    df['year'] = df.Date.apply(lambda x: x.year)
    df['month'] = df.Date.apply(lambda x: x.month)
    df['woy'] = df.Date.apply(lambda x: x.weekofyear)

    # Calculate time competition open time in months
    df['CompetitionOpen'] = 12 * (df.year - df.CompetitionOpenSinceYear) + (df.month - df.CompetitionOpenSinceMonth)
    df['CompetitionOpen'] = df.CompetitionOpen.apply(lambda x: x if x > 0 else 0)

    # Promo open time in months
    df['PromoOpen'] = 12 * (df.year - df.Promo2SinceYear) + (df.woy - df.Promo2SinceWeek) / float(4)
    df['PromoOpen'] = df.PromoOpen.apply(lambda x: x if x > 0 else 0)
    df['PromoInterval'] = df.PromoInterval.apply(lambda x: x[0] if type(x) == str else 'N')

    df['StateHoliday'] = df.StateHoliday.apply(lambda x: 1 if (x != '0') else 0)

    df = pd.get_dummies(df, columns=[
        'PromoInterval',
        'StoreType',
        'Assortment'
    ])
    df.drop([
        'Date',
        'Open',
        # 'Store',
        'Promo2SinceYear',
        'Promo2SinceWeek',
        'CompetitionOpenSinceMonth',
        'CompetitionOpenSinceYear',
        'year'], axis=1, inplace=True)
    # bug ValueError: all feature_names must be alphanumerics
    df = df.rename(columns=lambda x: x.replace('_', ''))

    return df

# Load data
t = process_time()
data = pd.read_csv('input/train.csv', parse_dates=['Date'], dtype={'StateHoliday': str})
store = pd.read_csv('input/store.csv')
test = pd.read_csv('input/test.csv', parse_dates=['Date'], dtype={'StateHoliday': str})

data = data[:1000]
test = data[:100]

print('training data loaded in ', process_time() - t)

# exclude closed store
data = data[data['Open'] != 0]

# merge
t = process_time()
data = merge(data)
print('training data processed in ', process_time() - t)

# calculate avg of customer for each store
df_customer = data[['Store', 'Customers']].groupby(['Store']).mean().reset_index()
data = data.drop(['Customers'], axis=1)

data = data.merge(df_customer, on='Store', how='left')
data = data.drop(['Store'], axis=1)

# Make sure columns are in right order
X = data.sort_index(axis=1).fillna(0).drop(['Sales'], axis=1).astype(float)
y = data['Sales'].astype(float)

# Prepare Test Set
t = process_time()
test = merge(test)
test = test.merge(df_customer, on='Store')
test = test.drop(['Store'], axis=1).set_index('Id')
X_test_submission = test.sort_index(axis=1).fillna(0).astype(float)
print('test data processed in ', process_time() - t)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.10, random_state=2)

# xgb_model = xgb.XGBRegressor().fit(X_train, y_train)
# y_pred = xgb_model.predict(X_test)
# print("RSMPE: %.3f" % rmspe(y_pred, y_test))

# make submission
t = process_time()
print("start fitting xgb")
xgb_model = xgb.XGBRegressor().fit(X, y)
print("fitting finished after ", process_time() - t)
t = process_time()
y_pred = xgb_model.predict(X_test_submission)
result = pd.DataFrame({'Id': X_test_submission.index.values, 'Sales': y_pred}).set_index('Id')
result = result.sort_index()
result.to_csv('submission.xgb.csv')
result.to_csv('submission.xgb.excel.csv', sep=";", decimal=",")
print('submission created in ', process_time() - t)

# features = X.columns.values
#
# # params = {"objective": "reg:linear",
# #           "eta": 0.3,
# #           "max_depth": 8,
# #           "subsample": 0.7,
# #           "colsample_bytree": 0.7,
# #           "silent": 1
# #           }
# # num_trees = 300
#
# print("Train a XGBoost model")
# val_size = 100000
# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.10, random_state=2)
# dtrain = xgb.DMatrix(X_train[features], np.log(y_train + 1))
# dvalid = xgb.DMatrix(X_test, np.log(y_test + 1))
# dtest = xgb.DMatrix(X_test)
# watchlist = [(dvalid, 'eval'), (dtrain, 'train')]
# gbm = xgb.train(params, dtrain, num_trees, evals=watchlist, early_stopping_rounds=50, feval=rmspe_xg, verbose_eval=True)
#
# print("Validating")
# train_probs = gbm.predict(xgb.DMatrix(X_test))
# indices = train_probs < 0
# train_probs[indices] = 0
# error = rmspe(np.exp(train_probs) - 1, X_test['Sales'].values)
# print('error', error)
#
# print("Make predictions on the test set")
# test_probs = gbm.predict(xgb.DMatrix(X_test))
# indices = test_probs < 0
# test_probs[indices] = 0
# submission = pd.DataFrame({"Id": test["Id"], "Sales": np.exp(test_probs) - 1})
# submission.to_csv("xgboost_kscript_submission.csv", index=False)


#I finished my code then realised that It's not for Python :cryingRiver:

import sys

def ToISO(a):
	year=1980
	month=0
	day=0
	for i in reversed(range(0,7)):
		if a[i]=='1':
			year+=2**(6-i)
	for i in reversed(range(7,11)):
		if a[i]=='1':
			month+=2**(10-i)
	for i in reversed(range(11,16)):
		if a[i]=='1':
			day+=2**(15-i)
	if year>=1980 and year<2107 and day>0 and day<=31 and month>0 and month<=12:
		print('%s-%s-%s' % (year, month, day))
	else:
		print('ERROR')

def ToDOSFAT(a):
	year=list(map(int,date[0:4]))
	month=list(map(int,date[5:7]))
	day=list(map(int,date[8:10]))
	year=year[0]*1000+year[1]*100+year[2]*10+year[3]
	month=month[0]*10+month[1]
	day=day[0]*10+day[1]
	if year>=1980 and year<2107 and day>0 and day<=31 and month>0 and month<=12:
		year-=1980
		year='{0:07b}'.format(year)
		month='{0:04b}'.format(month)
		day='{0:05b}'.format(day)
		year+=month
		year+=day
		print(year)
	else:
		print('ERROR')

try:
	date=list(map(str, input()))
	if(len(date)!=16 or len(date)!=10):
		print('ERROR')
	elif(len(date)==16):
		ToISO(date)
	else:
		ToDOSFAT(date)

except:
	sys.exit(0)


# Advent of Code 2020 Day 1 Challenge

import sys

# This function is used to find two numbers in a list which add to a given sum using a hash map.
# Reference: https://www.geeksforgeeks.org/given-an-array-a-and-a-number-x-check-for-pair-in-a-with-sum-as-x/
def sum_search_2(sum_list, sum_val):
	# Create an empty hash set:
	s = set()
	# Iterate over the list, adding each value to the hash map:
	for i in range(0, len(sum_list)):
		# Find the difference between the desired sum value and the current i-th position value in the list:
		temp = sum_val - sum_list[i]
		# If the difference (temp) is in the hash map, these are the two solution values. Print them:
		if (temp in s):
			print("Solution is: ({},{})".format(sum_list[i], temp))
		# Add each newly seen value to the hash map:
		s.add(sum_list[i])

# This function is used to find three numbers in a list which add to a given sum using hash map.
# Reference: https://www.geeksforgeeks.org/find-a-triplet-that-sum-to-a-given-value/
def sum_search_3(sum_list, sum_val):
	# Iterate over the list, recursively:
	for i in range(0, len(sum_list) - 1):
		# Create an empty hash set:
		s = set()
		# Find the difference between the desired sum value and the current i-th position value in the list:
		temp = sum_val - sum_list[i]
		# Second iteration, adding each value to the hash map:
		for j in range(i + 1, len(sum_list)):
			# If the current difference (temp) minus the current j-th position value in the list is in the hash map, solution is found. Print it:
			if (temp - sum_list[j]) in s:
				print("Solution is: ({},{},{})".format(sum_list[i], sum_list[j], temp-sum_list[j]))
			# Add each newly seen value to the hash map:
			s.add(sum_list[j])

def main():
	# Create input array:
	input_list = []
	# Read file input and apend each line to a list:
	with open(sys.argv[1], 'r') as f:
		for line in f:
			input_list.append(int(line.strip()))
	f.close()
	# Execute defined functions:
	sum_search_2(input_list, 2020)
	sum_search_3(input_list, 2020)

if __name__ == "__main__":
	main()


from django.shortcuts import render, redirect
from django.contrib.auth.forms import UserCreationForm
from django.contrib import messages
from django.contrib.auth import logout as auth_logout
from users.forms import UserForm , UserUpdateForm
from django.contrib.auth.models import User
from django.contrib.auth.decorators import login_required

def signup(request):
    if request.method =='POST':
        form = UserForm(request.POST)
        messages.error(request,str(form.errors))
        if form.is_valid():
            form.save()
            username = form.cleaned_data.get('username')
            messages.success(request,f'Your account is created! You can now log in')
            return redirect('login')
    else:
        form = UserForm()
    return render(request,'users/signup.html', {'form':form})

def logout(request):
    auth_logout(request)
    return render(request,'users/logout.html')

def userSettings(request):
    if request.method =='POST':
        u_form = UserUpdateForm(request.POST, instance=request.user)
        if u_form.is_valid():
            u_form.save()
            messages.success(request,f'Your Profile has been updated!')
            return redirect('settings')
    else:
        u_form = UserUpdateForm(instance=request.user)

    context = {
        'u_form': u_form,
    }
    return render(request,'users/userSettings.html',context)

def profile(request):
    return render(request,'users/profile.html')

def deleteAccount(request):
    if request.method =='POST':
        user=User.objects.get(username=request.user.username)
        print(user)
        user.delete()
        return redirect('login')
    return render(request,'users/deleteAccount.html')


#Below example showing use of instance, class and static method and variables

class foodMenu:
    '''This class prints food items ordered by customer.'''
    appName = 'Eatery'                        #Static Variable defined here
    def __init__(self,itemtype,name,dessert): #constructor of the class
        self.itemtype = itemtype              #instance variables defined here
        self.name = name
        self.dessert = dessert

    def myOrder(self):                        #It is instance method as it is accessing instance variables
        print("It is: ", self.itemtype)
        print("Order of: ", self.name)
        print("Want this dessert:", self.dessert)

    @classmethod                             #It is class method using static variable of the class
    def myApp(cls):
        print("Welcome to our famous food joint:", cls.appName)

    @staticmethod                           #It is static method not using any of static or instance variable of class
    def tips(t):
        print('Thank you for the tip $$ of', t)

c1 = foodMenu('Veg','Veg Noodles', 'Vanilla icecream')
c1.myOrder()
c1.myApp()
c1.tips(10)
c2 = foodMenu('Non Veg', 'Fried Chicken','fruit cake')
c2.myOrder()


from aiogram.dispatcher import FSMContext
from aiogram.dispatcher.filters.state import StatesGroup, State
from aiogram import types, Dispatcher
from create_bot import dp
from aiogram.types import KeyboardButton, InlineKeyboardButton, ReplyKeyboardRemove
from dbase_bot import database
from aiogram.dispatcher.filters import Text

class FsmPrizes(StatesGroup):
    prizes = State()

async def prizes(message: types.Message):
    keyboard = types.ReplyKeyboardMarkup(resize_keyboard=True).add(
    KeyboardButton('Ежемесячные призы!')).add(
    KeyboardButton('Годовые призы!'))
    await FsmPrizes.prizes.set()
    await message.answer('Мы разыгрываем призы каждый месяц! А также супер-призы среди всех отзывов в конце года!',
                         reply_markup=keyboard)

async def month_prizes(message: types.Message):
    keyboard = types.ReplyKeyboardMarkup(resize_keyboard=True).add(
    KeyboardButton('Годовые призы!')).add(
    KeyboardButton('Отмена'))
    await message.answer('Первый приз: 2000 \u20BD\n'
                         'Второй приз: 1000 \u20BD\n'
                         'Третий приз: 500 \u20BD', reply_markup=keyboard)

async def years_prizes(message: types.Message):
    get_prizes = await database.view_prizes()
    keyboard = types.ReplyKeyboardMarkup(resize_keyboard=True).add(
    KeyboardButton('Ежемесячные призы!')).add(
    KeyboardButton('Отмена'))
    await message.answer('В конце года разыграем следующие призы:', reply_markup=keyboard)
    for i in get_prizes:
        await message.answer_photo(i["pics"], f'{i["description"]}')

def register_handlers_prizes(dp: Dispatcher):
    dp.register_message_handler(prizes, lambda message: 'Разыгрываемые' in message.text, state=None)
    dp.register_message_handler(prizes, lambda message: 'Разыгрываемые' in message.text, state='*')
    dp.register_message_handler(month_prizes, lambda message: 'Ежемесячные' in message.text, state=FsmPrizes.prizes)
    dp.register_message_handler(years_prizes, lambda message: 'Годовые' in message.text, state=FsmPrizes.prizes)


#!/usr/bin/env python
# coding: utf-8

import re
import argparse
import pickle
import json
import os
from datetime import datetime, timedelta

import boto3
import numpy as np
import pandas as pd
from  tqdm import tqdm
import editdistance
from sklearn.metrics.pairwise import cosine_similarity

from autolocal.aws import aws_config
from autolocal.documentdb import S3DocumentManager

from autolocal import AUTOLOCAL_HOME

def single_vector_per_doc(vectors):
    # vectors is a list of np arrays where:
    # dims: (LAYERS(3), TOKENS(varies), DIMENSIONS(1024))
    """
    https://towardsdatascience.com/elmo-helps-to-further-improve-your-word-embeddings-c6ed2c9df95f
    In the ELMo paper, there are 3 layers of word embedding,
    layer zero is the character-based context independent layer,
    followed by two Bi-LSTM layers. The authors have empirically
    shown that the word vectors generated from the first Bi-LSTM
    layer can better capture the syntax, and the second layer can
    capture the semantics better.
    """
    vectors = np.concatenate([v[2] for v in vectors], 0)
    return vectors

def read_metadata(args):
    # start_date = [int(d) for d in args.start_date.split("-")]
    # end_date = [int(d) for d in args.end_date.split("-")]
    # for year in range(start_date[0], end_date[0]):
    #     for month in range(start_date[1], end_date[1]):
    #         for day in range(start_date[2], end_date[2]):
    table = boto3.resource(
        'dynamodb',
        region_name=aws_config.region_name,
        ).Table(aws_config.db_document_table_name)
    s3_client = boto3.client(
        's3',
        region_name=aws_config.region_name
        )

    response = table.scan()
    data = response['Items']

    while 'LastEvaluatedKey' in response:
        response = table.scan(ExclusiveStartKey=response['LastEvaluatedKey'])
        data.extend(response['Items'])

    metadata = pd.DataFrame(data)
    metadata["date"] = [datetime.strptime(d, '%Y-%m-%d') for d in metadata["date"]]
    return metadata

def sentence_split(s):
    sentences = re.split('[.\n!?"\f]', s)
    return [s for s in sentences if len(s.strip())>0]

def tokenize(s):
    tokens = re.findall(r'\w+', s)
    return tokens

def read_queries():
    table = boto3.resource(
        'dynamodb',
        region_name=aws_config.region_name,
        ).Table(aws_config.db_query_table_name)
    queries = table.scan()["Items"]
    return queries

def parse_dates(start_date, end_date):
    starting_dates_for_filtering = {
        'upcoming_only': datetime.now(),
        'upcoming': datetime.now(),
        'this_week': datetime.now() - timedelta(weeks=1),
        'this_year': datetime.now() - timedelta(days=365),
        'this_month': datetime.now() - timedelta(weeks=5),
        'past_six_months':datetime.now() - timedelta(days=183),
        'last_six_months':datetime.now() - timedelta(days=183),
        'past': None,
        'all': None
    }
    if start_date:
        start_date = datetime.strptime(start_date, '%Y-%m-%d')
    else:
        start_date = starting_dates_for_filtering['upcoming']
    if end_date:
        end_date = datetime.strptime(end_date, '%Y-%m-%d')
    if start_date and end_date:
      assert(start_date <= end_date)
    return start_date, end_date

def find_relevant_doc_ids(queries, metadata, start_date=None, end_date=None, agenda_only=False):

    cities = set()

    # filter metadata to only those files that match the query municipality and time_window
    for query in queries:
        cities.update(query["municipalities"])

    potential_documents = metadata

    print("start", start_date)
    print("end", end_date)
    potential_documents = potential_documents[potential_documents["date"] >= start_date]
    if end_date:
        potential_documents = potential_documents[potential_documents["date"] <= end_date]

    potential_documents = potential_documents[[(c in cities) for c in potential_documents["city"]]]

    if agenda_only:
        potential_documents = potential_documents[potential_documents["doc_type"]=="Agenda"]

    return list(potential_documents['doc_id'])

def read_docs(doc_ids, document_manager):
  log_every = 100

  documents = {}
  n_docs_total = len(doc_ids)

  i = 0
  n_docs_read = 0
  for doc_id in doc_ids:
    doc = document_manager.get_doc_by_id(doc_id)
    try:
      doc_string = document_manager.get_doc_text(doc)
    except Exception as e:
      print("could not read document {}".format(doc_id))
      print(e)
      print("adding document")
      document_manager.add_doc(doc)
      print("document added")
      try:
        print("reading document {}".format(doc_id))
        doc_string = document_manager.get_doc_text(doc)
        print("document read")
      except:
        doc_string = None
    if doc_string:
      vector_data = document_manager.get_doc_vectors(doc)
      if vector_data:
        documents[doc_id] = {
          "original_text": doc_string,
          "sentences": vector_data["sentences"],
          "vectors": vector_data["vectors"]
        }
    n_docs_read+=1
    if n_docs_read % log_every == 0:
        print("{} of {} documents read".format(n_docs_read, n_docs_total))

  return documents

def select_relevant_docs(municipalities, all_docs, metadata, start_date=None, end_date=None):
    # filter metadata to only those files that match the query municipality and time_window
    potential_documents = metadata
    start_date, end_date = parse_dates(start_date, end_date)
    if start_date:
        potential_documents = potential_documents[potential_documents["date"] >= start_date]
    if end_date:
        potential_documents = potential_documents[potential_documents["date"] >= end_date]
    potential_documents = potential_documents[[(c in municipalities) for c in potential_documents["city"]]]
    # filter all docs to only filenames in subset of metadata
    doc_ids = list(potential_documents['doc_id'])
    urls = list(potential_documents['url'])
    docs_to_return = []
    for i in range(len(doc_ids)):
        f = doc_ids[i]
        u = urls[i]
        if f in all_docs:
            docs_to_return.append({**all_docs[f], 'doc_id':f, 'url':u})
        else:
            print("file was not collected in 'all_docs': {}".format(f))
    # return [{**all_docs[f], 'filename':f, 'url':"example.com"} for f in potential_documents['local_path_txt'] if f in all_docs]
    return docs_to_return

def segment_docs(relevant_docs):
    min_section_length = 50 # tokens
    # TODO: this cuts of end of doc

    sections = []
    for doc in relevant_docs:
        original_text = doc["original_text"]
        pages = original_text.split('\f')
        page_numbers = []
        for p, page in enumerate(pages):
            page_sentences = sentence_split(page)
            # for each sentence, what page was it on?
            for sentence in page_sentences:
                sentence_tokens = tokenize(sentence)
                page_numbers.append(p+1)
        doc_sentences = doc["sentences"]
        doc_vectors = doc["vectors"]
        section = []
        section_tokens = 0
        if (len(doc_sentences) == len(doc_vectors)):
            for i in range(len(doc_sentences)):
                sentence = doc_sentences[i]
                page = page_numbers[i]
                sentence_vectors = doc_vectors[i]
                sentence_tokens = tokenize(sentence)
                section.append({
                    "sentence": sentence,
                    "page": page,
                    "sentence_vectors": sentence_vectors,
                    "sentence_tokens": sentence_tokens
                })
                section_tokens += len(sentence_tokens)
                if section_tokens >= min_section_length:
                    section_text = ". ".join([s["sentence"].strip() for s in section])
                    sections.append({
                        "sentences": section,
                        "section_text": section_text,
                        "doc_id": doc["doc_id"],
                        "url": doc["url"]
                    })
                    section = []
                    section_tokens = 0
    return sections

def set_casing(x, casing="lower_non_acronyms"):
    if casing=="cased":
        return x
    elif casing=="lower":
        return x.lower()
    elif casing=="lower_non_acronyms":
        return (x if x.isupper() else x.lower())
    else:
        raise Exception

# TODO: use vectors to find closes words to keywords
def score_doc_sections(doc_sections, orig_keywords, vectorizer):
    orig_keywords = [k.strip() for k in orig_keywords]
    keywords = []
    for k in orig_keywords:
        words = k.split(" ")
        for word in words:
            keywords.append(word)
    vector_data = vectorizer.vectorize(" ".join(keywords))
    keyword_vectors = single_vector_per_doc(vector_data["vectors"])
    doc_sections_scores = []
    for s, section in enumerate(doc_sections):
        section_vectors = single_vector_per_doc([s["sentence_vectors"] for s in section["sentences"]])
        section_text = section['section_text']
        no_keywords_found = True
        for k in orig_keywords:
            if bool(re.search("([^\w]|^)" + k + "([^\w]|$)", section_text)):
            # if k in section_text:
            #     TODO: consider casing
            # TODO: lemmatize!
                no_keywords_found = False
        for k in orig_keywords:
            if k.islower():
                if bool(re.search("([^\w]|^)" + k + "([^\w]|$)", section_text.lower())):
                    no_keywords_found = False
        if no_keywords_found:
            score = 0
        elif section_vectors.shape[0]>0:
            similarities = cosine_similarity(section_vectors, keyword_vectors)
            # if threshold_similarity > -1:
            #     similarities = similarities*(similarities>threshold_similarity)
            keyword_similarities = np.mean(similarities, axis=0)
            # score = np.sum(keyword_similarities*keyword_weights)
            score = np.mean(keyword_similarities)
        else:
            score = 0
        doc_sections_scores.append(score)

    return doc_sections_scores

def text_is_too_similar(a, b):
    # if there are only 2 edits to get from one text to the other, it's not good
    return editdistance.eval(a, b) < 50

# make sure we're not giving similar text among the top k (e.g. shows up on both minutes and agenda)
def check_repeated_text(top_k, section_text):
    for old in top_k:
        if text_is_too_similar(old[1]["section_text"], section_text):
            return True
    return False

def select_top_k(doc_sections, doc_sections_scores, k):
    sorted_sections = sorted(zip(doc_sections_scores, doc_sections), key=lambda pair: pair[0], reverse=True)
    top_k = []
    text_returned = []
    for x in sorted_sections:
        score = x[0]
        if score > 0:
            doc_id = x[1]["doc_id"]
            starting_page = x[1]["sentences"][0]["page"]
            ending_page = x[1]["sentences"][-1]["page"]
            section_text = x[1]["section_text"]
            if check_repeated_text(top_k, section_text):
                print("this excpert has already been returned")
            else:
                top_k.append(x)
            if len(top_k) >= k:
                break
    return top_k

def update_with_top_k(results, top_k_sections, query):
    for section in top_k_sections:
        x = section[1]
        x.update(query)
        results.append(x)
    return results

def write_results(results, query_id, batch):

    results_to_return = []
    for result in results:
        new_result = {
            "start_page": result["sentences"][0]["page"],
            'doc_id': result['doc_id'],
            'section_text': result['section_text']
        }
        results_to_return.append(new_result)

    print({
            'id': "{}_{}".format(query_id, datetime.now()),
            'email_address': result['email_address'],
            'query_id': result['id'],
            'recommendations': results_to_return,
            'keywords': result['keywords'],
            'municipalities': result['municipalities']
        })

    batch.put_item(
       Item={
            'id': "{}_{}".format(query_id, datetime.now()),
            'email_address': result['email_address'],
            'query_id': result['id'],
            'recommendations': results_to_return,
            'keywords': result['keywords'],
            'municipalities': result['municipalities'],
            'time': "{}".format(datetime.now())
        }
    )

def run_queries(document_manager, input_args):
    k = input_args.k
    start_date, end_date = parse_dates(input_args.start_date, input_args.end_date)
    print("reading queries")
    queries = read_queries()
    desired_subscription_status = input_args.query_type
    queries = [q for q in queries if ('subscription_status' in q and q['subscription_status'] == desired_subscription_status)]
    if input_args.filter:
        queries = [q for q in queries if ('email_address' == input_args.filter)]
    print("reading metadata")
    metadata = read_metadata(input_args)
    print("finding relevant doc_ids")
    relevant_doc_ids = find_relevant_doc_ids(queries, metadata, start_date = start_date, end_date = end_date, agenda_only=input_args.agenda_only)
    print("reading {} relevant documents".format(len(relevant_doc_ids)))
    # (not actually *all*, but all the ones we care about for queries)
    all_docs = read_docs(relevant_doc_ids, document_manager)
    print("read {} relevant documents".format(len(all_docs)))

    for q, orig_query in enumerate(queries):
        query = orig_query
        results = []
        print("running query {} of {}".format(q, len(queries)))
        query_id = query["id"]
        if input_args.emails != "production":
            query["email_address"] = input_args.emails
        email_address = query["email_address"]
        print("email address: {}".format(email_address))
        keywords = query["keywords"]
        print("keywords: {}".format(keywords))
        municipalities = query["municipalities"]
        print("municipalities: {}".format(municipalities))
        relevant_docs = select_relevant_docs(municipalities, all_docs, metadata, input_args.start_date, input_args.end_date)
        print("{} relevant documents identified for this query".format(len(relevant_docs)))
        print("segmenting documents")
        doc_sections = segment_docs(relevant_docs)
        print("{} document sections to choose from".format(len(doc_sections)))
        print("scoring documents")
        doc_sections_scores = score_doc_sections(
            doc_sections,
            keywords,
            document_manager.vectorizer
        )
        top_k_sections = select_top_k(doc_sections, doc_sections_scores, k)
        results = update_with_top_k(results, top_k_sections, query)
        #print(results)
        if len(results) == 0:
            print("no results found")
        else:
            print("sending email")
            # update dynamo db table
            table = boto3.resource(
                'dynamodb',
                region_name=aws_config.region_name,
                ).Table(aws_config.db_recommendation_table_name)
            with table.batch_writer() as batch:
                write_results(results, query_id, batch)

        # DEBUG

    # print("sending emails")
    # this will be handled by dynamodb/lambda function
    # send_emails(results, args)
    print("finished")

if __name__=='__main__':

    parser = argparse.ArgumentParser(description='Section documents and rank by relevance to queries')

    # parser.add_argument('--email', type=str, required=True,
    #     help='Who should I send emails to?\nUse `--email P` to send to the actual addresses in the queries.')

    parser.add_argument('--start_date', type=str, default=None,
        # TODO implement this
        # help="".join([
        #     'Documents have dates associated with them. ',
        #     'What is the earliest date of the documents that we should return?\n',
        #     'Format is YYYY-MM-DD, e.g. 2019-12-4 for December 4, 2019.\n',
        #     'Default is to include Minutes from meetings that took place this ',
        #     'past week and Agendas for upcoming meetings'
        # ]))
        help="".join([
            'Documents have dates associated with them. ',
            'What is the earliest date of the documents that we should return?\n',
            'Format is YYYY-MM-DD, e.g. 2019-12-4 for December 4, 2019.\n',
            'Default is to include only upcoming meetings'
        ]))

    parser.add_argument('--end_date', type=str, default=None,
        help="".join([
            'Documents have dates associated with them. ',
            'What is the latest date of the documents that we should return?\n',
            'Format is YYYY-MM-DD, e.g. 2019-12-4 for December 4, 2019.\n',
            'Default is to include *all* documents after the start date.'
        ]))

    parser.add_argument('--k', type=int, default=5,
        help="".join([
            'We will return the top k results. Default is k=5.'
        ]))

    parser.add_argument('--agenda_only', action='store_true',
        help="".join([
            "If this flag is included, return only agenda documents, no minutes."
        ]))

    parser.add_argument('--query_type', type=str, default='testing',
        help="".join([
            "Queries have a type, e.g. 'subscribed'. By default, run only queries labeled 'testing'."
        ]))

    parser.add_argument('--emails', type=str, default=None,
        help="".join([
            'Send all results to this one email address. If the flag is set to "production", send the actual emails to the actual users'
        ]))

    parser.add_argument('--filter', type=str, default=None,
        help="".join([
            'Filter queries to only those with this email address set. If empty, run all queries of this type.'
        ]))

    args = parser.parse_args()
    print(args)

    document_manager = S3DocumentManager()

    run_queries(
        document_manager=document_manager,
        input_args=args
    )


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Date    : 2017-07-07 17:14:53
# @Author  : Wangmengcn (eclipse_sv@163.com)
# @Link    : https://eclipsesv.com
# @Version : $Id$

from other_app import app
from collector.basicinfo.CPanda import Panda
from collector.basicinfo.CQuanmin import Quanmin

@app.task
def save_panda_data():
    panda = Panda()
    panda.get_room_infos()

@app.task
def save_quanmin_data():
    quanmin = Quanmin()
    quanmin.get_room_infos()


# Given an array w of positive integers,
# where w[i] describes the weight of index i,
# write a function pickIndex which randomly picks an index in proportion to its weight.

# Note:
# 1 <= w.length <= 10000
# 1 <= w[i] <= 10^5
# pickIndex will be called at most 10000 times.

# Example 1:
# Input:
# ["Solution","pickIndex"]
# [[[1]],[]]
# Output: [null,0]

# Example 2:
# Input:
# ["Solution","pickIndex","pickIndex","pickIndex","pickIndex","pickIndex"]
# [[[1,3]],[],[],[],[],[]]
# Output: [null,0,1,1,1,0]

# Explanation of Input Syntax:
# The input is two lists: the subroutines called and their arguments.
# Solution's constructor has one argument,
# the array w. pickIndex has no arguments.
# Arguments are always wrapped with a list, even if there aren't any.

# M1. 顺序查找
from ramdom import random
class Solution:

    def __init__(self, w: List[int]):

        self.prefix_sums = []
        prefix_sum = 0
        for weight in w:
            prefix_sum += weight
            self.prefix_sums.append(prefix_sum)
        self.total_sum = prefix_sum

    def pickIndex(self) -> int:

        target = self.total_sum * random.random()
        for i, prefix_sum in enumerate(self.prefix_sums):
            if target < prefix_sum:
                return i

# Your Solution object will be instantiated and called as such:
# obj = Solution(w)
# param_1 = obj.pickIndex()

# M2. 折半查找
class Solution:

    def __init__(self, w: List[int]):
        self.prefix_sums = []
        prefix_sum = 0
        for weight in w:
            prefix_sum += weight
            self.prefix_sums.append(prefix_sum)
        self.total_sum = prefix_sum

    def pickIndex(self) -> int:
        target = self.total_sum * random.random()
        low, high = 0, len(self.prefix_sums)
        while low < high:
            mid = low + high >> 1
            if target > self.prefix_sums[mid]:
                low = mid + 1
            else:
                high = mid
        return low

# Your Solution object will be instantiated and called as such:
# obj = Solution(w)
# param_1 = obj.pickIndex()

LeetCode/Python3/Design/528. Random Pick with Weight.py


# Downloadable pre-built toolchains

load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
load(":name-version-sha.bzl", "preferred_release")

def deps():
    """
    Convenience array/list of `http_archive` definitions suitable for a WORKSPACE but allows
    changes to the list of arch binaries to be self-contained in the toolchain dir.

    Returns a list of http_archive similar to:

    http_archive(
        name = "git_town_darwin_amd64",

        sha256 = "...",
        urls = [ "..." ],
    )

    http_archive(
        name = "git_town_darwin_arm64",
        ...
    )
    """

    p = preferred_release()
    for a in p:
        name = a.replace("-", "_")
        http_archive(name = name, build_file_content = "exports_files(['git-town'])", urls = [p[a]["url"]], sha256 = p[a]["sha256"])


import pandas as pd
import glob
from concurrent.futures import ThreadPoolExecutor
import os

def edit_acts(ticker):
    print(ticker)
    quote = pd.read_csv(glob.glob(ticker+'/*-quote.csv')[0], index_col=0)

    date = glob.glob(ticker+'/*')[0][25:33]
    date_dash = date[:4]+'-'+date[4:6]+'-'+date[6:]

    quote_start = quote.index.get_loc(date_dash + ' 09:06:00')
    quote_end = quote.index.get_loc(date_dash + ' 15:00:00')

    print(quote_start)
    print(quote_end)

    action = pd.read_csv('action_list/actions0-'+str(ticker[9:])+'-'+date+'.csv', index_col=0)
    action = action[quote_start:quote_end]
    action.index = range(quote_end-quote_start)

    action.to_csv('expert_actions/action'+str(ticker[9:])+'.csv')
    print('expert_actions/action'+str(ticker)+'.csv: done')

if __name__ == '__main__':

    tickers = []

    for ticker in glob.glob('../data1/*'):
        tickers.append(ticker)

    with ThreadPoolExecutor(max_workers=16) as Executor:
        jobs = [Executor.submit(edit_acts, t) for t in tickers]


import pandas as pd, json, pickle as pkl, numpy as np, pickle
from sklearn.preprocessing import MultiLabelBinarizer

with open("all_opportunities_df.pkl", 'rb') as infile:
    df = pkl.load(infile)

# Prefix Ages and manipulate and unify the ages group and fix an age type (9 to 6)
for i, el in enumerate(df.ages,1):
    for age in range(len(el)):
        #print('age', el[age])
        if el[age] != '':
            el[age] = "Ages_" + el[age]

print('result: ',df.ages)

# Unify REACH
df.reach = df.reach.apply(lambda x: 'Missouri' if x == ['Missouri', 'Illinois'] else x)
df.reach = df.reach.apply(lambda x: np.nan if x == [] else x)
df.reach = df.reach.apply(lambda x: str(x).strip("[]''"))
df.reach = df.reach.apply(lambda x: 'None' if x == '' else x)
df.reach = df.reach.apply(lambda x: np.nan if x == 'None' else x)

print(df.reach)
print(df.attention)
df.attention = df.attention.apply(lambda x: ['None'] if x == [] or x == None else x)
for i, row in enumerate(df.attention, 1):
    for element in range(len(row)):
        if row[element] == 'Gifted and talented students' or row[element] == 'Gifted and Talented':
            row[element] = "Att_" + "GTS"
        else:
            if row[element] =='Students with disabilities' or row[element] == 'Students with disabilites':
                row[element] = "Att_" + "SwDis"
            else:
                if row[element] == 'Students at risk from dropping out of school':
                    row[element] = "Att_" + "SRDoS"
                else:
                    row[element] = "Att_" + row[element]
print(df.attention)

# LANGUAGE unifying
df.language = df.language.apply(lambda x: ['None'] if x == [] or x == None else x)
for i, row in enumerate(df.language,1):
    for el in range(len(row)):
        row[el] = "L_" + row[el]

print(df.areaOfInterest)
# Replacing/abbreviating strings in ATTENTION
for i,row in enumerate(df.areaOfInterest,1):
    for el in range(len(row)):
        if (row[el] == 'Earth and Environmental Science' or row[el] == 'earth and environmental science' or row[el] == 'earth and Environmental Science'):
            row[el] = "AoI_EES"
        else:
            if (row[el] == 'Computer Science' or row[el] =='computer science'):
                row[el] = "AoI_CS"
            else:
                if (row[el] == 'Coding/Programming'):
                    row[el] = "AoI_" + 'Coding/Programming'
                else:
                    if (row[el] == 'General Science'):
                        row[el] = "AoI_GS"
                    else:
                        if (row[el] == 'Architecture and Design'):
                            row[el] = "AoI_AaD"
                        else:
                            if (row[el] == 'Space Science/Aerospace/Rocketry'):
                                row[el] = "AoI_SSAR"
                            else:
                                row[el] = "AoI_" + row[el]
print(df.areaOfInterest)

# TYPEOFOPPORTUNITY stripping the list brackets
df.typeOfOpportunity = df.typeOfOpportunity.apply(lambda x: str(x).strip("[]'"))

# run the OHE on AGES, ATTENTION, LANGUAGE, AREAOFINTEREST
mlb = MultiLabelBinarizer()
variables = [df.ages,df.attention, df.language, df.areaOfInterest]
for el in variables:
     df = df.join(pd.DataFrame(mlb.fit_transform(el),
                          columns=mlb.classes_,
                          index=df.index))

#Varibales Checking
for n,el in enumerate(df.reach,1):
    print(n, el)
df.reach[df.reach.isna()]
df.attention[df.attention.isna()]

for n, el in enumerate(df.attention,1):
    print(n, el)
for n, el in enumerate(df.language,1):
    print(n, el)
for n, el in enumerate(df.areaOfInterest,1):
    print(n, el)
for n, el in enumerate(df.areaOfInterest,1):
    print(n, el)

# Check are there any locationCity's format is not correct
for i in range(1, len(df)):
    if df.locationCity[i][0].isdigit():
        print(df.locationCity[i])
        location = df.locationCity[i].split(",")
        print(location)
        print(len(location[1]))
        df.locationCity[i] = location[1][1:len(location[1])]
        print(df.locationCity[i])

#Store the data
with open("cleaned_data.pkl", "wb") as outfile:
    pickle.dump(df, outfile)

#df.to_csv("data/all_opportunities_df_cleaned.csv")
print(df)

import os

if os.path.isfile("all_opportunities_df_cleaned.csv") == True:
    old_dataframe = pd.read_csv("all_opportunities_df_cleaned.csv")
    old_dataframe = old_dataframe.drop(['Unnamed: 0'], axis=1)
    result = pd.concat([old_dataframe, df], ignore_index=True, sort=False)
    print(result)
    result.drop(columns=result.columns[-1], axis=1, inplace=True)
    print(result)
    result.to_csv("all_opportunities_df_cleaned.csv")
else:
    df.to_csv("all_opportunities_df_cleaned.csv")
#print(old)
#print(list(set(old)- set(new)))