import argparse
import numpy as np
import torch
import os
import sys
import time
import random
import IPython

from rlmamr.my_env.box_pushing_MA import BoxPushing_harder as BP_MA
from rlmamr.my_env.osd_ma_single_room import ObjSearchDelivery_v4 as OSD_S_4

from rlmamr.MA_cen_condi_ddrqn.team_cen_condi import Team_RNN
from rlmamr.MA_cen_condi_ddrqn.utils.memory_cen_condi import ReplayMemory_rand, ReplayMemory_epi
from rlmamr.MA_cen_condi_ddrqn.utils.utils import save_check_point
from rlmamr.MA_cen_condi_ddrqn.learning_methods_cen_condi import QLearn_squ_cen_condi_0

from IPython.core.debugger import set_trace

ENVIRONMENTS = {
        'BP_MA': BP_MA,
        'OSD_S_4': OSD_S_4
        }

QLearns = [QLearn_squ_cen_condi_0]

def train(env_name, grid_dim, obs_one_hot, target_flick_prob, agent_trans_noise, target_rand_move,
          h_speed_ps, tb_m_speed, tb_m_noisy, f_p_obj_tc, f_l_obj_tc, d_pen,
          env_terminate_step, n_env, n_agent, cen_mode, total_epi, replay_buffer_size, sample_epi,
          dynamic_h, init_h, end_h, h_stable_at, eps_l_d, eps_l_d_steps, eps_end, eps_e_d, softmax_explore, h_explore, db_step,
          optim, l_rate, discount, huber_l, g_clip, g_clip_v, g_clip_norm, g_clip_max_norm,
          start_train, train_freq, target_update_freq, trace_len, sub_trace_len, batch_size,
          sort_traj, rnn, mlp_layer_size, rnn_layer_num, rnn_h_size, gru, run_id,
          eval_freq, resume, save_ckpt, seed, save_dir, device, **kwargs):

    # define the name of the directory to be created
    os.makedirs("./performance/"+save_dir+"/train", exist_ok=True)
    os.makedirs("./performance/"+save_dir+"/test", exist_ok=True)
    os.makedirs("./performance/"+save_dir+"/check_point", exist_ok=True)
    os.makedirs("./policy_nns/"+save_dir, exist_ok=True)

    if seed is not None:
        seed = seed *10
        random.seed(seed)
        np.random.seed(seed)
        torch.manual_seed(seed)
    torch.set_num_threads(1)

    # create env
    ENV = ENVIRONMENTS[env_name]
    if env_name.startswith('OSD'):
        env = ENV(terminate_step=env_terminate_step,
                  human_speed_per_step=[h_speed_ps],
                  TB_move_speed=tb_m_speed,
                  TB_move_noisy=tb_m_noisy,
                  fetch_pass_obj_tc=f_p_obj_tc,
                  fetch_look_for_obj_tc=f_l_obj_tc,
                  delay_delivery_penalty=d_pen)
    else:
        env = ENV(tuple(grid_dim),terminate_step=env_terminate_step)

    # create replay buffer
    if sample_epi:
        memory = ReplayMemory_epi(env.n_agent, env.obs_size, batch_size, size=replay_buffer_size)
    else:
        memory = ReplayMemory_rand(env.n_agent, env.obs_size, trace_len, batch_size, size=replay_buffer_size)

    # collect hyper params:
    hyper_params = {'cen_mode': cen_mode,
                    'soft_action_selection': softmax_explore,
                    'epsilon_linear_decay': eps_l_d,
                    'epsilon_linear_decay_steps': eps_l_d_steps,
                    'epsilon_end': eps_end,
                    'h_explore': h_explore,
                    'epsilon_exp_decay': eps_e_d,
                    'dynamic_h': dynamic_h,
                    'hysteretic': (init_h, end_h),
                    'optimizer': optim,
                    'learning_rate': l_rate,
                    'discount': discount,
                    'huber_loss': huber_l,
                    'grad_clip': g_clip,
                    'grad_clip_value': g_clip_v,
                    'grad_clip_norm': g_clip_norm,
                    'grad_clip_max_norm': g_clip_max_norm,
                    'sample_epi': sample_epi,
                    'trace_len': trace_len,
                    'sub_trace_len': sub_trace_len,
                    'batch_size': batch_size,
                    'sort_traj': sort_traj,
                    'device': device}

    model_params = {'mlp_layer_size': mlp_layer_size,
                    'rnn_layer_num': rnn_layer_num,
                    'rnn_h_size': rnn_h_size,
                    'GRU': gru}

    # create team
    team = Team_RNN(env, n_env, memory, env.n_agent, QLearns[cen_mode], h_stable_at,
            save_dir=save_dir, nn_model_params=model_params, eval_freq=eval_freq, **hyper_params)

    t = time.time()
    training_count=0
    target_updating_count = 0

    step = 0

    # continue training using the lastest check point
    if resume:
        team.load_check_point(run_id)
        step = team.step_count

    while team.episode_count < total_epi:
        team.step(run_id)
        if (not step % train_freq) and team.episode_count >= start_train:
            # update hysteretic learning rate
            if db_step:
                team.update_hysteretic(step)
            else:
                team.update_hysteretic(team.episode_count)

            for _ in range(n_env):
                team.train()

            # update epsilon
            if db_step:
                team.update_epsilon(step)
            else:
                team.update_epsilon(team.episode_count)

            training_count += 1
            #save_check_point(team.agents, step, team.hysteretic, team.epsilon, save_dir)
        if not step % target_update_freq:
            team.update_target_net()
            if (time.time()-t) / 3600 >= 23.0 and save_ckpt:
                save_check_point(team.cen_controller, step, team.episode_count, team.hysteretic, team.epsilon, save_dir, team.memory, run_id, team.TEST_PERFORM)
                sys.exit()
            target_updating_count += 1
            print('[{}]run, [{:.1f}K] took {:.3f}hr to finish {} episodes {} trainning and {} target_net updating (eps={})'.format(
                    run_id, step/1000, (time.time()-t)/3600, team.episode_count, training_count, target_updating_count, team.epsilon), flush=True)
        step += 1

    if team.episode_count == total_epi and save_ckpt:
        save_check_point(team.cen_controller, step, team.episode_count, team.hysteretic, team.epsilon, save_dir, team.memory, run_id, team.TEST_PERFORM)
        print("Finish entire training ... ")
    team.envs_runner.close()

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--env_name',           action='store',        type=str,             default='OSD_S_4',     help='Domain name')
    parser.add_argument('--env_terminate_step', action='store',        type=int,             default=150,           help='Maximal steps for termination')
    parser.add_argument('--grid_dim',           action='store',        type=int,  nargs=2,   default=[6,6],         help='Grid world size')
    parser.add_argument('--obs_one_hot',        action='store_true',                                                help='Whether represents observation as a one-hot vector')
    parser.add_argument('--target_flick_prob',  action='store',        type=float,           default=0.3,           help='Probability of not observing target in Capture Target domain')
    parser.add_argument('--agent_trans_noise',  action='store',        type=float,           default=0.1,           help='Agent dynamics noise in Capture Target domain')
    parser.add_argument('--target_rand_move',   action='store_true',                                                help='Whether apply target random movement')
    parser.add_argument('--n_env',              action='store',        type=int,             default=1,             help='Number of envs running in parallel')
    parser.add_argument('--n_agent',            action='store',        type=int,             default=2,             help='Number of agents')
    parser.add_argument('--cen_mode',           action='store',        type=int,             default=0,             help='Index of learning algorithm')

    #OSD params
    parser.add_argument('--h_speed_ps',         action='store',        type=int,  nargs='+', default=[18],          help='Time-step cost for each work step of human task')
    parser.add_argument('--tb_m_speed',         action='store',        type=float,           default=0.6,           help='Turtlebot move speed m/s')
    parser.add_argument('--tb_m_noisy',         action='store',        type=float,           default=0.0,           help='Turtlebot dynamics noise')
    parser.add_argument('--f_p_obj_tc',         action='store',        type=int,             default=4,             help='Time-step cost for finishing macro-action Pass_obj by Fetch robot')
    parser.add_argument('--f_l_obj_tc',         action='store',        type=int,             default=6,             help='Time-step cost for finishing macro-action Look_for_obj by Fetch robot')
    parser.add_argument('--d_pen',              action='store_true',                                                help='Whether apply penatly for delayed tool delivery')

    parser.add_argument('--total_epi',          action='store',        type=int,             default=40*1000,       help='Number of training episodes')
    parser.add_argument('--replay_buffer_size', action='store',        type=int,             default=1000,          help='Number of episodes/sequences in replay buffer')
    parser.add_argument('--sample_epi',         action='store_true',                                                help='Whether use full-episode-based replay buffer or not')
    parser.add_argument('--dynamic_h',          action='store_true',                                                help='Whether apply hysteritic learning rate decay or not')
    parser.add_argument('--init_h',             action='store',        type=float,           default=1.0,           help='Initial value of hysteretic learning rate')
    parser.add_argument('--end_h',              action='store',        type=float,           default=1.0,           help='Ending value of hysteretic learning rate')
    parser.add_argument('--h_stable_at',        action='store',        type=int,             default=4*1000,        help='Decaying period according to episodes/steps')

    parser.add_argument('--eps_l_d',            action='store_true',                                                help='Whether use epsilon linear decay for exploartion or not')
    parser.add_argument('--eps_l_d_steps',      action='store',        type=int,             default=4*1000,        help='Decaying period according to episodes/steps')
    parser.add_argument('--eps_end',            action='store',        type=float,           default=0.1,           help='Ending value of epsilon')
    parser.add_argument('--eps_e_d',            action='store_true',                                                help='Whether use episode-based epsilon linear decay or not')
    parser.add_argument('--softmax_explore',    action='store_true',                                                help='Whether apply softmac for exploration')
    parser.add_argument('--h_explore',          action='store_true',                                                help='whether use history-based policy for exploration or not')
    parser.add_argument('--db_step',            action='store_true',                                                help='Whether use step-based decaying manner or not')

    parser.add_argument('--optim',              action='store',         type=str,            default='Adam',        help='Optimizer')
    parser.add_argument('--l_rate',             action='store',         type=float,          default=0.0006,        help='Learning rate')
    parser.add_argument('--discount',           action='store',         type=float,          default=0.95,          help='Discount factor')
    parser.add_argument('--huber_l',            action='store_true',                                                help='Whether use huber loss or not')
    parser.add_argument('--g_clip',             action='store_true',                                                help='Whether use gradient clip or not')
    parser.add_argument('--g_clip_v',           action='store',         type=float,          default=0.0,           help='Absolute Value limitation for gradient clip')
    parser.add_argument('--g_clip_norm',        action='store_true',                                                help='Whether use norm-based gradient clip')
    parser.add_argument('--g_clip_max_norm',    action='store',         type=float,          default=0.0,           help='Norm limitation for gradient clip')

    parser.add_argument('--start_train',        action='store',         type=int,            default=2,             help='Training starts after a number of episodes')
    parser.add_argument('--train_freq',         action='store',         type=int,            default=30,            help='Updating performs every a number of steps')
    parser.add_argument('--target_update_freq', action='store',         type=int,            default=5000,          help='Updating target net every a number of steps')
    parser.add_argument('--trace_len',          action='store',         type=int,            default=10,            help='The length of each sequence saved in replay buffer when not using full-episode-based replay buffer')
    parser.add_argument('--sub_trace_len',      action='store',         type=int,            default=1,             help='Minimal length of a sequence for traning data filtering')
    parser.add_argument('--sort_traj',          action='store_true',                                                help='Whether sort sequences based on its valid length after squeezing experiences or not, redundant param for pytorch version later than 1.1.0')
    parser.add_argument('--batch_size',         action='store',         type=int,            default=16,            help='Number of episodes/sequences in a batch')

    parser.add_argument('--rnn',                action='store_true',                                                help='Whether using rnn-based agent')
    parser.add_argument('--mlp_layer_size',     action='store',         type=int,            default=32,            help='MLP layer dimension of decentralized policy-net')
    parser.add_argument('--rnn_layer_num',      action='store',         type=int,            default=1,             help='Number of RNN layers')
    parser.add_argument('--rnn_h_size',         action='store',         type=int,            default=64,            help='RNN hidden layer dimension')
    parser.add_argument('--gru',                action='store_true',                                                help='Whether use gru or not')

    parser.add_argument('--eval_freq',          action='store',         type=int,            default=100,           help='Pause training every 100 episodes for evaluation')
    parser.add_argument('--resume',             action='store_true',                                                help='Whether use saved ckpt to continue training or not')
    parser.add_argument('--save_ckpt',          action='store_true',                                                help='Whether save ckpt or not')
    parser.add_argument('--run_id',             action='store',         type=int,            default=0,             help='Index of a run')
    parser.add_argument('--seed',               action='store',         type=int,            default=None,          help='Random seed of a run')
    parser.add_argument('--save_dir',           action='store',         type=str,            default=None,          help='Directory name for storing trainning results')
    parser.add_argument('--device',             action='store',         type=str,            default='cpu',         help='Which device (CPU/GPU) to use.')

    params = vars(parser.parse_args())

    train(**params)

if __name__ == '__main__':
    main()


'''
*****************************************
*                                       *
* Unate Recursive Complement Calculator *
*                                       *
* Python SDK : 3.8.1                    *
* Author : Siddhi Pandare               *
* Version : 1.0.0                       *
*                                       *
*****************************************
'''

# Parses Input Array to a PCN list.
def Input_PCN ( file ):
    array =File_to_list (file) # Get Input Parsed into a List of Numbers.
    vars_in = array[0][0] # Numbers of Total Variables in Function.
    cubes_in = array[1][0] # Numbers of Total Cubes in Function.
    all_cubes = [  [ '11' for y in range(0, vars_in)]  for x in range(0, cubes_in) ]
    for x in range (2,len(array)):
        for y in range (1,len(array[x]) ):
            number = array[x][y]
            ins = '01'
            if number < 0:
                number = abs(number)
                ins = '10'
            all_cubes[x-2][number-1]=ins
    return all_cubes,cubes_in,vars_in

# Parses input File in List seperated by lines.
def File_to_list (file ):
    file=file.read()
    file=file.split("\n")
    list_inputs=[[int(i) for i in x.split(" ")  if i != ""] for x in file if x != '' ]
    return list_inputs

#Returns Shannon's Positive and Negative Cofactor
def Cofactor_Calculator(function,bin_var,type_c,vars_in):
    ps_c = []
    apped = 0
    bound = '01'
    if type_c == '1': # Positive.
        bound = '10'
    for x in range (len(function)):
        if not function[x][bin_var] == bound:
            ps_c.append([]) # Inster New Cube.
            for y in range (len(function[x])):
                ps_c[apped].append(function[x][y])
            ps_c[apped][bin_var] = '11'
            apped+=1
    n_t_p = 0
    for x in range (len(ps_c)):
        count = 0
        for y in range (vars_in):
            if ps_c[x][y] == '11' :
                 count+=1
        if count == vars_in:
            n_t_p = 1
            lis_t_p = x
    if n_t_p == 1 :
        rs_c = []
        rs_c.append([]) # Inster New Cube.
        rs_c[0] = ['11'] * vars_in
        return rs_c
    return ps_c

def Binate_Var ( function , vars_c ):
    Tru_Apps = [0] * vars_c
    Com_Apps = [0] * vars_c
    Binate = [0] * vars_c
    Binate_Flag = 0 # Initial flag for Binate Function is false.
    # Check for Binate Variables.
    for y in range ( vars_c ): # Parse all variables.
        for x in range ( len( function ) ): # On all functions.
            if function[x][y] == '01' :
                Tru_Apps[y]+=1
            elif function[x][y] == '10' :
                Com_Apps[y]+=1
            if Tru_Apps[y] > 0 and Com_Apps[y] > 0 : # Var Is Binate.
                Binate[y] = 1 # Update Binate vars table.
                Binate_Flag = 1 # Function is Binate.
    # If function is Binate, choose the Binate variable which apperas in the most cubes.
    if Binate_Flag == 1 :
        Binate_Apps = 0 # Threshold.
        Tie_Flag = 0 # Tie flag.
        for y in range ( vars_c ) :
            if Binate[y] == 1 : # Iff variable is Binate.
                Apps_Temp = Tru_Apps[y] + Com_Apps[y] # Count Binate Variable Xi Appearences.
                if  Binate_Apps < Apps_Temp :
                        Binate_Apps = Apps_Temp # Update Threshold.
                        Tie_Flag = 0 # Reset Tie Flag.
                elif Binate_Apps == Apps_Temp :
                        Tie_Flag = 1 # Invert Tie Flag.
        # Iff there is a tie on the above procedure.
        if  Tie_Flag == 1 :
            # Let T denote the number o cubes where this variable appears in true form.
            # Let C denote the number of cubes where this variable appears in complement form.
            # Choose the variable that has the smallest value of |T-C|.
            T_minus_C = [-1] * vars_c
            Ratio_Tie_Flag = 0 # Tie flag.
            for y in range ( vars_c ) :
                if Binate[y] == 1 :
                    if  Binate_Apps == Tru_Apps[y] + Com_Apps[y]  : # Count Binate Var Xi Appearences.
                        T_minus_C[ y ] = abs( Tru_Apps[y] - Com_Apps[y] ) # Get |T-C|.
                        Min_Ratio = T_minus_C[ y ]
            # Get Smaller |T-C| Ratio.
            for y in range ( vars_c ) :
                if  T_minus_C[y] < Min_Ratio and T_minus_C[y] >= 0 :
                    Min_Ratio = T_minus_C[y]
            # Look for Ratio Tie.
            Min_Ratio_Apps = 0
            for y in range ( vars_c ) :
                if ( T_minus_C[y] == Min_Ratio ):
                    Min_Ratio_Apps += 1
                    if Min_Ratio_Apps > 1:
                        Ratio_Tie_Flag = 1
                        break
            if  Ratio_Tie_Flag == 1 : # Again a Tie.
                # Choose the variable with the lowest index.
                for y in range (vars_c) :
                    if  T_minus_C[ y ] == Min_Ratio :
                        return y
            else: # Not a Ratio Tie.
                for y in range (vars_c) :
                    if  T_minus_C[ y ] == Min_Ratio :
                        return y
        else:
            for y in range (vars_c) :
                if Binate[y] == 1 :
                    Apps_Temp = Tru_Apps[y] + Com_Apps[y] # Count Binate Variable Xi Appearences.
                    if ( Binate_Apps == Apps_Temp ) :
                        return y
    else:
        # Choose the unate variable in the most cubes.
        # First Get Unate Table.
        Unate = [0] * vars_c
        for y in range (vars_c) :
            Unate[y] = Com_Apps[y]
            if  Tru_Apps[y] > 0 :
                Unate[y] = Tru_Apps[y]
        # Search for Most Unate Variables.
        Most_Unate = 0
        for y in range (vars_c) :
            if  Unate[y] > Most_Unate :
                Most_Unate = Unate[y]
        # Search if There are more than One.
        Unate_Apps = 0
        for y in range (vars_c) :
            if Unate[y] == Most_Unate :
                Unate_Apps+=1
        # Get Unate Tie */
        Unate_Tie_Flag = 0
        if  Unate_Apps > 1 :
            Unate_Tie_Flag = 1
        if Unate_Tie_Flag == 1 :
            # Choose the Variable with the lowest index.
            for y in range (vars_c) :
                if  Unate[y] == Most_Unate :
                    return y
        else :
            for y in range (vars_c) :
                if  Unate[y] == Most_Unate :
                    return y

#Complements a simple function straight away
def get_complement(function,vars_in):
    lst_complement=[["11" for i in range (0,vars_in)] for j in range(0,vars_in)]
    k=0
    counter=0
    for x in range(0,vars_in):
        if function[0][x]=="11":
            k+1
            continue
        elif function[0][x]=="01":
            lst_complement[k][x]="10"
            k=k+1
        elif function[0][x]=="10":
            lst_complement[k][x]="01"
            k=k+1

    val=["11"]*vars_in
    lst_complement=[x for x in lst_complement if x != val]
    return lst_complement

#ANDing of two functions
def PCN_AND ( function , bin_var , var_type ) :
    val = '01'
    if var_type == '0':
        val = '10'
    for x in range(len( function )):
        function[x][bin_var]  = val
    return function

#ORing of functions
def PCN_OR ( function_p , function_n  ) :
    function_ret = []
    function_ret.extend(function_p)
    function_ret.extend(function_n)
    return function_ret

#Function to write output into file
def printer(Out,vars_in,file):
    fh=open("Output"+file,"w")
    fh.write(str(vars_in)+"\n")
    fh.write(str(len(Out))+"\n")
    for x in Out:
        y=[i for i in x if i != "11"]
        fh.write(str(len(y))+" ")
        for i in range(0,len(y)):
            if y[i] == "01":
                fh.write(str(i+1))
            elif y[i] == "10":
                fh.write(str(-(i+1)))
            if i != len(y)-1:
                fh.write(" ")
        fh.write("\n")
    fh.close()

# # Calculation of complement by shannon's expression.
def Complement( function , vars_in ) :
    if  len( function ) == 0 :  # F has no cubes.
        return [[ '11' for y in range(0, vars_in)]]
    elif len( function ) == 1 : # F has Just 1 cube.
        return get_complement ( function , vars_in )
    else :  # Must Do recursion
        binate_var = Binate_Var ( function , vars_in )
        P_C = Cofactor_Calculator ( function , binate_var , '1' , vars_in )
        N_C = Cofactor_Calculator ( function , binate_var , '0' , vars_in )
        P_C = Complement( P_C , vars_in )
        N_C = Complement( N_C , vars_in )
        P_C = PCN_AND( P_C , binate_var , '1' )
        N_C = PCN_AND( N_C , binate_var , '0' )
    return PCN_OR( P_C, N_C )

#Main function
def main():
    print("-----------------------------------------")
    print("--UNATE RECURSIVE COMPLEMENT CALCULATOR--")
    print("-----------------------------------------")
    file= input("Enter File: ") #"part5.pcn"
    f_in=open(file,"r")
    # Parse Input Table
    Input_F,cubes_in,vars_in = Input_PCN ( f_in )
    Out = Complement(Input_F,vars_in)
    printer(Out,vars_in,file)
    print("--COMPLEMENTED BOOLEAN FUNCTION PCN IS ---")
    for x in Out:
        print(x)
    f_in.close()

main()


from django.http import HttpResponseRedirect
from base.models import PermUser, PermUserObj, PermGroup, UserPermStrategy

def rights_required(perm_code, obj_name=None, redirect_url='/base/login/'):
    """
    Decorator for views that checks whether a user has a service permission
    enabled, redirecting to the redirect page if necessary.
    """

    def wrapper(f):
        def decorate(request, *arg, **kwargs):
            if not request.user.is_authenticated:
                return HttpResponseRedirect(redirect_url)

            user = request.user

            # 超级用户
            if user.is_superuser:
                check_result = True
            # 公司职员
            elif user.is_staff:
                group = user.groups.first()
                check_result = PermGroup.has_perm(perm_code, group)
            # 客户账号
            else:
                if obj_name:
                    obj_id = request.POST.get('obj_id', '')
                    if not obj_id:
                        return HttpResponseRedirect(redirect_url)
                    check_result = PermUserObj.has_perm(
                        perm_code, obj_name, obj_id, user)
                else:
                    if user.is_parent:
                        check_result = PermUser.has_perm(perm_code, user)
                    elif user.is_child:
                        check_result = UserPermStrategy.has_perm(
                            perm_code, user)

            if check_result:
                return f(request, *arg, **kwargs)
            else:
                return HttpResponseRedirect(redirect_url)

        return decorate

    return wrapper


__author__ = 'alefur'
from PyQt5.QtWidgets import QGridLayout, QComboBox

from spsClient.widgets import Coordinates, ValueGB, ControlPanel, CommandsGB, CmdButton, DoubleSpinBoxGB, CustomedCmd, AbortButton

class CoordBoxes(QGridLayout):
    def __init__(self):
        QGridLayout.__init__(self)
        self.widgets = [DoubleSpinBoxGB('X', -10, 10, 5),
                        DoubleSpinBoxGB('Y', -10, 10, 5),
                        DoubleSpinBoxGB('Z', -10, 10, 5),
                        DoubleSpinBoxGB('U', -2, 2, 5),
                        DoubleSpinBoxGB('V', -2, 2, 5),
                        DoubleSpinBoxGB('W', -2, 2, 5)]

        for i, spinbox in enumerate(self.widgets):
            self.addWidget(spinbox, i // 3, i % 3)

class MoveCmd(CustomedCmd):
    def __init__(self, controlPanel):
        CustomedCmd.__init__(self, controlPanel=controlPanel, buttonLabel='MOVE')

        self.combo = QComboBox()
        self.combo.addItems(['absolute', 'relative'])
        self.combo.currentIndexChanged.connect(self.resetCoords)

        self.addWidget(self.combo, 0, 1)

    @property
    def spinboxes(self):
        return self.controlPanel.commands.coordBoxes.widgets

    def resetCoords(self, ind):
        if ind == 0:
            vals = [float(valueGB.value.text()) for valueGB in self.controlPanel.coordinates.widgets]
        else:
            vals = 6 * [0]

        for spinbox, val in zip(self.spinboxes, vals):
            spinbox.setValue(val)

    def buildCmd(self):
        labels = ['X', 'Y', 'Z', 'U', 'V', 'W']
        values = [spinbox.getValue() for spinbox in self.spinboxes]

        cmdStr = '%s slit move %s ' % (self.controlPanel.actorName, self.combo.currentText())
        cmdStr += (" ".join(['%s=%.5f' % (label, value) for label, value in zip(labels, values)]))

        return cmdStr

class SetRepCmd(CustomedCmd):
    def __init__(self, controlPanel):
        CustomedCmd.__init__(self, controlPanel=controlPanel, buttonLabel='SET')

        self.combo = QComboBox()
        self.combo.addItems(['work', 'tool'])
        self.combo.currentIndexChanged.connect(self.resetCoords)

        self.addWidget(self.combo, 0, 1)

    def resetCoords(self, ind):
        for spinbox in self.spinboxes:
            spinbox.setValue(0)

    @property
    def spinboxes(self):
        return self.controlPanel.commands.coordBoxes.widgets

    def buildCmd(self):
        labels = ['X', 'Y', 'Z', 'U', 'V', 'W']
        values = [spinbox.getValue() for spinbox in self.spinboxes]

        cmdStr = '%s slit set %s ' % (self.controlPanel.actorName, self.combo.currentText().lower())
        cmdStr += (" ".join(['%s=%.5f' % (label, value) for label, value in zip(labels, values)]))

        return cmdStr

class SlitCommands(CommandsGB):
    def __init__(self, controlPanel):
        CommandsGB.__init__(self, controlPanel)
        self.statusButton = CmdButton(controlPanel=controlPanel, label='STATUS',
                                      cmdStr='%s slit status' % controlPanel.actorName)
        self.connectButton = CmdButton(controlPanel=controlPanel, label='CONNECT',
                                       cmdStr='%s connect controller=slit' % controlPanel.actorName)
        self.initButton = CmdButton(controlPanel=controlPanel, label='INIT',
                                    cmdStr='%s slit init' % controlPanel.actorName)
        self.abortButton = AbortButton(controlPanel=controlPanel, cmdStr='%s slit abort' % controlPanel.actorName)
        self.goHomeButton = CmdButton(controlPanel=controlPanel, label='GO HOME',
                                      cmdStr='%s slit move home' % controlPanel.actorName)
        self.coordBoxes = CoordBoxes()

        self.moveCmd = MoveCmd(controlPanel=controlPanel)
        self.setRepCmd = SetRepCmd(controlPanel=controlPanel)

        self.grid.addWidget(self.statusButton, 0, 0)
        self.grid.addWidget(self.connectButton, 0, 1)
        self.grid.addWidget(self.initButton, 1, 0)
        self.grid.addWidget(self.abortButton, 1, 1)
        self.grid.addLayout(self.coordBoxes, 2, 0, 2, 3)
        self.grid.addLayout(self.moveCmd, 4, 0, 1, 2)
        self.grid.addLayout(self.setRepCmd, 5, 0, 1, 2)
        self.grid.addWidget(self.goHomeButton, 6, 0, 1, 1)

    @property
    def buttons(self):
        return [self.statusButton, self.connectButton, self.initButton, self.abortButton,
                self.goHomeButton, self.moveCmd.button, self.setRepCmd.button]

class SlitPanel(ControlPanel):
    def __init__(self, controlDialog):
        ControlPanel.__init__(self, controlDialog)

        self.mode = ValueGB(self.moduleRow, 'slitMode', 'Mode', 0, '{:s}')
        self.state = ValueGB(self.moduleRow, 'slitFSM', '', 0, '{:s}')
        self.substate = ValueGB(self.moduleRow, 'slitFSM', '', 1, '{:s}')
        self.info = ValueGB(self.moduleRow, 'slitInfo', 'Info', 0, '{:s}')
        self.location = ValueGB(self.moduleRow, 'slitLocation', 'Location', 0, '{:s}')

        self.coordinates = Coordinates(self.moduleRow, 'slit', title='Position')
        self.work = Coordinates(self.moduleRow, 'slitWork', title='Work')
        self.tool = Coordinates(self.moduleRow, 'slitTool', title='Tool')

        self.commands = SlitCommands(self)

        self.grid.addWidget(self.mode, 0, 0)
        self.grid.addWidget(self.state, 0, 1)
        self.grid.addWidget(self.substate, 0, 2)
        self.grid.addWidget(self.location, 0, 3)

        self.grid.addWidget(self.info, 1, 0, 1, 6)
        self.grid.addWidget(self.coordinates, 2, 0, 1, 6)
        self.grid.addWidget(self.work, 3, 0, 1, 6)
        self.grid.addWidget(self.tool, 4, 0, 1, 6)

        self.grid.addWidget(self.commands, 0, 7, 5, 4)

    @property
    def customWidgets(self):
        return self.coordinates.widgets + self.work.widgets + self.tool.widgets


"""
A simple logistic regression (with gradient descent) model working on a dataset of 4298563 instances.
The model gave an accuracy of 63.5% - training and 68.37% - testing with iterations = 500 and learning rate = 0.05.
Similar results obtained for all possible combinations of iterations = 100 and learning rate = 0.1.
The model proves to be underfitting for our dataset due to being too simple for the data
(inference from similar values for training and test accuracies, Jtrain & jtest - similar)

 """

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

#preprocessing
data = pd.read_csv('data.csv')
print(data.shape)

"""
VECTOR SHAPES
X = n x m
Y = 1 x m
w = n x 1
Z = 1 x m
A = 1 x m
m: the number of datasets
n: the number of features

"""

#preprocessing-2
train_set_X = data.iloc[:4000000, :-1].values.transpose()
train_set_Y = data.iloc[:4000000, 61].values.reshape(1,train_set_X.shape[1])
test_set_X = data.iloc[4000000: , :-1].values.transpose()
test_set_Y = data.iloc[4000000:, 61].values.reshape(1,test_set_X.shape[1])
print(test_set_X.shape)
print(test_set_Y.shape)

#helper function: sigmoid
def sigmoid(z):
  return 1.0 / (1 + np.exp(-z))

#helper function: parameter initialization
def initialize(dim):
    """ intializes vector w of shape(dim,1) and b as 0 """
    w = np.zeros((dim,1))
    b = 0
    return w, b

#helper function: forward & back propagation
def propagate(w, b, X, Y):
    m = X.shape[1]

    #FORWARD
    Z = np.dot(w.transpose(), X) + b
    #print(Z.shape)
    A = sigmoid(Z)
    #print(A.shape)
    cost = -(np.dot(Y, np.log(A).transpose())+ np.dot((1-Y), np.log(1-A).transpose()))

    #BACKWARD
    dZ = A - Y
    dw = np.dot(X, dZ.transpose())/m
    db = np.sum(dZ)/m

    gradients = {"dw" : dw,
                 "db" : db}
    return gradients, cost

"""w, b, X, Y = np.array([[1.],[2.]]), 2., np.array([[1.,2.,-1.],[3.,4.,-3.2]]), np.array([[1,0,1]])
grads, cost = propagate(w, b, X, Y)
print ("dw = " + str(grads["dw"]))
print ("db = " + str(grads["db"]))
print ("cost = " + str(cost))"""

#helper function: optimize via Gradient Descent
def gradientDescent(w, b ,X, Y, iterations, alpha, print_cost):
    costs  = []
    for i in range(iterations):

        #forward & back propagation
        grads, cost = propagate(w, b ,X, Y)
        dw = grads["dw"]
        db = grads["db"]

        #weight updation
        w = w - alpha*dw
        b = b - alpha*db

        if (i%100) == 0:
            costs.append(cost)

        if print_cost and (i%100==0):
            print("the cost after "+ str(i) +" iterations is: "+ str(cost))

    params = {"w" : w,
              "b" : b}
    grads = {"dw" : dw,
             "db" : db}
    return params, grads, costs

#helper function: to predict new values
def predict(w, b, X):
    """ w is a (n x 1) matrix, X is a (m x n) matrix """
    m = X.shape[1]
    Y_prediction = np.zeros((1,m))
    w = w.reshape(X.shape[0], 1)  #no idea why this exist?

    Z = np.dot(w.transpose(), X)+b
    A = sigmoid(Z)

    for i in range(A.shape[1]):
        if A[0][i]<=0.5:
            Y_prediction[0][i] = 0
        else:
            Y_prediction[0][i] = 1

    return Y_prediction

#model
def model(X_train, Y_train, X_test, Y_test, iterations, alpha, print_cost):

    #initialize
    w, b = initialize(X_train.shape[0])

    #fw & bp
    params, grads, costs = gradientDescent(w, b, X_train, Y_train, iterations, alpha, print_cost)

    #retreive parameters
    w = params["w"]
    b = params["b"]

    Y_prediction_train = predict(w, b, X_train)
    Y_prediction_test = predict(w, b, X_test)

    #print accuracies
   # print(Y_prediction_train.shape)
    #print(Y_train.shape)
    print("train accuracy: {} %".format(100 - np.mean(np.abs(Y_prediction_train - Y_train)) * 100))
    print("test accuracy: {} %".format(100 - np.mean(np.abs(Y_prediction_test - Y_test)) * 100))

    d = {"costs": costs,
         "Y_prediction_test": Y_prediction_test,
         "Y_prediction_train" : Y_prediction_train,
         "w" : w,
         "b" : b,
         "learning_rate" : alpha,
         "iterations": iterations}

    return d

d = model(train_set_X, train_set_Y, test_set_X, test_set_Y, iterations = 500, alpha = 0.05, print_cost = True)

"""
OUTPUT :    the cost after 0 iterations is: [[2772588.72223999]]
            the cost after 100 iterations is: [[2636107.19196553]]
            the cost after 200 iterations is: [[2624253.75377114]]
            the cost after 300 iterations is: [[2623146.61367674]]
            the cost after 400 iterations is: [[2623040.22673616]]
            train accuracy: 63.58705 %
            test accuracy: 68.30719144703127 %
"""


"""
data.json - файл с данными о некоторых математиках прошлого.
1. Необходимо написать функцию, которая считает эти данные из файла. Параметр функции - имя файла.
2. Написать функцию сортировки данных по ФАМИЛИИ в поле "name" (у тех у кого она есть).
Например для Rene Descartes фамилия это Descartes, у Pierre de Fermat - Fermat и т.д.
Если фамилии нет, то использовать имя, например Euclid.
3. Написать функцию сортировки по дате смерти из поля "years".
Обратите внимание на сокращение BC. - это означает до н.э.
4. Написать функцию сортировки по количеству слов в поле "text"
"""
import json

# 1. Необходимо написать функцию, которая считает эти данные из файла. Параметр функции - имя файла.

def read_json(filename):
    with open(filename, "r") as json_file:
        data = json.load(json_file)
        return data

print(f'Json data: {read_json("data.json")}')

# 2. Написать функцию сортировки данных по ФАМИЛИИ в поле "name" (у тех у кого она есть).
# Если фамилии нет, то использовать имя, например Euclid.

def split_name(path):
    dicts_list = read_json(path)
    for dict in dicts_list:
        splited_name = dict['name'].split(' ')
        dict['name'] = splited_name
    return dicts_list

def sort_by_last_name(last_name_dict):
    last_name = last_name_dict['name']
    last_name = last_name[-1]
    first_name = last_name_dict['name']
    first_name = first_name[0]
    return last_name, first_name

def join_name(path):
    sorted_by_last_name_list = sorted(split_name(path), key=sort_by_last_name)
    for dict in sorted_by_last_name_list:
        joined_name = ' '.join(dict['name'])
        dict['name'] = joined_name
    return sorted_by_last_name_list

print(f'Sorted by last name: {join_name("data.json")}')

# 3. Написать функцию сортировки по дате смерти из поля "years".

def split_year(path):
    dicts_list = read_json(path)
    for dict in dicts_list:
        splited_years = dict['years'].split(' ')
        dict['years'] = splited_years
        if dict['years'][-1] == 'BC.':
            dict['years'][-2] = f'-{dict["years"][-2]}'
            dict['years'].pop(-1)
    return dicts_list

def sort_by_year(year_dict):
    year = float(year_dict['years'][-1])
    return year

def join_year(path):
    sorted_by_year_list = sorted(split_year(path), key=sort_by_year)
    for dict in sorted_by_year_list:
        if float(dict['years'][-1]) < 0:
            dict['years'][-1] = dict['years'][-1].replace('-', '')
            dict['years'].append('BC.')
        joined_year = ' '.join(dict['years'])
        dict['years'] = joined_year
    return sorted_by_year_list

print(f'Sorted by last year: {join_year("data.json")}')

# 4. Написать функцию сортировки по количеству слов в поле "text"

def split_text(path):
    dicts_list = read_json(path)
    for dict in dicts_list:
        splited_text = dict['text'].split(' ')
        dict['text'] = splited_text
    return dicts_list

def join_text(path):
    sorted_by_words_number_list = sorted(split_text(path), key=lambda x: len(x.get('text')))
    for dict in sorted_by_words_number_list:
        joined_text = ' '.join(dict['text'])
        dict['text'] = joined_text
    return sorted_by_words_number_list

print(f'Sorted by words number in text field: {join_text("data.json")}')


#!/usr/bin/python

import click
import pandas as pd
from src import DataPreprocessing
from src import CategoryPrediction
from src import SentenceSimilarity

@click.group()
def cli():
    pass

'''
1. Preprocessing (print lemma, token, stopword)
2. Classify tokenized question to category (Doc2vec category model)
3. find closest question tag (Doc2vec question model of specific category)
4. find closest question tag from all questions (not category specific)
'''
@cli.command()
@click.option('--question', "question", prompt='Question to process', help='Question to process')
def run(question):
    click.echo("\n" + "The question is: " + question + "\n")

    '''
    Tokenize input question
    '''
    dp = DataPreprocessing()
    tokens = dp.clean_sentence(question)
    for token in tokens:
        click.echo("Token: " + " " + token.get("token") +
                   ", Lemma: " + " " + token.get("lemma") +
                   ", is stop word: " + " " + str(token.get("is_stop")))

    tokens_for_pred = [token.get("lemma") for token in tokens]

    '''
    Predict category from tokenized question
    '''
    click.echo("\n" + "Predicting Category... ")
    cp = CategoryPrediction()
    prediction = cp.predict(tokens_for_pred, model_to_use="doc2vec")
    predicted_category = prediction.get("category")
    click.echo("Predicted Category is: " + predicted_category +
               ", Prediction Score: " + str(prediction.get("score")))

    '''
    Predict Question and Answer from the predicted Category.
    '''
    click.echo("\n" + "Predicted From " + predicted_category + "Category: ")
    ss1 = SentenceSimilarity(predicted_category)
    similar_sentence_for_category = ss1.find_similar_sentence(tokens_for_pred)

    # get answer and question from tag
    data = pd.read_pickle('resources/processed_data/category_' + predicted_category + '.pkl')
    question = data.iloc[similar_sentence_for_category.get("sentence")]["Name"]
    answer = data.iloc[similar_sentence_for_category.get("sentence")]["FullDescription"]

    click.echo("\n" + "Question: " + question +
               "\n" + "Answer: " + answer +
               "\n" + "Score: " + str(similar_sentence_for_category.get("score")) + "\n")

    '''
    Predict Question and Answer from all Categories.
    Used for reference
    '''
    ss = SentenceSimilarity()
    similar_sentence = ss.find_similar_sentence(tokens_for_pred)
    click.echo("Predicted From all Categories: (USED FOR REFERENCE)")

    # get answer and question from tag
    data = pd.read_pickle('resources/processed_data/category_' + "all" + '.pkl')
    question = data.iloc[similar_sentence.get("sentence")]["Name"]
    answer = data.iloc[similar_sentence.get("sentence")]["FullDescription"]

    click.echo("\n" + "Question: " + question +
               "\n" + "Answer: " + answer +
               "\n" + "Score: " + str(similar_sentence.get("score")) + "\n")

if __name__ == "__main__":
    cli()


import numpy as np
from tensorflow.keras.datasets import cifar100, mnist
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Conv2D, Flatten, MaxPool2D, Dropout
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.utils import to_categorical
from sklearn.preprocessing import MinMaxScaler, StandardScaler, RobustScaler, PowerTransformer, QuantileTransformer
import time
import matplotlib.pyplot as plt

'''
# 10개의 이미지를 분류하는 것
# 컬러 데이터

(x_train, y_train), (x_test, y_test) = mnist.load_data()

# print(x_train.shape, y_train.shape) # (60000, 28, 28) (60000,) -> 3차원
# print(x_test.shape, y_test.shape) # (10000, 28, 28) (10000,)

print(np.unique(y_train))

x_train = x_train.reshape(60000, 28*28)
x_test = x_test.reshape(10000,  28*28)

scaler = StandardScaler()
x_train = scaler.fit_transform(x_train)
x_test = scaler.transform(x_test)

x_train = x_train.reshape(60000, 28, 28, 1)
x_test = x_test.reshape(10000,  28, 28, 1)

# 전처리 하기 -> one-hot-encoding
y_train = to_categorical(y_train)
y_test = to_categorical(y_test)

print(y_train.shape, y_test.shape)
'''

# modeling
model = Sequential()
model.add(Conv2D(128, kernel_size=(2, 2),
                    padding='valid', input_shape=(28, 28, 1), activation='relu'))
model.add(Dropout(0.2))
model.add(Conv2D(128, (2,2), padding='same', activation='relu'))
model.add(MaxPool2D())

model.add(Conv2D(128, (2,2),padding='valid', activation='relu'))
model.add(Dropout(0.2))
model.add(Conv2D(128, (2,2), padding='same', activation='relu'))
model.add(MaxPool2D())

model.add(Conv2D(64, (2,2), padding='valid', activation='relu'))
model.add(Dropout(0.2))
model.add(Conv2D(64, (2,2), padding='same', activation='relu'))
model.add(MaxPool2D())

model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(32, activation='relu'))
model.add(Dense(10, activation='softmax'))

model.summary()

model.save('./_save/keras45_1_save_model.h5')

'''
# compile
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc'])

es = EarlyStopping(monitor='val_loss', patience=10, verbose=1, mode='min')

start_time = time.time()
hist = model.fit(x_train, y_train, epochs=100, verbose=1, callbacks=[es], validation_split=0.2,
shuffle=True, batch_size=256)
end_time = time.time() - start_time

# evaluate
loss = model.evaluate(x_test, y_test)
print("걸린시간: ", end_time)
print('category: ', loss[0])
print('accuracy: ', loss[1])

# 시각화
plt.figure(figsize=(9,5))

# 1
plt.subplot(2, 1, 1) # 2개의 플롯을 할건데, 1행 1열을 사용하겠다는 의미
plt.plot(hist.history['loss'], marker='.', c='red', label='loss')
plt.plot(hist.history['val_loss'], marker='.', c='blue', label='val_loss')
plt.grid()
plt.title('loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(loc='upper right')

# 2
plt.subplot(2, 1, 2) # 2개의 플롯을 할건데, 1행 2열을 사용하겠다는 의미
plt.plot(hist.history['acc'])
plt.plot(hist.history['val_acc'])
plt.grid()
plt.title('acc')
plt.ylabel('acc')
plt.xlabel('epoch')
plt.legend(['acc', 'val_acc'])

plt.show()

'''

'''
dropout 실행
걸린시간:  603.1085088253021
category:  2.0480213165283203
accuracy:  0.4699999988079071

Conv1D
걸린시간:  221.4401044845581
category:  0.025295158848166466
accuracy:  0.9943000078201294

'''


import numpy as np
from scipy.stats import rankdata
from functools import partial
from nptyping import Array
from sklearn.metrics import pairwise_distances
from sklearn.base import BaseEstimator, TransformerMixin

class BoostedSURF(BaseEstimator, TransformerMixin):

    """sklearn compatible implementation of the boostedSURF algorithm

    Gediminas Bertasius, Delaney Granizo-Mackenzie, Ryan Urbanowicz, Jason H. Moore.
    Boosted Spatially Uniform ReliefF Algorithm for Genome-Wide Genetic Analysis.

    author: Jernej Vivod

    """

    def __init__(self, n_features_to_select=10, phi=5,
            dist_func=lambda w, x1, x2: np.sum(w*np.logical_xor(x1, x2), 1), learned_metric_func=None):

        self.n_features_to_select = n_features_to_select  # Number of features to select.
        self.phi = phi                                    # the phi parameter (update weights when iteration_counter mod phi == 0)
        self.dist_func = dist_func                        # Distance function to use.
        self.learned_metric_func = learned_metric_func    # learned metric function.

    def fit(self, data, target):

        """
        Rank features using BoostedSURF feature selection algorithm

        Args:
            data : Array[np.float64] -- matrix of examples
            target : Array[np.int] -- vector of target values of examples

        Returns:
            self
        """

        # Run Boosted SURF feature selection algorithm.
        if self.learned_metric_func != None:
            self.rank, self.weights = self._boostedSURF(data, target, self.phi, self.dist_func,
                    learned_metric_func=self.learned_metric_func(data, target))
        else:
            self.rank, self.weights = self._boostedSURF(data, target, self.phi, self.dist_func)

        return self

    def transform(self, data):

        """
        Perform feature selection using computed feature ranks

        Args:
            data : Array[np.float64] -- matrix of examples on which to perform feature selection

        Returns:
            Array[np.float64] -- result of performing feature selection
        """

        msk = self.rank <= self.n_features_to_select  # Compute mask.
        return data[:, msk]  # Perform feature selection.

    def fit_transform(self, data, target):

        """
        Compute ranks of features and perform feature selection
        Args:
            data : Array[np.float64] -- matrix of examples on which to perform feature selection
            target : Array[np.int] -- vector of target values of examples

        Returns:
            Array[np.float64] -- result of performing feature selection
        """

        self.fit(data, target)
        return self.transform(data)

    def _boostedSURF(self, data, target, phi, dist_func, **kwargs):

        """Compute feature scores using boostedSURF algorithm

        Args:
            data : Array[np.float64] -- Matrix containing examples' data as rows
            target : Array[np.int] -- matrix containing the example's target variable value
            phi: int -- parameter specifying number of iterations before recomputing distance weights
            dist_func : Callable[[Array[np.float64], Array[np.float64]], Array[np.float64]] -- function for evaluating
            distances between examples. The function should acept two examples or two matrices of examples and return the dictances.
            **kwargs: can contain argument with key 'learned_metric_func' that maps to a function that accepts a distance
            function and indices of two training examples and returns the distance between the examples in the learned
            metric space.

        Returns:
            Array[np.int], Array[np.float64] -- Array of feature enumerations based on the scores, array of feature scores

        """

        # Initialize distance weights.
        dist_weights = np.ones(data.shape[1], dtype=np.float)

        # weighted distance function
        dist_func_w = partial(dist_func, dist_weights)

        # Initialize weights.
        weights = np.zeros(data.shape[1], dtype=np.int)

        for idx in np.arange(data.shape[0]):

            # Recompute distance matrix.
            if np.mod(idx, phi) == 0:
                dist_weights = np.maximum(weights, np.ones(data.shape[1], dtype=np.float))
                dist_func_w = partial(dist_func, dist_weights)
                if 'learned_metric_func' in kwargs:
                    dist_func_w_learned = partial(kwargs['learned_metric_func'], dist_func_w)

            # Compute distances from current examples to all other examples.
            if 'learned_metric_func' in kwargs:
                dists = dist_func_w_learned(idx, np.arange(data.shape[0]))
            else:
                dists = dist_func_w(data[idx, :], data)

            # Compute mean and standard deviation of distances and set thresholds.
            t_next = np.mean(dists[np.arange(data.shape[0]) != idx])
            sigma_nxt = np.std(dists[np.arange(data.shape[0]) != idx])
            thresh_near = t_next - sigma_nxt/2.0
            thresh_far = t_next + sigma_nxt/2.0

            # Get mask of examples that are close.
            msk_close = dists < thresh_near
            msk_close[idx] = False

            # Get mask of examples that are far.
            msk_far = dists > thresh_far

            # Get examples that are close.
            examples_close = data[msk_close, :]
            target_close = target[msk_close]

            # Get examples that are far.
            examples_far = data[msk_far, :]
            target_far = target[msk_far]

            # Get considered features of close examples.
            features_close = data[idx, :] != examples_close
            # Get considered features of far examples.
            features_far = data[idx, :] == examples_far

            # Get mask for close examples with same class.
            msk_same_close = target_close == target[idx]

            # Get mask for far examples with same class.
            msk_same_far = target_far == target[idx]

            ### WEIGHTS UPDATE ###

            # Get penalty weights update values for close examples.
            wu_close_penalty = np.sum(features_close[msk_same_close, :], 0)
            # Get reward weights update values for close examples.
            wu_close_reward = np.sum(features_close[np.logical_not(msk_same_close), :], 0)

            # Get penalty weights update values for far examples.
            wu_far_penalty = np.sum(features_far[msk_same_far, :], 0)
            # Get reward weights update values for close examples.
            wu_far_reward = np.sum(features_far[np.logical_not(msk_same_far), :], 0)

            # Update weights.
            weights = weights - (wu_close_penalty + wu_far_penalty) + (wu_close_reward + wu_far_reward)

            ### /WEIGHTS UPDATE ###

        # Create array of feature enumerations based on score.
        rank = rankdata(-weights, method='ordinal')
        return rank, weights


#!/urs/env/bin python

""" Bagging meta-estimator for 8 classes (1,2,3,4,5,6,7,8)
"""

__author__ = 'Nanxiang Zhao'
__email__ = 'samzhao@umich.edu'

import numpy as np
from sklearn.ensemble import BaggingClassifier
from sklearn.model_selection import cross_val_score

# training the model
clf = BaggingClassifier(max_samples = 0.65)

# loading the data
training = np.load('eight_class_train.npy')
testing = np.load('eight_class_test.npy')

# cross-validation
scores = cross_val_score(clf, training[:, 0:20], training[:, 20], cv=5)
print(scores.mean())
print(scores.std())

# predict on testing data
clf = clf.fit(training[:, 0:20], training[:, 20])
answers = clf.predict(testing[:, 0:20])
accuracy = np.sum(answers == testing[:, 20]) / float(len(testing[:,20]))
score_prob = clf.predict_proba(testing[:, 0:20])
print(score_prob)
print(accuracy)


def strassen_naive(a, b):
    # Non empty matrices check
    try:
        r1, c1 = len(a), len(a[0])
        r2, c2 = len(b), len(b[0])
    except IndexError as e:
        raise IndexError("Can't Multiple empty matrices") from e

    # Cols(a) = Rows(b)
    cond1 = c1 != r2
    if c1 != r2:
        raise NameError('Matrix Multiplication not Possible')

    # Compute matrix product
    out = []
    for i in range(r1):
        li = []
        for j in range(c2):
            # Compute dot product of ith row of a with jth column of b
            ans = 0
            for k in range(c1):
                ans += a[i][k] * b[k][j]
            li.append(ans)
        out.append(li)

    return out


import discord
from discord.ext import commands
import requests
import os

OWL_BOT_TOKEN = os.environ.get("OWL_BOT_TOKEN")
HEADER = {"Authorization": OWL_BOT_TOKEN}

class Dictionary(commands.Cog):
    """
    Dictionary Module
    """

    def __init__(self, bot):
        self.bot = bot

    @commands.command(help="Get word definitions")
    async def dictionary(self, ctx, word):
        with requests.get(
            url=f"https://owlbot.info/api/v4/dictionary/{word}/",
            headers={"Authorization": "Token 64154ef64d2de67c9f031ac98798fb57eaaf2f41"},
        ) as response:
            rsp = response.json()
            await ctx.send(
                f"```word: {rsp['word']}\ndefinition: {rsp['definitions'][0]['definition']}```"
            )
            try:
                await ctx.send(response.json()["definitions"][0]["image_url"])

            except:
                await ctx.send("no image available")

def setup(bot):
    bot.add_cog(Dictionary(bot))


import ndimage as ndi
import numpy
import math
from scipy._lib.six import string_types

def gaussianFilter(input, sigma = 4, order=0, output=None,
                  mode="reflect", cval=0.0, truncate=4.0):
    """Multidimensional Gaussian filter.

    Parameters
    ----------
    %(input)s
    sigma : scalar or sequence of scalars
        Standard deviation for Gaussian kernel. The standard
        deviations of the Gaussian filter are given for each axis as a
        sequence, or as a single number, in which case it is equal for
        all axes.
    order : {0, 1, 2, 3} or sequence from same set, optional
        The order of the filter along each axis is given as a sequence
        of integers, or as a single number.  An order of 0 corresponds
        to convolution with a Gaussian kernel. An order of 1, 2, or 3
        corresponds to convolution with the first, second or third
        derivatives of a Gaussian. Higher order derivatives are not
        implemented
    %(output)s
    %(mode)s
    %(cval)s
    truncate : float
        Truncate the filter at this many standard deviations.
        Default is 4.0.

    Returns
    -------
    gaussian_filter : ndarray
        Returned array of same shape as `input`.

    Notes
    -----
    The multidimensional filter is implemented as a sequence of
    one-dimensional convolution filters. The intermediate arrays are
    stored in the same data type as the output. Therefore, for output
    types with a limited precision, the results may be imprecise
    because intermediate results may be stored with insufficient
    precision.

    """
    input = numpy.asarray(input)
    output, return_value = _get_output(output, input)
    orders = _normalize_sequence(order, input.ndim)
    if not set(orders).issubset(set(range(4))):
        raise ValueError('Order outside 0..4 not implemented')
    sigmas = _normalize_sequence(sigma, input.ndim)
    axes = list(range(input.ndim))
    axes = [(axes[ii], sigmas[ii], orders[ii])
                        for ii in range(len(axes)) if sigmas[ii] > 1e-15]
    if len(axes) > 0:
        for axis, sigma, order in axes:
            gaussian_filter1d(input, sigma, axis, order, output,
                              mode, cval, truncate)
            input = output
    else:
        output[...] = input[...]
    return return_value

def gaussian_filter1d(input, sigma, axis=-1, order=0, output=None,
                      mode="reflect", cval=0.0, truncate=4.0):
    """One-dimensional Gaussian filter.

    Parameters
    ----------
    %(input)s
    sigma : scalar
        standard deviation for Gaussian kernel
    %(axis)s
    order : {0, 1, 2, 3}, optional
        An order of 0 corresponds to convolution with a Gaussian
        kernel. An order of 1, 2, or 3 corresponds to convolution with
        the first, second or third derivatives of a Gaussian. Higher
        order derivatives are not implemented
    %(output)s
    %(mode)s
    %(cval)s
    truncate : float, optional
        Truncate the filter at this many standard deviations.
        Default is 4.0.

    Returns
    -------
    gaussian_filter1d : ndarray

    """
    if order not in range(4):
        raise ValueError('Order outside 0..3 not implemented')
    sd = float(sigma)
    # make the radius of the filter equal to truncate standard deviations
    lw = int(truncate * sd + 0.5)
    weights = [0.0] * (2 * lw + 1)
    weights[lw] = 1.0
    sum = 1.0
    sd = sd * sd
    # calculate the kernel:
    for ii in range(1, lw + 1):
        tmp = math.exp(-0.5 * float(ii * ii) / sd)
        weights[lw + ii] = tmp
        weights[lw - ii] = tmp
        sum += 2.0 * tmp
    for ii in range(2 * lw + 1):
        weights[ii] /= sum
    # implement first, second and third order derivatives:
    if order == 1:  # first derivative
        weights[lw] = 0.0
        for ii in range(1, lw + 1):
            x = float(ii)
            tmp = -x / sd * weights[lw + ii]
            weights[lw + ii] = -tmp
            weights[lw - ii] = tmp
    elif order == 2:  # second derivative
        weights[lw] *= -1.0 / sd
        for ii in range(1, lw + 1):
            x = float(ii)
            tmp = (x * x / sd - 1.0) * weights[lw + ii] / sd
            weights[lw + ii] = tmp
            weights[lw - ii] = tmp
    elif order == 3:  # third derivative
        weights[lw] = 0.0
        sd2 = sd * sd
        for ii in range(1, lw + 1):
            x = float(ii)
            tmp = (3.0 - x * x / sd) * x * weights[lw + ii] / sd2
            weights[lw + ii] = -tmp
            weights[lw - ii] = tmp
    return ndi.correlate1d(input, weights, axis, output, mode, cval, 0)

def medianFilter(input, size=3, footprint=None, output=None,
                  mode="reflect", cval=0.0, origin=0):
    """
    Calculates a multidimensional median filter.

    Parameters
    ----------
    %(input)s
    %(size_foot)s
    %(output)s
    %(mode)s
    %(cval)s
    %(origin)s

    Returns
    -------
    median_filter : ndarray
        Return of same shape as `input`.

    """
    return ndi._rank_filter(input, 0, size, footprint, output, mode, cval,
                        origin, 'median')

def _normalize_sequence(input, rank, array_type=None):
    """If input is a scalar, create a sequence of length equal to the
    rank by duplicating the input. If input is a sequence,
    check if its length is equal to the length of array.
    """
    if hasattr(input, '__iter__'):
        normalized = list(input)
        if len(normalized) != rank:
            err = "sequence argument must have length equal to input rank"
            raise RuntimeError(err)
    else:
        normalized = [input] * rank
    return normalized

def _get_output(output, input, shape=None):
    if shape is None:
        shape = input.shape
    if output is None:
        output = numpy.zeros(shape, dtype=input.dtype.name)
        return_value = output
    elif type(output) in [type(type), type(numpy.zeros((4,)).dtype)]:
        output = numpy.zeros(shape, dtype=output)
        return_value = output
    elif type(output) in string_types:
        output = numpy.typeDict[output]
        output = numpy.zeros(shape, dtype=output)
        return_value = output
    else:
        if output.shape != shape:
            raise RuntimeError("output shape not correct")
        return_value = None
    return output, return_value

def _check_axis(axis, rank):
    if axis < 0:
        axis += rank
    if axis < 0 or axis >= rank:
        raise ValueError('invalid axis')
    return axis

def _extend_mode_to_code(mode):
    """Convert an extension mode to the corresponding integer code.
    """
    if mode == 'nearest':
        return 0
    elif mode == 'wrap':
        return 1
    elif mode == 'reflect':
        return 2
    elif mode == 'mirror':
        return 3
    elif mode == 'constant':
        return 4
    else:
        raise RuntimeError('boundary mode not supported')


# nwes.daum.net 에서 JTBC 뉴스의 헤드라인, 간략뉴스를 추출해서
# jtbcnews 컬렉션에 저장
# 필드명 : news, desc

from bs4 import BeautifulSoup
import requests
from pymongo import MongoClient

url = 'https://media.daum.net/cp/310?regDate=20190502'

headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) \
    AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36'}

res = requests.get(url, headers=headers)
html = BeautifulSoup(res.text, 'lxml')

news = []
desc = []

# 뉴스 헤드라인
for item in html.select('strong.tit_thumb a'):
    news.append(item.text.strip())
news.pop(); news.pop()

# 간략 뉴스
for item in html.select('div.desc_thumb span'):
    desc.append(item.text.strip())

# print(news)
# print(desc)

######################################################################

client = MongoClient('mongodb://192.168.180.133:27017')

db = client.bigdata
coll = db.jtbcnews

coll.remove({})

for i in range(len(news)):
    newOne = { 'news': news[i], 'desc': desc[i] }
    coll.insert_one(newOne)

client.close()

2. [Python_Codes] Fabio Ko's Python Programming (파이썬 프로그래밍)/mongo/04JTBC_news_mongo.py


#
# @lc app=leetcode.cn id=126 lang=python
#
# [126] 单词接龙 II
#

# @lc code=start
class Solution(object):
    def findLadders(self, beginWord, endWord, wordList):
        """
        :type beginWord: str
        :type endWord: str
        :type wordList: List[str]
        :rtype: List[List[str]]
        """
        wordList = set(wordList)
        res = self.bfs(beginWord, endWord, wordList)
        return res

    def bfs(self, beginWord, endWord, wordList):
        # 建立转换列表
        changes = defaultdict(list)
        for s in wordList:
            for i in range(len(s)):
                new = s[:i] + '*' + s[i+1:]
                changes[new].append(s)
        # 记录节点和访问路径
        queue, nextq = [(beginWord, [beginWord])], []
        visited, res = set(), []
        while queue:
            # 遍历当前层节点，相邻下一层节点不入队
            while queue:
                curr, path = queue.pop(0)
                if curr == endWord:
                    res.append(path)
                visited.add(curr)
                for i in range(len(curr)):
                    new = curr[:i] + "*" + curr[i+1:]
                    for word in changes[new]:
                        if word not in visited:
                            # 节点未被访问，加入下一层队列并添加路径
                            nextq.append((word, path + [word]))
            if res:
                return res
            # 完成当前层遍历，遍历下一层所有节点
            queue, nextq = nextq, queue
        return []
# @lc code=end


"""
    html-pdf-webservice

    Copyright 2014 Nathan Jones
    See LICENSE for more details
"""

import re
from locust import HttpLocust, TaskSet

wikipedia_main_page_html = "wikipedia-main-page.html"

def print_document(l, html_file_name):
    with open(html_file_name, "r") as html_file:
        l.client.post(
            url="/",
            files={
                "html": html_file
            }
        )

def print_simple_document(l):
    print_document(l, "sample.html")

def print_wikipedia_home_page(l):
    print_document(l, wikipedia_main_page_html)

def download_wikipedia_home_page(l):
    response = l.client.get("https://en.wikipedia.org/wiki/Main_Page")
    # External CSS and image links will be in scheme-relative form, such as
    # "//host/path". These URLs will break when the page is loaded with the
    # file:// scheme by wkhtmltopdf. Replace them with https:// URLs.
    html = re.sub('([^:])//', r'\1https://', response.text)
    with open(wikipedia_main_page_html, "w") as f:
        f.write(html.encode('utf8'))

class UserBehavior(TaskSet):
    tasks = {
        print_simple_document: 1,
        print_wikipedia_home_page: 1,
    }

    def on_start(self):
        download_wikipedia_home_page(self)

class WebsiteUser(HttpLocust):
    task_set = UserBehavior
    min_wait = 5000
    max_wait = 9000


import gettext
gettext.bindtextdomain("expeyes")
gettext.textdomain('expeyes')
_ = gettext.gettext

import expeyes.eyes17, expeyes.eyeplot17 as eyeplot,sys
VER = sys.version[0]
if VER == '3':
	from tkinter import *
else:
	from Tkinter import *

def update():
	v = p.get_voltage('A1')
	if v == v:   #is it NaN
		A1.config(text=_('A1 = %5.3f volts'%(v)))
	v = p.get_voltage('A2')
	if v == v:   #is it NaN
		A2.config(text=_('A2 = %5.3f volts'%(v)))

	v = p.get_voltage('A3')
	if v == v:   #is it NaN
		A3.config(text=_('A3 = %5.3f volts'%(v)))

	w.after(500,update)

p=expeyes.eyes17.open()
w = Tk()
A1 = Label(text=_(''), font=("Helvetica", 26))
A1.pack(side=TOP)
A2 = Label(text=_(''), font=("Helvetica", 26))
A2.pack(side=TOP)
A3 = Label(text=_(''), font=("Helvetica", 26))
A3.pack(side=TOP)
Button(text=_('QUIT'), command=sys.exit).pack(side=TOP)
w.title(_('EYES-17: Measuring voltage'))
w.after(500,update)
w.mainloop()


# Most code in this file was borrowed from https://github.com/lengstrom/fast-style-transfer/blob/master/src/transform.py

import tensorflow as tf

class Transform:
    def __init__(self, n_styles, mode='train'):
        self.n_styles = n_styles
        if mode == 'train':
            self.reuse = None
        else:
            self.reuse = True

    def net(self, image, style_index):
        conv1 = self._conv_layer(image, 16, 9, 1, name='conv1')
        cinst1 = self._conditional_instance_norm(conv1, style_index, 'cinst1')

        conv2 = self._conv_layer(cinst1, 32, 3, 2, name='conv2')
        cinst2 = self._conditional_instance_norm(conv2, style_index, 'cinst2')

        conv3 = self._conv_layer(cinst2, 32, 3, 2, name='conv3')
        cinst3 = self._conditional_instance_norm(conv3, style_index, 'cinst3')

        resid1 = self._residual_block(cinst3, 3, name='resid1')
        resid2 = self._residual_block(resid1, 3, name='resid2')
        resid3 = self._residual_block(resid2, 3, name='resid3')
        resid4 = self._residual_block(resid3, 3, name='resid4')
        resid5 = self._residual_block(resid4, 3, name='resid5')

        conv_t1 = self._conv_tranpose_layer(resid5, 32, 3, 1, name='convt1')
        cinst4 = self._conditional_instance_norm(
            conv_t1, style_index, 'cinst4')

        conv_t2 = self._conv_tranpose_layer(cinst4, 16, 3, 1, name='convt2')

        conv_t3 = self._conv_layer(conv_t2, 3, 9, 1, relu=False, name='convt3')
        return (tf.nn.tanh(conv_t3)+1)*127.5

    def _reflection_padding(self, net, padding):
        return tf.pad(net, [[0, 0], [padding[0], padding[0]], [padding[1], padding[1]], [0, 0]], "REFLECT")

    def _conv_layer(self, net, num_filters, filter_size, strides, padding='VALID', relu=True, name=None):
        weights_init = self._conv_init_vars(
            net, num_filters, filter_size, name=name)
        strides_shape = [1, strides, strides, 1]
        net = self._reflection_padding(net, (filter_size//2, filter_size//2))
        net = tf.nn.conv2d(net, weights_init, strides_shape, padding=padding)
        net = self._instance_norm(net, name=name)
        if relu:
            net = tf.nn.relu(net)
        return net

    def _conv_tranpose_layer(self, net, num_filters, filter_size, strides, name=None):
        _, rows, cols, _ = [i.value for i in net.get_shape()]
        # Upsample
        net = tf.image.resize_images(
            net, (rows*2, cols*2), tf.image.ResizeMethod.NEAREST_NEIGHBOR)
        return self._conv_layer(net, num_filters, filter_size, strides, name=name)

    def _residual_block(self, net, style_index, filter_size=3, name=None):
        batch, rows, cols, channels = [i.value for i in net.get_shape()]
        tmp = self._conv_layer(net, 32, filter_size, 1,
                               padding='VALID', relu=True, name=name + '_1')
        return self._conv_layer(tmp, 32, filter_size, 1, padding='VALID', relu=False, name=name + '_2') + net

    def _conditional_instance_norm(self, x, style_index, scope_bn):
        with tf.variable_scope(scope_bn, reuse=self.reuse):
            shift = tf.get_variable(name=scope_bn+'shift', shape=[
                                    self.n_styles, x.shape[-1]], initializer=tf.constant_initializer([0.]), trainable=True)  # label_nums x C
            scale = tf.get_variable(name=scope_bn+'scale', shape=[
                                    self.n_styles, x.shape[-1]], initializer=tf.constant_initializer([1.]), trainable=True)  # label_nums x C
        shift = tf.gather(shift, style_index)
        scale = tf.gather(scale, style_index)
        x = self._instance_norm(x, name=scope_bn, shift=shift, scale=scale)
        return x

    def _instance_norm(self, net, name=None, shift=None, scale=None):
        _, _, _, channels = [i.value for i in net.get_shape()]
        var_shape = [channels]
        mu, sigma_sq = tf.nn.moments(net, [1, 2], keep_dims=True)
        if shift == None or scale == None:
            with tf.variable_scope(name, reuse=self.reuse):
                shift = tf.get_variable(
                    'shift', initializer=tf.zeros(var_shape), dtype=tf.float32)
                scale = tf.get_variable(
                    'scale', initializer=tf.ones(var_shape), dtype=tf.float32)
        epsilon = 1e-3
        normalized = (net - mu) / (sigma_sq + epsilon) ** (.5)
        return scale * normalized + shift

    def _conv_init_vars(self, net, out_channels, filter_size, transpose=False, name=None):
        _, _, _, in_channels = [i.value for i in net.get_shape()]
        if not transpose:
            weights_shape = [filter_size, filter_size,
                             in_channels, out_channels]
        else:
            weights_shape = [filter_size, filter_size,
                             out_channels, in_channels]
        with tf.variable_scope(name, reuse=self.reuse):
            weights_init = tf.get_variable('weight', shape=weights_shape,
                                           initializer=tf.contrib.layers.variance_scaling_initializer(),
                                           dtype=tf.float32)
        return weights_init

    def _depthwise_conv_layer(self, net, num_filters, filter_size, strides, padding='SAME', relu=True, channel_mul=1,
                              name=None):
        depthwise_weights_init, pointwise_weights_init = self._depthwiseconv_init_vars(net, num_filters, channel_mul,
                                                                                filter_size, name=name)
        strides_shape = [1, strides, strides, 1]
        net = tf.nn.separable_conv2d(net, depthwise_weights_init, pointwise_weights_init, strides_shape,
                                     padding=padding)
        net = self._instance_norm(net, name=name)
        if relu:
            net = tf.nn.relu(net)
        return net

    def _depthwiseconv_init_vars(self, net, out_channels, channel_multiplier, filter_size, name=None):
        _, _, _, in_channels = [i.value for i in net.get_shape()]
        depthwise_weights_shape = [filter_size,
                                   filter_size, in_channels, channel_multiplier]
        pointwise_weights_shape = [
            1, 1, in_channels * channel_multiplier, out_channels]

        with tf.variable_scope(name, reuse=self.reuse):
            depthwise_weights = tf.get_variable('depthwise_weight', shape=depthwise_weights_shape,
                                                initializer=tf.contrib.layers.variance_scaling_initializer(),
                                                dtype=tf.float32)
            pointwise_weights = tf.get_variable('pointwise_weight', shape=pointwise_weights_shape,
                                                initializer=tf.contrib.layers.variance_scaling_initializer(),
                                                dtype=tf.float32)
        return depthwise_weights, pointwise_weights


# coding: utf-8

from flask import Blueprint, render_template, redirect, url_for, request
from apps.blog.models import Post
from apps.blog.admins.forms import PostForm
from auth import admin_required

mod = Blueprint(
    'admins.blog',
    __name__,
    url_prefix='/admins/blog',
    template_folder='templates'
)

@admin_required
@mod.route('/')
def index():
    posts = Post.query().order(-Post.created)
    return render_template(
        'blog/admins/index.html',
        posts=posts
    )

@admin_required
@mod.route('/post/new/', methods=['GET', 'POST'])
def post_new():
    form = PostForm()
    if form.validate_on_submit():
        post = Post()
        form.populate_obj(post)
        post.put()
        return redirect(url_for('admins.blog.index'))
    return render_template(
        'blog/admins/post_new.html',
        form=form
    )

@admin_required
@mod.route('/post/<int:post_id>/', methods=['GET', 'POST'])
def post(post_id):
    post = Post.get_by_id(post_id)
    if not post:
        return redirect(url_for('admins.blog.index'))
    form = PostForm(obj=post)
    if request.method == 'POST' and 'delete' in request.form:
        post.key.delete()
        return redirect(url_for('admins.blog.index'))
    if form.validate_on_submit():
        form.populate_obj(post)
        post.put()
        return redirect(url_for('admins.blog.index'))
    return render_template(
        'blog/admins/post.html',
        form=form
    )


import numpy as np
import pytest

from pandas.core.dtypes.common import is_float_dtype

from pandas import isna

class TestSetValue:
    def test_set_value(self, float_frame):
        for idx in float_frame.index:
            for col in float_frame.columns:
                float_frame._set_value(idx, col, 1)
                assert float_frame[col][idx] == 1

    def test_set_value_resize(self, float_frame):

        res = float_frame._set_value("foobar", "B", 0)
        assert res is None
        assert float_frame.index[-1] == "foobar"
        assert float_frame._get_value("foobar", "B") == 0

        float_frame.loc["foobar", "qux"] = 0
        assert float_frame._get_value("foobar", "qux") == 0

        res = float_frame.copy()
        res._set_value("foobar", "baz", "sam")
        assert res["baz"].dtype == np.object_

        res = float_frame.copy()
        res._set_value("foobar", "baz", True)
        assert res["baz"].dtype == np.object_

        res = float_frame.copy()
        res._set_value("foobar", "baz", 5)
        assert is_float_dtype(res["baz"])
        assert isna(res["baz"].drop(["foobar"])).all()
        msg = "could not convert string to float: 'sam'"
        with pytest.raises(ValueError, match=msg):
            res._set_value("foobar", "baz", "sam")


# -*- coding: utf-8 -*-
"""
Created on Mon Feb 25 11:47:36 2019

@author: harshvardhan
"""

import urllib.request
import requests

cityName = input("Enter the city name please: ")

url = "http://api.openweathermap.org/data/2.5/weather?q="+cityName+"&appid=e9185b28e9969fb7a300801eb026de9c"
resp = urllib.request.urlopen(url)   # GET request to REST API
print(resp.read())

print(" * "*20)

resp = requests.get(url)
print (resp.text)


import dash
import dash_table
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output, State
import numpy as np
import pandas as pd
import plotly.graph_objects as go
from sklearn.utils import shuffle
from database import get_data_cache, get_all_food_data
import ETL
import random
from app_setup import app, colors

def description():
    return html.Div(children=[dcc.Markdown('''
        # CaloTracker
        This is an interactive tool for you to search and log different foods from our existing
        database of foods. Your caloric total will be divided between food and exercise, with labels
        for each individual entry to really help you break your intake down! Just search for a food
        or exercise using the dropdown bars, and the full info will appear in the table. Change
        your serving quantity and time spend working out straight in the table, as well; the total
        calories will automatically update in the table and in the graph. And, if you can't find the
        food or exercise that you're looking for, query it yourself from our API, and the dropdown
        bars will contain any new results. Happy logging!''',
        style={'paddingLeft': '5%', 'paddingRight': '5%', 'marginTop': 50, 'marginBottom': 25
        })])

def food_log():
    df = shuffle(get_all_food_data(False))
    opts = df['food_name'].unique()
    #data = df.to_dict('records')
    return html.Div([
        dash_table.DataTable(
            id='foods-table',
            columns=[{
                "name": 'Food Name', "id": 'food_name'},
                {"name": 'Brand Name', "id": 'brand_name'},
                {"name": 'Serving Unit', "id": 'serving_unit'},
                {"name": 'Serving Quantity', "id": 'serving_qty', 'editable': True},
                {"name": 'Total Calories', "id": 'nf_calories'}],
            data=[],
            row_deletable=True,
            fixed_rows={ 'headers': True, 'data': 0 },
            style_cell={'minWidth': '65px',
                'width': '65px',
                'maxWidth': '65px',
                'overflow': 'hidden',
                'textOverflow': 'ellipsis'},
            style_table={
                'maxHeight': '300px',
                'overflowY': 'scroll',
                'margin-left': '7px',
                'padding-right': '7%'
                },
            filter_action="native",
            sort_action="native",
            sort_mode='multi',
            css=[{ 'selector': 'td.cell--selected, td.focused',
                'rule': 'background-color: #D9D9DA !important;'},
                { 'selector': 'td.cell--selected *, td.focused *',
                'rule': 'color: #FC6D41 !important;'}],
            style_header={'backgroundColor': '#A9BA9C',
                'fontWeight': 'bold',
                'textAlign': 'center'},
            style_data_conditional=[{
                'if': {'row_index': 'odd'},
                'backgroundColor': '#D9D9DA'}
                ]),
        html.Br(),
        html.Div([dcc.Dropdown(
            id='food-log',
            options=[{'label': i, 'value': i} for i in opts],
            placeholder="Select a food")],
            style={'margin-left': '11%', 'color': '#274228', 'width': '80%'}),
        html.Div([html.Button('Add Food',
            id='adding-food-button',
            n_clicks=0)],
            style={'margin-left': '11%', 'display': 'inline-block', 'color': '#274228'})],
        style={'width': '48%', 'display': 'inline-block'})

def exer_log():
    db = get_data_cache(allow_cached=True)
    df = shuffle(db[2])
    #data = df.to_dict('records')
    opts = df['name'].unique()
    return html.Div([
        dash_table.DataTable(
            id='exer-table',
            columns=[{
                "name": 'Exercise Name', "id": 'exer_name'},
                {"name": 'Duration (Mins)', "id": 'duration_min', 'editable': True},
                {"name": 'Calories Burned', "id": 'cals_burned'}],
            data=[],
            fixed_rows={ 'headers': True, 'data': 0 },
            style_cell={'minWidth': '65px',
                'width': '65px',
                'maxWidth': '65px',
                'overflow': 'hidden',
                'textOverflow': 'ellipsis',
                'fontColor': '#274228'},
            style_table={
                'maxHeight': '300px',
                'overflowY': 'scroll',
                'margin-right': '5%',
                'padding-left': '9%'
                },
            row_deletable=True,
            filter_action="native",
            sort_action="native",
            sort_mode='multi',
            style_data_conditional=[{
                'if': {'row_index': 'odd'},
                'backgroundColor': '#D9D9DA'}
                ],
            style_header={'backgroundColor': '#A9BA9C',
                'fontWeight': 'bold',
                'textAlign': 'center'},
            css=[{ 'selector': 'td.cell--selected, td.focused',
            'rule': 'background-color: #D9D9DA !important;'},
            { 'selector': 'td.cell--selected *, td.focused *',
            'rule': 'color: #FC6D41 !important;'}]
        ),
        html.Br(),
        html.Div([dcc.Dropdown(
            id='exer-log',
            options=[{'label': i, 'value': i} for i in opts],
            placeholder="Select an exercise")],
            style={'margin-left': '17%', 'color': '#274228'}),
        html.Div([html.Button('Add Exercise',
            id='adding-exercise-button',
            n_clicks=0)],
            style={'margin-left': '17%',
            'display': 'inline-block', 'color': '#274228', 'fontColor': '#A9BA9C'})],
        style={'width': '48%', 'display': 'inline-block'})

def food_search():
    """"""
    return html.Div([
        dcc.Input(id='food-search', type='text', value=''),
        html.Button('Find my food!', id='food-submit', n_clicks=0,
        style={'margin-left': '5px'}),
        html.P(id='food-output')],
        style={'display': 'inline-block',
        'align-items': 'center',
        'margin-left': '14%',
        'margin-right': '5%'}
    )

def exer_search():
    """"""
    return html.Div([
        dcc.Input(id='exer-name-search', type='text', value=''),
        html.Button('Find my exercise!', id='exer-submit', n_clicks=0,
        style={'margin-left': '5px'}),
        html.P(id='exer-output')],
        style={'display': 'inline-block',
        'align-items': 'center',
        'margin-right': '14%',
        'margin-left': '5%'}
    )

def dynamic_layout():
    return html.Div([
        description(),
        dcc.Graph(id='log-graph', style={'width': '90%', 'margin-left': '5%', 'margin-right': '5%'}),
        food_log(),
        exer_log(),
        html.Br(),
        html.Br(),
        dcc.Markdown('''Couldn't find what you were looking for? Try searching the API with the name of a food,
        exercise,or a list of either separated by a common and space (and please be patient)! Foods go in the
        left search bar, and exercises on the right.''', style={'paddingLeft': '5%', 'paddingRight': '5%'}),
        food_search(),
        exer_search(),
        html.Br(),
        dcc.Link('Return Home', href='/app',
        style={'color': '#274228'}),
        html.Br(),
        dcc.Link('Go to CaloPlanner', href='/apps/app2',
        style={'color': '#274228'})],
        id='app1-content')

layout = dynamic_layout()

@app.callback(
    Output('foods-table', 'data'),
    [Input('adding-food-button', 'n_clicks'),
    Input('foods-table', 'data_timestamp')],
    [State('food-log', 'value'),
    State('foods-table', 'data'),
    State('foods-table', 'columns')])
def update_food_row(n_clicks, serving, food, rows, columns):

    df = get_all_food_data(True)
    info = df[df['food_name']==food]

    def add_row(r):
        if len(info) == 0:
            pass
        else:
            rowdict = info.to_dict('records')[0]
            vals = rowdict.values()
            r.append({c['id']: v for (c,v) in zip(columns, vals)})
        return r

    if len(rows) == 0:
        add_row(rows)

    else:
        test = [list(r.values()) for r in rows]
        flat_list = [item for sublist in test for item in sublist]
        if food in flat_list:
            pass
        else:
            add_row(rows)

    for r in rows:
        update = df[df['food_name'] == r['food_name']]
        rowdict = update.to_dict('records')[0]
        temp1 = rowdict['nf_calories']
        temp2 = rowdict['serving_qty']
        r['nf_calories'] = float(temp1)/float(temp2) * float(r['serving_qty'])
    return rows

@app.callback(
    Output('exer-table', 'data'),
    [Input('adding-exercise-button', 'n_clicks'),
    Input('exer-table', 'data_timestamp')],
    [State('exer-log', 'value'),
    State('exer-table', 'data'),
    State('exer-table', 'columns')])
def update_exer_row(n_clicks, duration, exercise, rows, columns):

    db = get_data_cache(allow_cached=True)
    df = db[2]
    info = df[df['name']==exercise]

    def add_erow(r):
        if len(info) == 0:
            pass
        else:
            rowdict = info.to_dict('records')[0]
            vals = rowdict.values()
            r.append({c['id']: v for (c,v) in zip(columns, vals)})
        return r

    if len(rows) == 0:
        add_erow(rows)

    else:
        test = [list(r.values()) for r in rows]
        flat_list = [item for sublist in test for item in sublist]
        if exercise in flat_list:
            pass
        else:
            add_erow(rows)

    for r in rows:
        update = df[df['name'] == r['exer_name']]
        rowdict = update.to_dict('records')[0]
        temp1 = rowdict['nf_calories']
        temp2 = rowdict['duration_min']
        r['cals_burned'] = float(temp1)/float(temp2) * float(r['duration_min'])
    return rows

@app.callback(
    Output('log-graph', 'figure'),
    [Input('foods-table', 'derived_viewport_data'),
     Input('exer-table', 'derived_viewport_data')],
     [State('foods-table', 'columns'),
     State('exer-table', 'columns')])
def display_output(frows, erows, fcols, ecols):

    fcals = []
    ecals = []
    ftraces = []
    etraces = []

    if frows is None:
        pass
    else:
        for f in frows:
            trace = go.Bar(name = f['food_name'],
                x = ['Food'],
                y = [f['nf_calories']],
                marker_color = colors[random.randint(1,9)])
            ftraces.append(trace)
            fcals.append(f['nf_calories'])

    if erows is None:
        pass
    else:
        for e in erows:
            trace = go.Bar(name = e['exer_name'],
                x = ['Exercise'],
                y = [e['cals_burned']],
                marker_color = colors[random.randint(1,9)])
            etraces.append(trace)
            ecals.append(e['cals_burned'])

    total = np.sum(fcals) - np.sum(ecals)
    totaltrace = [go.Bar(name = 'Total Caloric Intake',
                    x = ['Total'],
                    y = [total],
                    marker_color = colors[random.randint(1,9)])]

    traces = ftraces + etraces + totaltrace

    return {
        'data': traces,
        'layout': go.Layout(barmode='stack',
                    title='Personal Calorie Log',
                    font = dict(color = '#274228'),
                    yaxis = dict(title='Number of Calories'),
                    showlegend=False)}

@app.callback(
    Output('food-output', 'children'),
    [Input('food-submit', 'n_clicks')],
    [State('food-search', 'value')]
)
def foodapi(click, term):
    if click == 0:
        return
    else:
        hits = ETL.user_food_query(term)
        return u'Your query returned {} results!'.format(hits)

@app.callback(
    Output('exer-output', 'children'),
    [Input('exer-submit', 'n_clicks')],
    [State('exer-name-search', 'value')]
)
def exerapi(click, term):
    if click == 0:
        return
    else:
        hits = ETL.user_exercise_query(term)
        return u'Your query returned {} results!'.format(hits)

@app.callback(
    Output('exer-log', 'options'),
    [Input('exer-output', 'children')]
)
def update_exer_drop(refresh):
    df = shuffle(get_data_cache()[2])
    opts = df['name'].unique()
    return [{'label': i, 'value': i} for i in opts]

@app.callback(
    Output('food-log', 'options'),
    [Input('food-output', 'children')]
)
def update_food_drop(refresh):
    df = shuffle(get_all_food_data(False))
    opts = df['food_name'].unique()
    return [{'label': i, 'value': i} for i in opts]

# if __name__ == '__main__':
#     app.run_server(debug=True, port=1050, host='0.0.0.0')


from imports import  *
#TO DO fix slow work of Boruta

def get_permutation_importances(X, y, params,features,shuffle=True,categorical_feats=None):
    # Go over fold and keep track of CV score (train and valid) and feature importances
    # Shuffle target if required
    y = y.copy()
    if shuffle:
        # Here you could as well use a binomial distribution
        y = y.copy().sample(frac=1.0)
    # Fit LightGBM in RF mode, yes it's quicker than sklearn RandomForest
    dtrain = lgb.Dataset(X, y, free_raw_data=False, silent=True)
    # Fit the model
    if categorical_feats is not None:
        clf = lgb.train(params=params, train_set=dtrain, num_boost_round=200, categorical_feature=categorical_feats)
    else:
        clf = lgb.train(params=params, train_set=dtrain, num_boost_round=200)
    # Get feature importances
    imp_df = pd.DataFrame()
    imp_df["feature"] = list(features)
    imp_df["importance_gain"] = clf.feature_importance(importance_type='gain')
    imp_df["importance_split"] = clf.feature_importance(importance_type='split')
    imp_df['trn_score'] = roc_auc_score(y, clf.predict(X))

    return imp_df

def get_boruta_importances(X, y, features,seed=None,fast=False,n_fold=20):

    start_time = timer(None)
    if fast:
        rfc = RandomForestRegressor(n_estimators=10, n_jobs=4,
                                 max_depth=3,max_features=0.5)
        boruta_selector = BorutaPy(rfc, n_estimators=100, verbose=2)
        folds = KFold(n_splits=n_fold, shuffle=True, random_state=11)
        (train_index, valid_index)=list(folds.split(X[features],y))[0]
        boruta_selector.fit(X[features].iloc[train_index].values, y.iloc[train_index].values)
    else:
        rfc = RandomForestRegressor(n_estimators=200, n_jobs=4, class_weight='balanced',
                                 max_depth=6)
        boruta_selector = BorutaPy(rfc, n_estimators='auto', verbose=2)
        boruta_selector.fit(X[features].values, y.values)
    timer(start_time)
    # Get feature importances
    imp_df = pd.DataFrame()
    imp_df["feature"] = list(features)
    imp_df['rank']=boruta_selector.ranking_
    imp_df = imp_df.sort_values('rank', ascending=True).reset_index(drop=True)
    return imp_df

def get_selection_stability(X, y, names, order=1):
    rlasso = RandomizedLasso(alpha=0.04)
    rlasso.fit(X, y)
    ranks=np.abs(rlasso.scores_)
    minmax = MinMaxScaler()
    ranks = minmax.fit_transform(order*np.array([ranks]).T).T[0]
    imp_df = pd.DataFrame()
    imp_df["feature"] = list(names)
    ranks = map(lambda x: round(x,2), ranks)
    imp_df['rank']=ranks
    return imp_df

def get_feature_ranking_matrix(X, y, names, order=1):
    # Using Linear Regression
    lr = LinearRegression(normalize=True)
    lr.fit(X,y)
    imp_df = pd.DataFrame()
    imp_df["feature"] = list(names)
    imp_df["LinReg"] = ranking(np.abs(lr.coef_))
    # Using Ridge
    ridge = Ridge(alpha = 7)
    ridge.fit(X,y)
    imp_df['Ridge'] = ranking(np.abs(ridge.coef_))
    # Using Lasso
    lasso = Lasso(alpha=.05)
    lasso.fit(X, y)
    imp_df["Lasso"] = ranking(np.abs(lasso.coef_))
    #Random Forest feature ranking
    rf = RandomForestRegressor(n_jobs=-1, n_estimators=50, verbose=3)
    rf.fit(X,y)
    imp_df["RF"] = ranking(rf.feature_importances)
    # Create empty dictionary to store the mean value calculated from all the scores
    imp_df["mean"]=np.mean(imp_df.iloc[:,1:].values,axis=1)
    return imp_df

def ranking(ranks, order=1):
    minmax = MinMaxScaler()
    ranks = minmax.fit_transform(order*np.array([ranks]).T).T[0]
    ranks = map(lambda x: round(x,2), ranks)
    return ranks

def timer(start_time=None):
    if not start_time:
        start_time = datetime.now()
        return start_time
    elif start_time:
        thour, temp_sec = divmod((datetime.now() - start_time).total_seconds(), 3600)
        tmin, tsec = divmod(temp_sec, 60)
        print('\n Time taken: %i hours %i minutes and %s seconds.' % (thour, tmin, round(tsec, 2)))