"""
Created on Sat September 16 21:15:06 2023

Description: Data preprocessing functions for:
Paper url: https://www.biorxiv.org/content/10.1101/2023.09.11.557230v2

@author: Ekaterina Kuzmina, ekaterina.kuzmina@skoltech.ru
@author: Dmitrii Kriukov, dmitrii.kriukov@skoltech.ru
"""

import numpy as np
import h5py
import mat73
from scipy.ndimage import gaussian_filter1d

def load_h5_file(path, filename):
    with h5py.File(path+filename, 'r') as h5file:
        ans = {}
        for key, item in h5file.items():
            ans[key] = item[()]
    return ans

def save_h5_file(path, filename, data):
    with h5py.File(path + filename, 'w') as h5file:
        for key, item in data.items():
            h5file[key] = item
    print('File {} successfully saved!'.format(filename))

### Grasping Data Processing ###

# Divide functions for Reachings and Graspings
## Reachings
def extract_and_resample(spikes, fs=0.01, max_len=2800):
    # 1. Extract data excluding None
    # from data_dict['spikes'] into list (n_units x 1)
    xspikes = []
    for s_arr in spikes:
        if s_arr[0] is not None:
            for i in range(len(s_arr[0])):
                if s_arr[0][i] is not None:
                    xspikes.append(s_arr[0][i])

    # 2. Resample with fs sampling rate
    time = np.arange(0., max_len, fs)
    data = np.zeros((len(xspikes), time.shape[0]))
    for n_idx, neuron in enumerate(xspikes):
        hist = np.zeros(time.shape)
        for spike_time in neuron:
            t_strt = int(spike_time // fs)
            t_end = t_strt + 1

            in_range = time[t_strt] <= spike_time < time[t_end]
            if in_range:
                hist[t_strt] += 1
            else:
                hist[t_end] += 1
        data[n_idx] = hist
    return data

def cut_condition_wise(data, go_cue, instruction_cue, move_start,
                       move_end, fs=0.01, max_len=2800):

    time = np.arange(0., max_len, fs)
    cond_num = len(instruction_cue)
    for_each_cond = [instruction_cue[i].shape[0] for i in range(cond_num)]

    cond_dict = {}
    for cc in range(cond_num):
        one_cond_trials = []
        for c_idx in range(for_each_cond[cc]):
            gc = go_cue[cc][c_idx]
            gc_idx = int(gc // fs)

            tstrt, tend = instruction_cue[cc][c_idx], move_end[cc][c_idx]
            # +1 we round the top border;
            # +1 in extract_and_resample [t_strt,t_end), so we shifted spikes one step ahead
            tstrt_idx, tend_idx = int(tstrt // fs), int(tend // fs) + 2

            crop = data[:, tstrt_idx : tend_idx]
            new_gc_idx = gc_idx - tstrt_idx
            one_cond_trials.append([crop, new_gc_idx])

        cond_dict[str(cc)] = one_cond_trials

    return cond_dict

def align_pad_smooth(cond_dict, sigma=10, signal_len='min'):
    from scipy.ndimage import gaussian_filter1d

    aligned_cond_dict = {} # dict of aligned data
    cond_gc_dict = {} # dict of idx of aligment point

    for cond_num, trials_list in cond_dict.items():
        # collect all go cue idxs
        gc_start = [trials_list[i][1] for i in range(len(trials_list))]

        # the most frequent go cue idx
        values, counts = np.unique(gc_start, return_counts=True)
        gc_align = values[np.argmax(counts)]

        # find the idx of signals where go cue is not the most frequent one
        idx_to_align = np.argwhere(gc_start != gc_align)
        idx_to_align = idx_to_align.reshape((idx_to_align.shape[0]))

        # calculate difference and shift signal to align go cue
        aligned_trials = [trials_list[i][0] for i in range(len(trials_list))]
        for idx in idx_to_align:
            signal = trials_list[idx][0]
            diff = gc_start[idx] - gc_align

            if np.sign(diff) == 1:
                # remove diff number of elems from beginning
                aligned_signal = signal[:, diff:]

            elif np.sign(diff) == -1:
                # add diff number of zeros to beginning
                aligned_signal = np.concatenate((np.zeros((signal.shape[0], np.abs(diff))),
                                                 signal), axis=1)
            aligned_trials[idx] = aligned_signal

        # calc max and min lens of signals
        lens = [aligned_trials[i].shape[1] for i in range(len(aligned_trials))]
        signal_max_len, signal_min_len = max(lens), min(lens)

        trials_list_padded = []
       # pad to maximal len
        if signal_len == 'max':
            for trial in aligned_trials:
                to_pad = np.zeros((trial.shape[0],
                                   signal_max_len - trial.shape[1]))
                trials_list_padded.append(np.concatenate((trial, to_pad),
                                                         axis=1))
        # OR cut to the shortest len
        elif signal_len == 'min':
            for trial in aligned_trials:
                trials_list_padded.append(trial[:, :signal_min_len])

        # average by trials
        averaged_trials = np.mean(np.asarray(trials_list_padded), axis=0)

        # smooth with gauss 25 ms s.d.
        smthd_avrg_trial = []
        for ntrial in averaged_trials:
            smthd_avrg_trial.append(gaussian_filter1d(ntrial, sigma=sigma,
                                                      mode='constant'))
        aligned_cond_dict[str(cond_num)] = np.asarray(smthd_avrg_trial)
        cond_gc_dict[str(cond_num)] = gc_align

    return aligned_cond_dict, cond_gc_dict

def align_mean_conds(cond_dictq, gc_dict):

    cond_num = len(gc_dict.keys())
    neuron_num = cond_dictq['2'].shape[0]

    # collect all go cue idxs
    gc_start = [gc_dict[str(i)] for i in range(cond_num)]

    # the most frequent go cue idx
    values, counts = np.unique(gc_start, return_counts=True)
    gc_align = values[np.argmax(counts)]

    # find the idx of signals where go cue is not the most frequent one
    idx_to_align = np.argwhere(gc_start != gc_align)
    idx_to_align = idx_to_align.reshape((idx_to_align.shape[0]))

    avrg_aligned_conds = [cond_dictq[str(i)] for i in range(cond_num)]
    for cond_idx in idx_to_align:
        cond_signal = cond_dictq[str(cond_idx)]
        diff = gc_start[cond_idx] - gc_align

        if np.sign(diff) == 1:
            # remove diff number of elems from beginning
            aligned_signal = cond_signal[:, diff:]

        elif np.sign(diff) == -1:
            # add diff number of zeros to beginning
            aligned_signal = np.concatenate((np.zeros((cond_signal.shape[0],
                                                       np.abs(diff))),
                                             cond_signal), axis=1)
        avrg_aligned_conds[cond_idx] = aligned_signal

    min_len = min([avrg_aligned_conds[i].shape[1] for i in range(cond_num)])
    processed_data = np.empty((cond_num, min_len, neuron_num))
    for condn in range(cond_num):
        signal = avrg_aligned_conds[condn]
        processed_data[condn] = signal[:, :min_len].T

    return processed_data, gc_align

# For 6th dataset
def sort_cues(cond_times_dict, instr_cue, go_cue, ms_cue, me_cue):

    # Which instruction cue for which cond
    cond_num = len(cond_times_dict.keys())
    sort_instr_cue = {x:[] for x in range(cond_num)}
    cond_lens = {x: cond_times_dict[x].shape[0] for x in range(cond_num)}

    for instr_time in instr_cue:
        cond_mins = []
        for cond_idx in range(cond_num):
            diff = np.abs(np.asarray(cond_times_dict[cond_idx] - instr_time))
            cond_mins.append(np.amin(diff))

        cond_min_idx = np.where(cond_mins == np.amin(cond_mins))[0][0]
        sort_instr_cue[cond_min_idx].append(instr_time)

    # Sort go_cue, ms_cue, me_cue according to instr_cue
    go_cell = [[] for i in range(cond_num)]
    instr_cell = [[] for i in range(cond_num)]
    stmv_cell = [[] for i in range(cond_num)]
    endmv_cell = [[] for i in range(cond_num)]

    for gc_time, ms_time, me_time in zip(go_cue, ms_cue, me_cue):
        cond_mins = []
        in_cond_mins_idx = []

        for cond_idx in range(cond_num):
            diff = np.abs(np.asarray(sort_instr_cue[cond_idx] - gc_time))
            cond_mins.append(np.amin(diff))
            in_cond_mins_idx.append(np.where(diff == np.amin(diff))[0][0])

        cond_min_idx = np.where(cond_mins == np.amin(cond_mins))[0][0]
        instr_idx = in_cond_mins_idx[cond_min_idx]
        instr = sort_instr_cue[cond_min_idx][instr_idx]

        go_cell[cond_min_idx].append(gc_time)
        instr_cell[cond_min_idx].append(instr)
        stmv_cell[cond_min_idx].append(ms_time)
        endmv_cell[cond_min_idx].append(me_time)

    go_cell_f = [np.asarray(j) for j in go_cell]
    instr_cell_f = [np.asarray(j) for j in instr_cell]
    stmv_cell_f = [np.asarray(j) for j in stmv_cell]
    endmv_cell_f = [np.asarray(j) for j in endmv_cell]

    return go_cell_f, instr_cell_f, stmv_cell_f, endmv_cell_f

def extract_and_process_grasp(neural_data, data_dict, align_cue,
                              fixed_align_cue=None,
                              signal_len='max', sigma=10):
    """
    neural_data: list of [times x neurons] arrays
    """
    cond_arange = np.unique(data_dict['ObjectInd'])
    cond_seq = data_dict['ObjectInd']
    all_conds_prep = {}
    aligning_cue = {}

    for cond_idx in cond_arange:
        idxs = np.argwhere(cond_seq == cond_idx)
        idxs = idxs.reshape((idxs.shape[0]))

        trials, time = [], []
        wsm_idx, fg_idx = [], []

        for i in idxs:
            trials.append(neural_data[i])
            time.append(data_dict['TimeBins'][i][0])

            # WristStartMove -> MaxApp -> FingerGrasp
            wsm = data_dict['EventTimes']['WristStartMove'][i]
            fg = data_dict['EventTimes']['FingerGrasp'][i]

            diff = np.abs(data_dict['TimeBins'][i][0] - wsm)
            wsm_idx.append(np.where(diff == np.amin(diff))[0][0])
            diff = np.abs(data_dict['TimeBins'][i][0] - fg)
            fg_idx.append(np.where(diff == np.amin(diff))[0][0])

        # choose according to which cue align
        if align_cue == 'wsm':
            al_idx = wsm_idx
        elif align_cue == 'fg':
            al_idx = fg_idx

        v, c = np.unique(al_idx, return_counts=True)
        al_cue = v[np.argmax(c)] if v[np.argmax(c)] != 0 else int(np.mean(v))

        # We might hard code al_cue
        if fixed_align_cue:
            al_cue = fixed_align_cue

        # find the idx of signals where go cue is NOT aligning
        idx_to_align = np.argwhere(np.asarray(al_idx) != al_cue)
        idx_to_align = idx_to_align.reshape((idx_to_align.shape[0]))

        # calculate difference and shift signal to align go cue
        aligned_trials = [trials[j] for j in range(len(trials))]
        for i2a in idx_to_align:
            diff = al_idx[i2a] - al_cue

            if np.sign(diff) == 1:
                # remove diff number of elems from beginning
                aligned_signal = trials[i2a][diff:, :]

            elif np.sign(diff) == -1:
                # add diff number of zeros to beginning
                aligned_signal = np.concatenate((np.zeros((np.abs(diff),
                                                           trials[i2a].shape[1])),
                                                 trials[i2a]), axis=0)
            aligned_trials[i2a] = aligned_signal

        # calc max and min lens of signals
        lens = [aligned_trials[i].shape[0] for i in range(len(aligned_trials))]
        signal_max_len, signal_min_len = max(lens), min(lens)

        trials_list_padded = []
       # pad to maximal len
        if signal_len == 'max':
            for trial in aligned_trials:
                trial = trial.T
                to_pad = np.zeros((trial.shape[0],
                                   signal_max_len - trial.shape[1]))
                trials_list_padded.append(np.concatenate((trial, to_pad),
                                                         axis=1))
        # OR cut to the shortest len
        elif signal_len == 'min':
            for trial in aligned_trials:
                trial = trial.T
                trials_list_padded.append(trial[:, :signal_min_len])

        # average by trials
        averaged_trials = np.mean(np.asarray(trials_list_padded), axis=0)

        # smooth with gauss 25 ms s.d.
        smthd_avrg_trial = []
        for ntrial in averaged_trials:
            smthd_avrg_trial.append(gaussian_filter1d(ntrial, sigma=sigma,
                                                      mode='constant'))
        all_conds_prep[str(int(cond_idx) - 1)] = np.asarray(smthd_avrg_trial)
        aligning_cue[str(int(cond_idx) - 1)] = al_cue

    return all_conds_prep, aligning_cue

def load_grasping_dataset(didx):
    from utils.config import grasp_d_names, grasp_d_params, GRASP_DATA_PATH, DEXT

    d_dict = load_h5_file(GRASP_DATA_PATH, grasp_d_names[didx]+DEXT)
    cut_idx = grasp_d_params[grasp_d_names[didx]]['cutoffs']

    datas = d_dict['data'][:, cut_idx[0]:cut_idx[1], :]
    times = d_dict['time'][:datas.shape[1]]
    go_cue = d_dict['align_cue_idx'] - cut_idx[0]

    return [i for i in datas], list(times), go_cue

### LFP Data Processing ###

def extract_and_process_grasp_LFP(neural_data, dd, signal_len='max', sigma=10):
    """
    neural_data: list of [times x neurons] arrays
    """
    all_conds_prep = {}
    aligning_cue = {}

    cond_arange = np.unique(dd['tgtDir'])
    cond_seq = dd['tgtDir']
    for cond_idx in cond_arange:
        idxs = np.argwhere(cond_seq == cond_idx)
        idxs = idxs.reshape((idxs.shape[0]))

        trials = []
        start, end = [], []
        go_cue, start_move = [], []
        show_target = []

        for i in idxs:
            trials.append(neural_data[i])

            start.append(dd['idx_trial_start'][i])
            show_target.append(dd['idx_tgtOnTime'][i])
            go_cue.append(dd['idx_goCueTime'][i])
            start_move.append(dd['idx_movement_on'][i])
            end.append(dd['idx_endTime'][i])

        v, c = np.unique(go_cue, return_counts=True)
        al_cue = v[np.argmax(c)] if v[np.argmax(c)] != 0 else int(np.mean(v))

        # find the idx of signals where go cue is NOT aligning
        idx_to_align = np.argwhere(np.asarray(go_cue) != al_cue)
        idx_to_align = idx_to_align.reshape((idx_to_align.shape[0]))

        # calculate difference and shift signal to align go cue
        aligned_trials = [trials[j] for j in range(len(trials))]
        for i2a in idx_to_align:
            diff = int(go_cue[i2a] - al_cue)

            if np.sign(diff) == 1:
                # remove diff number of elems from beginning
                aligned_signal = trials[i2a][diff:, :]

            elif np.sign(diff) == -1:
                # add diff number of zeros to beginning
                aligned_signal = np.concatenate((np.zeros((np.abs(diff),
                                                            trials[i2a].shape[1])),
                                                    trials[i2a]), axis=0)
            aligned_trials[i2a] = aligned_signal

        # calc max and min lens of signals
        lens = [aligned_trials[i].shape[0] for i in range(len(aligned_trials))]
        signal_max_len, signal_min_len = max(lens), min(lens)

        trials_list_padded = []
        # pad to maximal len
        if signal_len == 'max':
            for trial in aligned_trials:
                trial = trial.T
                to_pad = np.zeros((trial.shape[0],
                                    signal_max_len - trial.shape[1]))
                trials_list_padded.append(np.concatenate((trial, to_pad),
                                                            axis=1))
        # OR cut to the shortest len
        elif signal_len == 'min':
            for trial in aligned_trials:
                trial = trial.T
                trials_list_padded.append(trial[:, :signal_min_len])

        # average by trials
        averaged_trials = np.mean(np.asarray(trials_list_padded), axis=0)

        # smooth with gauss
        smthd_avrg_trial = []
        for ntrial in averaged_trials:
            smthd_avrg_trial.append(gaussian_filter1d(ntrial, sigma=sigma,
                                                        mode='constant'))
        all_conds_prep[cond_idx] = np.asarray(smthd_avrg_trial)
        aligning_cue[cond_idx] = int(al_cue)

    return all_conds_prep, aligning_cue, cond_arange

def align_mean_conds_LFP(cond_dictq, gc_dict):

    cond_num = len(gc_dict.keys())
    gc_start_key = list(gc_dict.keys())
    neuron_num = cond_dictq[gc_start_key[0]].shape[0]

    # collect all go cue idxs
    gc_start = [gc_dict[i] for i in gc_start_key]

    # the most frequent go cue idx
    values, counts = np.unique(gc_start, return_counts=True)
    gc_align = values[np.argmax(counts)]

    # find the idx of signals where go cue is not the most frequent one
    idx_to_align = np.argwhere(gc_start != gc_align)
    idx_to_align = idx_to_align.reshape((idx_to_align.shape[0]))

    avrg_aligned_conds = [cond_dictq[i] for i in gc_start_key]
    for cond_idx in idx_to_align:
        cond_signal = cond_dictq[gc_start_key[cond_idx]]
        diff = gc_start[cond_idx] - gc_align

        if np.sign(diff) == 1:
            # remove diff number of elems from beginning
            aligned_signal = cond_signal[:, diff:]

        elif np.sign(diff) == -1:
            # add diff number of zeros to beginning
            aligned_signal = np.concatenate((np.zeros((cond_signal.shape[0],
                                                       np.abs(diff))),
                                             cond_signal), axis=1)
        avrg_aligned_conds[cond_idx] = aligned_signal

    min_len = min([avrg_aligned_conds[i].shape[1] for i in range(cond_num)])
    processed_data = np.empty((cond_num, min_len, neuron_num))
    for condn in range(cond_num):
        signal = avrg_aligned_conds[condn]
        processed_data[condn] = signal[:, :min_len].T

    return processed_data, gc_align

def lfp_dataset_processing(path, plot_all=False):
    print(path)
    data_dict = mat73.loadmat(path)
    dd = data_dict['trial_data']
    k = dd.keys()
    labels = ['M1', 'PMd', 'LeftS1Area2']
    in_labels = []
    for l in labels:
        ll = l + '_spikes'
        if ll in k:
            in_labels.append(l)

    data_to_return = {}
    for l in in_labels:
        print('For ', l)

        neural_data = dd[l + '_spikes']
        a, g, _ = extract_and_process_grasp_LFP(neural_data,
                                                dd, signal_len='max',
                                                sigma=12)
        print(a.keys())
        # 2. Выровнять по go cue средние кондишоныh
        proc, gc = align_mean_conds_LFP(a, g)
        print(proc.shape, gc)
        data_to_return[l] = [proc, gc]

    return data_to_return


"""
For reading and writing crap.
"""

import argparse
import csv
from typing import Tuple, List, Dict
import os
import time

import torch
from tqdm import tqdm

import constants  # for speed test script; not needed for data IO
import viewer

# lib functions

def which_exist(filenames: List[str]) -> List[str]:
    """
    Returns the subset of filenames that exist on disk.
    """
    return list(filter(lambda f: os.path.exists(f), filenames))

def split_tensor(
    data: torch.Tensor, classes: int = 1
) -> Tuple[torch.IntTensor, torch.FloatTensor]:
    """
    Splits 'all data' tensor into labels and features tensors.

    Returns 2-tuple of:
        (1) either a 1d (N) vector (if classes == 1),
                or a 2d (N x L) matrix (if classes > 1)
        (2) a 2d (N x D) matrix of datums x features
    """
    # have to be careful when selecting one column:
    # - selecting data[:, 0]  gives a 1d (N) vector
    # - selecting data[:, :1] gives a 2d (N x 1) matrix
    if classes == 1:
        labels = data[:, 0].type(torch.IntTensor)
    else:
        labels = data[:, :classes].type(torch.IntTensor)
    features = data[:, classes:]
    return labels, features

def csv_to_tensor(filename: str) -> torch.Tensor:
    """
    Loads all data from filename in csv format; returns in a single tensor.
    """
    with open(filename, "r") as f:
        rows = [r for r in csv.reader(f, quoting=csv.QUOTE_NONNUMERIC)]
    return torch.Tensor(rows)

def csv_to_tensors(filename: str) -> Tuple[torch.IntTensor, torch.FloatTensor]:
    """
    Loads data from filename in csv format; return label (col 0) and features
    (rest) tensors.
    """
    return split_tensor(csv_to_tensor(filename))

def tensor_to_csv(t: torch.Tensor, filename: str) -> None:
    """
    Writes tensor t to filename on disk in csv format, creating parent directories as needed.
    """
    os.makedirs(os.path.dirname(filename), exist_ok=True)
    with open(filename, "w") as f:
        w = csv.writer(f)
        w.writerows(t)

def tensor_to_bin(t: torch.Tensor, filename: str) -> None:
    """
    Writes tensor t to filename on disk in torch.save(...) format, creating
    directories as needed.
    """
    os.makedirs(os.path.dirname(filename), exist_ok=True)
    torch.save(t, filename)

def bin_to_tensor(filename: str) -> torch.Tensor:
    """
    Loads tensor from disk at filename in torch.save(...) format.
    """
    return torch.load(filename)

def bin_to_tensors(
    filename: str, classes: int = 1
) -> Tuple[torch.IntTensor, torch.FloatTensor]:
    """
    Loads data from filename in torch.save(...) format; return label
    (classes) and features (rest) tensors.

    Returns 2-tuple of:
        (1) either a 1d (N) vector (if classes == 1),
                or a 2d (N x L) matrix (if classes > 1)
        (2) a 2d (N x D) matrix of datums x features
    """
    return split_tensor(bin_to_tensor(filename), classes)

def bias_tensor(t: torch.Tensor) -> torch.Tensor:
    """
    Adds bias column (all 1s) to `t`.

    Arguments:
        t 2D (N x D) tensor

    Retuns:
          2D (N x D+1) tensor, with final bias column (all 1s)
    """
    n = len(t)
    bias_col = torch.ones(n)
    return torch.cat([t, bias_col], dim=1)

def labels_to_onehot(labels: torch.IntTensor, classes: int) -> torch.IntTensor:
    """
    Turns 1d (N) class-label tensor `labels` into 2d (N x classes) onehot
    tensor and returns it.

    Arguments:
        labels: 1d (N) vector of class labels
        classes: number of class label options (will be output cols)

    Returns:
        2d (N x classes) onehot label matrix
    """
    n, = labels.size()
    label_idx = labels.type(torch.LongTensor).view(-1, 1)
    return torch.IntTensor(n, classes).zero_().scatter_(1, label_idx, 1)

# script

def speedtest(n: int = 3) -> None:
    """
    Tests speed of loading csv vs torch.load(...) on a tensor. Does `n`
    iterations of tests on several different file sizes. Plots results to
    visdom.
    """
    tests = [
        ("big file, csv", csv_to_tensor, constants.TRAIN_NORM),
        ("big file, tensor", bin_to_tensor, constants.TRAIN_TENSOR),
        ("small file, csv", csv_to_tensor, constants.VAL_NORM),
        ("small file, tensor", bin_to_tensor, constants.VAL_TENSOR),
    ]
    results = {}  # type: Dict[str, List[float]]
    for desc, fn, arg in tqdm(tests):
        results[desc] = []
        for i in range(n):
            start = time.perf_counter()
            res = fn(arg)
            results[desc].append(time.perf_counter() - start)
            del res
    viewer.plot_jitter(results, "Data Loading Speeds")

def convert() -> None:
    """
    Converts files from this project from csv to torch.save(...) format.
    """
    worklist = [
        (constants.TRAIN_NORM, constants.TRAIN_TENSOR),
        (constants.VAL_NORM, constants.VAL_TENSOR),
        (constants.TEST_NORM, constants.TEST_TENSOR),
    ]

    # pre-check: don't convert if any dest. files exist
    existing = which_exist([out for inp, out in worklist])
    if len(existing) > 0:
        print(
            "ERROR: Not converting because the following files already "
            "exist: {}".format(existing)
        )
        return

    print("dataio.convert :: start")
    for csv_fn, bin_fn in worklist:
        print("\t Converting {} to {}...".format(csv_fn, bin_fn))
        # This is the actual line of code that does the conversion.
        tensor_to_bin(csv_to_tensor(csv_fn), bin_fn)
    print("dataio.convert :: finish")

def bias() -> None:
    """
    Adds bias term to files from this project (output from convert()).
    """
    worklist = [
        (constants.TRAIN_TENSOR, constants.TRAIN_BIAS),
        (constants.VAL_TENSOR, constants.VAL_BIAS),
        (constants.TEST_TENSOR, constants.TEST_BIAS),
    ]

    existing = which_exist([out for inp, out in worklist])
    if len(existing) > 0:
        print(
            "ERROR: Not adding bias because the following files already "
            "exist: {}".format(existing)
        )
        return

    print("dataio.bias :: start")
    for tensor_fn, bias_fn in worklist:
        print("\t Converting {} to {}...".format(tensor_fn, bias_fn))
        # This is the actual line of code that does the biasing.
        tensor_to_bin(bias_tensor(bin_to_tensor(tensor_fn)), bias_fn)
    print("dataio.bias :: finish")

def onehot() -> None:
    """
    Transform labels to onehot for files from this project (output from
    bias()).
    """
    worklist = [
        (constants.TRAIN_BIAS, constants.TRAIN_ONEHOT),
        (constants.VAL_BIAS, constants.VAL_ONEHOT),
        (constants.TEST_BIAS, constants.TEST_ONEHOT),
    ]

    existing = which_exist([out for inp, out in worklist])
    if len(existing) > 0:
        print(
            "ERROR: Not creating onehots because the following files "
            "already exist: {}".format(existing)
        )
        return

    print("dataio.onehot :: start")
    for bias_fn, onehot_fn in worklist:
        print("\t Converting {} to {}...".format(bias_fn, onehot_fn))
        # This is the actual code that does the onehot'ing.
        class_labels, data = bin_to_tensors(bias_fn)
        onehot_labels = labels_to_onehot(class_labels, 10).type(torch.FloatTensor)
        tensor_to_bin(torch.cat([onehot_labels, data], dim=1), onehot_fn)
    print("dataio.onehot :: finish")

def main() -> None:
    """
    Data utils: speed test, conversion, biasing, making one-hot (see body).
    """
    parser = argparse.ArgumentParser(description="Data stuff.")
    choice = parser.add_mutually_exclusive_group(required=True)
    choice.add_argument("--speedtest", action="store_true")
    choice.add_argument("--convert", action="store_true")
    choice.add_argument("--bias", action="store_true")
    choice.add_argument("--onehot", action="store_true")
    args = parser.parse_args()
    if args.speedtest:
        speedtest()
    if args.convert:
        convert()
    if args.bias:
        bias()
    if args.onehot:
        onehot()

if __name__ == "__main__":
    main()


#!/usr/bin/env python2

"""
AR.Drone's position controller.

PD controller. Needs slight tuning.

Tausif S., 2017

"""

from __future__ import division, print_function, absolute_import

# Import ROS libraries
import roslib
import rospy
import numpy as np

# Import class that computes the desired positions
from tf.transformations import euler_from_quaternion
from geometry_msgs.msg import TransformStamped, Twist

from std_msgs.msg import String
import math

class PositionController(object):
    """ROS interface for controlling the Parrot ARDrone in the Vicon Lab."""
    # write code here for position controller
    def __init__(self):
        """Initialize the PositionController class."""

        # Current position.
        self.current_trans_x = 0.
        self.current_trans_y = 0.
        self.current_trans_z = 0.

        # Current quaternion.
        self.current_rot_x = 0.
        self.current_rot_y = 0.
        self.current_rot_z = 0.
        self.current_rot_w = 0.

        self.current_yaw = 0

        # Desired postion for trajectory goal.
        self.desired_x = 0.
        self.desired_y = 0.
        self.desired_z = 0.
        self.desired_yaw = 0.

        # Placeholder for old variable for differentiation.
        self.x_old = 0.
        self.y_old = 0.
        self.z_old = 0.
        self.climb_rate_old = 0.

        self.x_desired_old = 0.
        self.y_desired_old = 0.
        self.z_desired_old = 0.

        self.desired_x_err = 1.
        self.desired_y_err = 1.
        self.desired_z_err = 1.
        self.desired_yaw_err = 1.

    def update_position_controller(self, dt):
        """Postion Controller.

        Parameters
        ----------
        dt: float
            Change in time (seconds) used for differentiation and intergration
            purposes.

        Returns
        -------
        controlled_data: 1x4 array
            This array stores the 4 computed by the position controller.
            [roll, pitch, yaw_rate, climb_rate]
        """

        g = 9.8                            # Acceleration due to gravity
        # damping_rt_x = 0.7                    # Damping ratio
        # damping_rt_y = 0.7
        rise_time_z = 1
        rise_time_yaw = 1.

        x_now = self.current_trans_x
        y_now = self.current_trans_y
        z_now = self.current_trans_z

        #---------------------------------------------------------------------
        # Step 1: Calculating current the mass-normalized thrust.
        #---------------------------------------------------------------------

        climb_rate_now = (z_now - self.z_old) / dt
        climb_accel = (climb_rate_now - self.climb_rate_old) / dt

        quaternion = np.array([self.current_rot_x,
                              self.current_rot_y,
                              self.current_rot_z,
                              self.current_rot_w])

        euler = euler_from_quaternion(quaternion)
        roll_angle = euler[0]
        pitch_angle = euler[1]
        yaw_angle = euler[2]

        self.current_yaw = yaw_angle

        # f = (climb_accel + g) / (np.cos(roll_angle) * np.cos(pitch_angle))
        f = (0 + g) / (math.cos(roll_angle) * math.cos(pitch_angle))

        self.z_old = z_now
        self.climb_rate_old = climb_rate_now

        #---------------------------------------------------------------------
        # Step 2: Determining the desired x/y accelerations.
        #---------------------------------------------------------------------
        x_rate_now = (x_now - self.x_old) / dt
        y_rate_now = (y_now - self.y_old) / dt

        # print("x/y rate: {0} {1}".format(x_rate_now, y_rate_now))

        desired_x_rate = (self.desired_x - self.x_desired_old)/dt
        desired_y_rate = (self.desired_y - self.y_desired_old)/dt
        # desired_z_rate = 0

        damping_rt_x = 1
        damping_rt_y = 1

        w_n_x = 1
        w_n_y = 1

        x_accel = 2.0*damping_rt_x*w_n_x*(desired_x_rate - x_rate_now) + math.pow(w_n_x, 2)*(self.desired_x - x_now)
        y_accel = 2.0*damping_rt_y*w_n_y*(desired_y_rate - y_rate_now) + math.pow(w_n_y, 2)*(self.desired_y - y_now)

        # print("x/y accel: {0} {1}".format(x_accel, y_accel))
        print("x/y/z des: {0} {1} {2}".format(self.desired_x, self.desired_y, self.desired_z))
        print("x/y/z now: {0} {1} {2}".format(x_now, y_now, z_now))

        self.x_old = x_now
        self.y_old = y_now
        self.x_desired_old = self.desired_x
        self.y_desired_old = self.desired_y

        #---------------------------------------------------------------------
        # Step 3: Calculating commanded roll and pitch.
        #---------------------------------------------------------------------
        roll_command_rt = -y_accel / f

        if roll_command_rt >= 1.:
            roll_command_rt = 1.
        if roll_command_rt <= -1.:
            roll_command_rt = -1.

        roll_command = math.asin(roll_command_rt)

        pitch_command_rt = x_accel / (f*math.cos(roll_command))

        if pitch_command_rt >= 1.:
            pitch_command_rt = 1.
        if pitch_command_rt <= -1.:
            pitch_command_rt = -1.

        pitch_command = math.asin(pitch_command_rt)

        if yaw_angle >= 0 or yaw_angle <= 0:
            roll_command_B = (roll_command*math.cos(yaw_angle)) + (pitch_command*math.sin(yaw_angle))
            pitch_command_B = (-roll_command*math.sin(yaw_angle)) + (pitch_command*math.cos(yaw_angle))
            roll_command = roll_command_B
            pitch_command = pitch_command_B

        #---------------------------------------------------------------------
        # Step 4: Calculating commanded climb and yaw rates.
        #---------------------------------------------------------------------
        tau_z = rise_time_z/2.2
        tau_yaw = rise_time_yaw/2.2

        climb_rate_command = (1/tau_z)*(self.desired_z - self.current_trans_z)

        if climb_rate_command >= 2:
            climb_rate_command = 2

        # Ensuring the yaw angles are in the respective quadrant.
        for i in range(0, 1):
            if ((355*(np.pi/180)) <= self.desired_yaw <= (5*(np.pi/180))):
                break
            elif (yaw_angle < 0) and ((6*(np.pi/180)) <= self.desired_yaw <= (354*(np.pi/180))):
                yaw_angle = yaw_angle + (2*np.pi)
                break
            else:
                pass

        yaw_rate_command = (0.85)*(self.desired_yaw - yaw_angle)

        # Updating the desired pose error.
        self.desired_x_err = np.absolute((np.absolute(x_now) - (self.desired_x + 0.5)) / (self.desired_x + 0.5))
        self.desired_y_err = np.absolute((np.absolute(y_now) - (self.desired_y + 0.5)) / (self.desired_y + 0.5))
        self.desired_z_err = np.absolute((z_now - (self.desired_z + 0.001)) / (self.desired_z + 0.001))
        self.desired_yaw_err = np.absolute((yaw_angle - (self.desired_yaw + 0.01)) / (self.desired_yaw + 0.01))

        # Returning the commanded roll/pitch/yaw/climb
        controlled_data = np.array([roll_command, pitch_command, yaw_rate_command, climb_rate_command])

        return controlled_data


import requests
from datetime import datetime
from decouple import config

user_input = input("Tell me which exercise you did: ")
today = datetime.today()
today_str = today.strftime("%d/%m/%Y")
time = datetime.now()
time_str = time.strftime("%H:%M:%S")

# Nutritionix API
nu_api_id = config('NU_API_ID')
nu_api_key = config('NU_API_KEY')
nu_api_endpoint = "https://trackapi.nutritionix.com/v2/natural/exercise"
nu_api_header = {
    "x-app-id": nu_api_id,
    "x-app-key": nu_api_key,
    "Content-Type": "application/json"
}
nu_api_body = {
    "query": user_input
}
exercise_response = requests.post(nu_api_endpoint, headers=nu_api_header, json=nu_api_body)
exercise_data = exercise_response.json()['exercises']

# Sheety API
# Query key must be the singular of spreadsheet name
sh_api_endpoint = config('SH_API_ENDPOINT')
sh_api_header = {
    "Content-Type": "application/json"
}
for exercise in exercise_data:
    sh_api_body = {
        "workout": {
            "date": today_str,
            "time": time_str,
            "exercise": exercise['name'].title(),
            "duration": exercise['duration_min'],
            "calories": exercise['nf_calories']
        }
    }
    sheet_response = requests.post(sh_api_endpoint, headers=sh_api_header, json=sh_api_body)
    sheet_response.raise_for_status()


import os
import re
import sys
import glob
import sklearn as skl
import math
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import sklearn.cluster
from sklearn.decomposition import PCA
from sklearn.decomposition import NMF

from data_grid import DataGrid
from data_grid_TiNiSn import DataGrid, DataGrid_TiNiSn_500C, DataGrid_TiNiSn_600C
from NMF import nmf_func
from scipy.signal import find_peaks
from scipy.stats import pearsonr

def nmf_cluster(a=10,b=11,deal="None",use_peak=1,visualize=1,pear_threshold=0.8,peak_threshold=3,contri_threshold=0.1,dataset=1):

    '''
    Performs NMF clustering on the passed dataset and checks for peak shifting

    Parameters
    ------------------------

    a  : int
       Number of clusters (start)

    b  : int
       Number of clusters (end)

    deal : "None", "log", "square", "fft", "optimal"
        Dataset transformation before passing to NMF

        "None"    : no transformation
        "log"     : take natural log of dataset
        "square"  : square the dataset
        "fft"     : take fourier transform of the dataset
        "optimal" : discard the dataset and return the optimal W&H found from NMFk (see NMFk Julia.ipynb)

    use_peak : 0,1
        Check for the same number of peaks for highly correlated basis vectors?

        0 : no
        1 : yes

    visualize : 0,1
        Plot the basis vectors obtained from NMF?

        0 : no
        1 : yes

    pear_threshold : float64, range(0,1)
        Threshold for Pearson correlation
        Recommended: 0.7, 0.8, 0.9

    peak_threshold : float64
        Threshold for finding the number of peaks

    contri_threshold : float64, range(0,1)
        Threshold for the minimum conribution of phases at a composition point on the wafer map
        Recommended: 0.08, 0.1, 0.15, 0.25, 0.3

    dataset : 1,2
        Dataset desired

        1  : TiNiSn_500C
        2  : TiNiSn_600C

    Returns
    ------------------------
    Cluster grid(s) with the provided number of clusters

    labels : array (m_samples,)
        Labels of each sample in the dataset

    '''

    # Visualize all the patterns for the different cluster no.
    def clust_visuals(a,b,deal,dataset=1):
        off_set = 0
        fig,ax = plt.subplots(figsize=(8,10))

        if deal=="None":
            for j in range(a,b):
                W,H = nmf_func(j,deal,dataset)
                for i in range(np.shape(W)[1]):

                    ax.plot(dataGrid.data[1][:,0], W[:,i]+off_set)

                ax.annotate(j,(3.5,off_set+50))
                off_set+=100
            ax.set_title('Diffraction Clusters from W matrix')

        elif deal=="log":
            for j in range(a,b):
                W,H = nmf_func(j,deal,dataset)
                for i in range(np.shape(W)[1]):

                    ax.plot(dataGrid.data[1][:,0], W[:,i]+off_set)

                ax.annotate(j,(3.5,off_set+np.log(50)))
                off_set+=np.log(100)
            ax.set_title('Diffraction Clusters from W matrix')

        elif deal=="square":
            for j in range(a,b):
                W,H = nmf_func(j,deal,dataset)
                for i in range(np.shape(W)[1]):

                    ax.plot(dataGrid.data[1][:,0], W[:,i]+off_set)

                ax.annotate(j,(3.5,off_set+np.square(50)))
                off_set+=np.square(100)
            ax.set_title('Diffraction Clusters from W matrix')

        elif deal=="fft":
            for j in range(a,b):
                W,H = nmf_func(j,deal,dataset)
                for i in range(np.shape(W)[1]):

                    ax.plot(dataGrid.data[1][:,0], W[:,i]+off_set)

                ax.annotate(j,(3.5,off_set+np.square(50)))
                off_set+=100
            ax.set_title('Diffraction Clusters from W matrix')

        elif deal=="optimal":
            for j in range(a,b):
                W,H = nmf_func(j,deal,dataset)
                for i in range(np.shape(W)[1]):

                    ax.plot(dataGrid.data[1][:,0], W[:,i]+off_set)

                ax.annotate(j,(3.5,off_set+50))
                off_set+=100
            ax.set_title('Diffraction Clusters from W matrix')

    # Check for clusters with same number of peaks
    # Took time to finally write this code

    def peaks_func (W,H,peak_threshold):

        # Visualize the clusters and their peaks
        peaks_length=[]
        for i in range(0,len(H)):
            peaks, _ = find_peaks(W[:,i], height=peak_threshold)
            peaks_length.append(len(peaks))
            #fig,ax = plt.subplots()
            #ax.plot(dataGrid.data[1][:,0],W[:,i])
            #ax.plot((peaks*0.005)+1.5, W[peaks,i], "x")
            #ax.set_title(str(i+1))

        #print(peaks_length)

        same_peaks=[]
        for i in range(0,len(peaks_length)-1):
            smpeaks=[]

            if peaks_length[i] in list(peaks_length[:i]):
                #print(i)
                continue

            #print('Yes')
            for j in range(i+1,len(peaks_length)):
                    if peaks_length[i]==peaks_length[j]:
                        if len(smpeaks)==0:
                            smpeaks.append(i)
                        smpeaks.append(j)

            if len(smpeaks)!=0:
                same_peaks.append(smpeaks)

        #print(same_peaks)
#         for i in range(0,len(same_peaks)):
#             fig,ax = plt.subplots()
#             for j in same_peaks[i]:
#                 ax.plot(dataGrid.data[1][:,0],W[:,j])
#             ax.set_title('Same No. of Peaks '+str(same_peaks[i]))
        return same_peaks

    def Pearson_func(W,H,pear_threshold):

        # Pearson Correlation for each cluster against another
        p_corr = np.zeros((len(H),len(H)))

        for i in range(0,len(H)):
            for j in range(0,len(H)):
                corr, _ = pearsonr(W[:,i],W[:,j])
                p_corr[i,j]=corr

        # Find correlations higher than a certain threshold
        corr_idx = [[j,i] for j in range(len(p_corr)) for i in range(j+1,len(p_corr)) if np.abs(p_corr[j][i])>=pear_threshold]

        #print(corr_idx)
        #for i in range(0,len(corr_idx)):
         #   fig,ax = plt.subplots()
          #  for j in corr_idx[i]:
           #     ax.plot(W[:,j])
            #ax.set_title('Pearson Corr. '+ str(corr_idx[i]))

        return corr_idx

    def peak_pearson_match(corr_idx,same_peaks):
        #Check for clusters with high correlation and same peaks
        peak_shift = []
        for i in range(len(corr_idx)):
            for j in range(len(same_peaks)):
                if corr_idx[i][0] in same_peaks[j] and corr_idx[i][1] in same_peaks[j]:
                    print('.........Found Peak and Pearson match!!!..........')
                    peak_shift.append(corr_idx[i])

        return peak_shift

    def reshape_func(H,use_idx):

        #Combine the weights of all the peakshifted ones in H and also delete the peakshifted phases leaving the first phase in W
        #corr_idx=[[1,6],[2,3,4],[5,7]]

        H_reshaped1=np.zeros(shape=(len(H),177))
        for i in range(len(H)):
            H_reshaped1[i]=H[i]

        #H_reshaped1= H_reshaped  #this code will modify H_reshaped as H_reshaped1 is modified: very weird!!!!!!

        #This is done outside this function
        #use_idx =[]
        #for i in range(len(peak_shift)):
            #use_idx.append(peak_shift[i])

        for i in range(len(use_idx)):
            #print(i)
            b0 = H[use_idx[i][0]]

            for j in range(len(use_idx[i])):
                if j==0:
                    #print(j)
                    #print(b0)
                    continue

                #print(j)
                #if np.all(b0==H_reshaped[1]):
                 #   print('Yes')

                #print(b0)

                b0=b0 + H[use_idx[i][j]]

            H_reshaped1[use_idx[i][0]]=b0

        idx=[]
        for i in range(len(use_idx)):
            idx = idx + list(use_idx[i][1:])
        #idx = [i+1 for i in idx]
        print('Here are the clusters indices taken out', idx)
        new_H = np.delete(H_reshaped1,idx,0)
        print('Former H is of shape', np.shape(H_reshaped1))
        print('new_H is of shape', np.shape(new_H))

        return new_H

    def contributions(new_H,contri_threshold):

        #Reshape new_H cutting off those clusters contributing below a threshold

        H_reshape = np.zeros(shape=(np.shape(new_H)))

        for j in range(0,177):
            p = new_H[:,j]/np.sum(new_H[:,j])
            for i in range(len(p)):
                if p[i]<contri_threshold:
                    p[i]=0
            H_reshape[:,j]=p

        #Reshape the H where the 3 most abundant clusters remain for each datapoint/dot

        Big_locations=[]
        H_reshaped = np.zeros(shape=(np.shape(new_H)))

        for j in range(0,177):
            b = list(H_reshape[:,j])
            locations = []
            minimum = min(b)
            for i in range(3):
                maxIndex = b.index(max(b))
                if max(b)==0:
                    maxIndex=0
                    continue
                locations.append(maxIndex)
                b[maxIndex] = minimum
                H_reshaped[maxIndex,j]=H_reshape[maxIndex,j]

            Big_locations.append(locations)
            new_H_reshaped = H_reshaped
            #print(locations,j)

        #print(np.shape(new_H_reshaped))

        return new_H_reshaped,Big_locations

    def form_labels(new_H_reshaped,Big_locations):

        #Form labels for single-phases point
        labels1=[]
        labels_clust1=[]
        oo=-1

        for i in range(len(Big_locations)):
            if len(Big_locations[i])==1:
                #print(Big_locations[i],i,oo)

                if Big_locations[i] in list(labels_clust1[:]):
                    #print(Big_locations[i],i,oo)
                    continue
                oo+=1
                labels1.append(oo)
                labels_clust1.append(Big_locations[i])
                #print(Big_locations[i],i,oo,'Yes')

        #Form labels for double-phased points
        #This took a while :-( but I finally made it more efficient
        labels2=[]
        labels_clust2=[]
        ooo=oo

        for i in range(len(Big_locations)):
            p=0

            if len(Big_locations[i])==2:
                #print(i)

                for ii,v in enumerate(labels_clust2):
                    if set(Big_locations[i]) == set(v):   #Check if they are equal regardless of arrangement
                        #print(Big_locations[i],i,ooo)
                        p=1
                        continue

                if p==1:
                    continue

                ooo+=1
                labels2.append(ooo)
                labels_clust2.append(sorted(Big_locations[i]))
                #print(Big_locations[i],i,ooo,"Yes")

        #Form labels for triple-phased points
        #This took a while too :-(

        labels3=[]
        labels_clust3=[]
        oooo=ooo

        for i in range(len(Big_locations)):
            p=0

            if len(Big_locations[i])==3:
                #print(i)

                for ii,v in enumerate(labels_clust3):
                    if set(Big_locations[i]) == set(v):  #Check if they are equal regardless of arrangement
                        #print(Big_locations[i],i,oooo)
                        p=1
                        continue

                if p==1:
                    continue

                oooo+=1
                labels3.append(oooo)
                labels_clust3.append(sorted(Big_locations[i]))
                #print(Big_locations[i],i,oooo,"Yes")

        all_labels=labels1+labels2+labels3
        labels_clust = labels_clust1+labels_clust2+labels_clust3

        #Get the cluster for each point
        labels=[]
        for val in range(0,177):
            labels.append(labels_clust.index(sorted(Big_locations[val])))

        #print(labels)
        #print(list(labels_clust))

#         #This part of the code is a savior
#         new_labels=[None]*177
#         for ii in range(0,177):
#             x,y=dataGrid.coord(ii+1)
#             new_grid=dataGrid.grid_num(16-x,y)
#             new_labels[new_grid-1]=labels[ii]

        return labels,all_labels,labels_clust
        #return new_labels,all_labels,labels_clust

    def get_cluster_grid(all_labels,labels):

        hues = [float(float(x)/float(len(all_labels))) for x in range(1,len(all_labels)+1)]
        cluster_grid = np.zeros(shape = (15,15,3))
        #cluster_grid = np.ones(shape = (15,15,3))
        for val in range(1,178):
            x,y = dataGrid.coord(val)
            cluster = labels[val-1]
            #cluster_grid[15-y][x-1] = matplotlib.colors.hsv_to_rgb([hues[cluster],1,1])
            cluster_grid[y-1][x-1] = matplotlib.colors.hsv_to_rgb([hues[cluster],1,1])

        return cluster_grid

    def main_func(i,deal,pear_threshold,peak_threshold,contri_threshold,dataset=1):

        W,H=nmf_func(i,deal,dataset)

        corr_idx = Pearson_func(W,H,pear_threshold)
        same_peaks = peaks_func(W,H,peak_threshold)

        if same_peaks ==[]:
            print("No clusters with the same number of peaks for the given threshold and no. of clusters")
        if corr_idx ==[]:
            print("No clusters with correlation above or equal to the given correlation threshold. No peak shifting will be considered")

        print(np.shape(corr_idx),np.shape(same_peaks))

        if corr_idx==[]:

            new_H = H

        elif same_peaks==[] or use_peak==0:
            use_idx =[]
            for i in range(len(corr_idx)):
                use_idx.append(corr_idx[i])

            new_H=reshape_func(H,use_idx)

        elif use_peak==1:

            peak_shift = peak_pearson_match(corr_idx,same_peaks)

            if peak_shift==[]:

                print('No match between Pearson and same peaks found. Just Pearson will be considered')
                use_idx =[]
                for i in range(len(corr_idx)):
                    use_idx.append(corr_idx[i])

                new_H=reshape_func(H,use_idx)

            else:

                use_idx =[]
                for i in range(len(peak_shift)):
                    use_idx.append(peak_shift[i])

                new_H=reshape_func(H,use_idx)

        new_H_reshaped,Big_locations = contributions(new_H,contri_threshold)

        #print(np.shape(Big_locations))

        labels,all_labels,labels_clust = form_labels(new_H_reshaped,Big_locations)

        cluster_grid = get_cluster_grid(all_labels,labels)

        return cluster_grid,labels

   # This is where the interesting stuffs happen

    if dataset==1:
        dataGrid = DataGrid_TiNiSn_500C()
        data = dataGrid.get_data_array()
    elif dataset==2:
        dataGrid = DataGrid_TiNiSn_600C()
        data = dataGrid.get_data_array()

    if visualize==1:
            clust_visuals(a,b,deal,dataset)

    if b-a > 8: x_width=15
    else: x_width=10

    Big_labels=[]
    fig = plt.figure(figsize=(x_width,8))
    fig.tight_layout()

    for i in range(a,b):

        print('.....................................'+str(i)+' Clusters'+'...........................................')

        cg,labels = main_func(i,deal,pear_threshold,peak_threshold,contri_threshold,dataset)
        Big_labels.append(labels)
        ax = fig.add_subplot(2,np.ceil((b-a)/2),i-a+1)
        ax.imshow(cg)
        ax.invert_yaxis()
        #ax.invert_xaxis()
        ax.title.set_text(i)
        ax.axis("off")

    #print(".................There are %d clusters in total..........." %len(all_labels))
    k=.05
    plt.subplots_adjust(left=k,right=(1-k),bottom=k,top=(1-k),wspace=k,hspace=k)
    plt.show()

    return Big_labels


import pandas as pd
import numpy as np

#loading data from this csv file and saving it to the variable df
df = pd.read_csv('eyecenter_data.csv')
#print(df)

#head() method gives first 5 rows by default
first_five_rows = df.head()

#shows first 10 rows
first_five_rows = df.head(10)

#Gives statistics for each column
#column_info = df.info()

#print data for a specific column. Two different ways of executing to get the same data:
column_data = df.email
column_data2 = df['email']

# need to use two sets of brackets
select_multiple_columns = df[['first_name', 'last_name', 'email']]
#print(select_multiple_columns)

#select a single row of data using iloc. Based on zero-index.
select_one_row = df.iloc[0]
#print(select_one_row)

# this will select the first three rows
select_multiple_rows = df.iloc[0:3]
#print(select_multiple_rows)

#filter data based on if someone is younger than 30
select_subset_age = df[df.age < 30]
select_subset_contacts = df[df.contacts_or_eyeglasses == 'contacts']
select_subset_eyeglasses = df[df.contacts_or_eyeglasses == 'eyeglasses']
select_subset_none = df[df.contacts_or_eyeglasses == 'none']

# or
select_subset_or = df[(df.age >40) | (df.contacts_or_eyeglasses == 'eyeglasses')]

# and
select_subset_and = df[(df.first_name == 'Jamie') & (df.last_name == 'Gullbrand')]

# isin command is used to identify & display a list of values in a column
isin_name = df[df.first_name.isin(['Jamie', 'Judy', 'Timothy'])]
#print(isin_name)

# Check to see if a specific columns contains a piece of a string, filters to show relevant rows of data.
contains_data = df[df['contacts_or_eyeglasses'].str.contains("glas")]
#print(contains_data)

# print to a csv only columns that contain a piece of a string 'Jam' in a first name.
def print_name():
	data_name_contains = df[df['last_name'].str.contains("kins")]
	data_name_contains.to_csv('output_name_contains.csv')

# practice resetting indices. Using [[]] when filtering multiple rows
df2 = df.iloc[[1,3,5]]
# resets index
reset_index_df2 = df2.reset_index()

# remove old index as a new column using drop = True.
# you can also modify an existing data frame rather than creating a new one using inplace=True
df2.reset_index(inplace=True, drop=True)
#print(df2)

# to_csv() method saves csv file in current directory
#df.to_csv('output.csv')

# Add a column - specific data for each row. This would append the column to the existing data set.
df['customer_number'] = [2,32,49,78,82,99,103,109,201,250]

# Add a column - same data for all rows.
df['current_customer'] = 'Yes'
df['age_discount'] = .80 # 20% off
df['eye_exam_cost'] = 150

# Add a column based on performing a function on existing data. Age 10 years ago
df['age_ten_years_ago'] = df['age'] -10

def exclamation (string):
	return string + "!!!"

def uppercase (string):
	return string.upper()

# Using the Apply function to apply a function to every value in the column.
df['Excited Name!'] = df.first_name.apply(exclamation)
df['UPPERCASE NAME'] = df.first_name.apply(uppercase)

# LAMBDA EX 1: Using the lambda function on a column
get_first_initial = lambda x: x[0]
df['first_initial'] = df.first_name.apply(get_first_initial)

# LAMBDA EX 2: check if someone is older than 65
df['Older than 65?'] = df.age.apply(lambda x: True \
if x > 65 else False)

# LAMBDA EX 3: Using lambda on multiple columns at once by taking in an 'entire row'
get_total_cost = lambda row: row['eye_exam_cost'] * row['age_discount'] \
if row['age'] > 65 else row['eye_exam_cost']

df['Total Price including age discount if older than 65']  = df.apply(get_total_cost, axis =1)

# LAMBDA EX 4: Another example of taking in data from across a full row:
df['customer_basic_info'] = df.apply(lambda row: "Full Name: {} {}, Email: {}, Age: {}"\
    .format(row.first_name,
            row.last_name,
            row.email,
            row.age),
    axis=1)

# Can change all of the columns name at once by setting .columns = to an updated list
#df.columns = ['Column1 Name', 'Column2 Name']

# Change columns name for specific columns. inplace=True lets us edit the original DataFrame
df.rename(columns={
	'email': 'EMAIL ADDRESS'
	}, inplace = True)

df.rename(columns={
	'first_name': 'FIRST NAME',
	'last_name': 'LAST NAME',
	}, inplace = True)

#----------------------------
# Pandas Aggregates

# basic functions with the data in the ages column
#print(df.age)
print("Mean: {}".format(df.age.mean()))
print("Median: {}".format(df.age.median()))
print("Standard Deviation: {}".format(df.age.std()))
print("Max Number: {}".format(df.age.max()))
print("Min Number: {}".format(df.age.min()))
print("Count Values in Column: {}".format(df.age.count()))
print("Number of unique values: {}".format(df.age.nunique()))
print("List of unique values: {}".format(df.age.unique()))

# groupby method in pandas allows you to calculate based on groups of data
# In this example, I'm grouping by city and looking at the average age. Should print totals for Portland, Buxton, Boston.
age_by_city = df.groupby('location_city').age.mean()
print("Grouped average ages by city: {}".format(age_by_city))

vision_insurance = df.groupby('vision_insurance').customer_number.count()
print("# of people with / without vision insurance: {}".format(vision_insurance))
print(type(vision_insurance)) # Series

# Using reset_index at the end will transform the type Series into a DataFrame
vision_insurance = df.groupby('vision_insurance').customer_number.count().reset_index()
print("# of people with / without vision insurance: {}".format(vision_insurance))
print(type(vision_insurance)) # DataFrame - because we added .reset_index()

# Calculating percentiles using numpy with a lambda function
age_percentiles = df.groupby('location_city').age.apply(lambda x: np.percentile(x,75)).reset_index()
print('75th percentile of age for each city')
print(age_percentiles)

#df.pivot(columns='ColumnToPivot',
#         index='ColumnToBeRows',
#         values='ColumnToBeValues')

#print(df)
#df.to_csv('output.csv')


from collections import Counter
def decToBinary(n):
  a=[]
  for i in range(3, -1, -1):
    k = n >> i;
    if (k & 1):
      a.append(1)
    else:
      a.append(0)
  return a
def convert(list):
  s = [str(i) for i in list]
  res = int("".join(s))

  return(res)
# Driver Code
s=int(input())
p = map(int,input().strip().split(' '))
num0=[]
num1=[]
for i in p:
  num0.append(decToBinary(i).count(0))
  num1.append(decToBinary(i).count(1))
print(num0)
print(num1)
num10 = []
for i in range(len(num0)):
  num10.append(num0[i]+num1[i])
print(num10)
c=Counter(num10)
numerr = []
if(convert(c.values())>1):
  numerr = decToBinary(convert(c.values()))
else:
  numerr=decToBinary(0)
numstr=""
for j in numerr:
  numstr += str(j)
print(numstr)


from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.shortcuts import get_object_or_404, render, render_to_response

from . import forms
from . import models
# Create your views here.

def availability_page(request):
    items = models.Available.objects.all()
    return render_to_response('availability.html', {'items': items})

def activity_list(request):
    activities = models.Activity.objects.all()
    return render(request, 'activities/activity_list.html', {'activities': activities})

def activity_detail(request, pk):
    activity = get_object_or_404(models.Activity, pk=pk)
    return render(request, 'activities/activity_detail.html', {'activity': activity})

def package_detail(request, activity_pk, package_pk):
    package = get_object_or_404(models.PartyPackage, activity_id=activity_pk, pk=package_pk)
    return render(request, 'activities/package_detail.html', {'package': package})

@login_required
def activity_create(request):
    form = forms.ActivityForm(request.POST or None)
    if form.is_valid():
        instance = form.save(commit=False)
        instance.save()
        messages.add_message(request, messages.SUCCESS,
                             "Activity added!")
    context = {
        'form': form
    }

    return render(request, "activities/activity_form.html", context)

@login_required
def availability_create(request):
    form = forms.AvailableForm(request.POST or None)
    if form.is_valid():
        instance = form.save(commit=False)
        instance.save()
        messages.add_message(request,  messages.SUCCESS,
                             "Person added!")

    context = {
            'form': form,
    }
    return render(request, "activities/available_form.html", context)

@login_required
def package_create(request):
    form = forms.PartyPackageForm(request.POST or None)
    if form.is_valid():
        instance = form.save(commit=False)
        instance.save()
        messages.add_message(request, messages.SUCCESS,
                             "Packageg added!")
    return render(request, "activities/package_form.html", {"form": form})


SECRET_KEY = 'test'
TEMPLATES = [
    {
        "BACKEND": "django.template.backends.django.DjangoTemplates",
        "DIRS": [],
        "APP_DIRS": True,
    }
]
DATABASES = {}
DATABASES["default"] = {
    "ENGINE": "django.db.backends.postgresql",
    "NAME": "resource_dev",
    "TEST": {"NAME": "resource_test"},
}
INSTALLED_APPS = (
    "django.contrib.sessions",
    "django.contrib.auth",
    "django.contrib.contenttypes",
    "tests"
)
AUTH_USER_MODEL = "tests.User"
ROOT_URLCONF = "tests.urls"

# these middlewares are required for the Django test client
# to set request.user and request.session
MIDDLEWARE = [
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
]


import argparse
import pickle

import matplotlib.pyplot as plt
import seaborn as sns
from sklearn import decomposition
from sklearn import pipeline
from sklearn import preprocessing
from sklearn import cluster
from sklearn import metrics

sns.set()

from typing import Dict, Any

def dummy_data():
    import numpy as np
    a = np.random.normal(0, 0.1, (100, 10))
    b = np.random.normal(0, 0.1, (100, 10))
    return np.vstack((a, b))

def main(logits_file_path: str) -> None:
    print(logits_file_path)  # load file here
    data = dummy_data()

    kmeans = cluster.KMeans(n_clusters=2)
    fun = pipeline.make_pipeline(preprocessing.StandardScaler(), decomposition.PCA(n_components=3))
    data_transformed = fun.fit_transform(data)

    ypred = kmeans.fit_predict(data_transformed)
    print("> silhouette score: {}".format(metrics.silhouette_score(data_transformed, ypred)))

    fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2)

    first = ax1.scatter(data_transformed[:100, 0], data_transformed[:100, 1], c=data_transformed[:100, 2], marker="o", label="Legitimate")
    second = ax1.scatter(data_transformed[100:200, 0], data_transformed[100:200, 1], c=data_transformed[100:200, 2], marker="x", label="Watermarked")
    ax1.figure.colorbar(second, ax=ax1, label="Third component")

    ax1.set_xlabel("First component")
    ax1.set_ylabel("Second component")
    ax1.set_title("Ground")
    for class_, m in zip([0, 1], ["o", "x"]):
        ax2.scatter(data_transformed[ypred == class_, 0], data_transformed[ypred == class_, 1], c=data_transformed[ypred == class_, 2], marker=m)

    ax2.figure.colorbar(second, ax=ax2, label="Third component")

    ax2.set_xlabel("First component")
    ax2.set_ylabel("Second component")
    ax2.set_title("Clustering labels")
    plt.legend()
    plt.show()

def load_file(file_path: str) -> Dict[str, Any]:
    with open(file_path, "rb") as f:
        return pickle.load(f)

def handle_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--logits_file",
        type=str,
        default=None,
        help="Results data file for plotting.")

    args = parser.parse_args()
    if args.logits_file is None:
        raise ValueError("Logits data file must be provided.")

    return args

if __name__ == "__main__":
    args = handle_args()
    main(args.logits_file)


import json
from typing import Optional

from fastapi import APIRouter, Depends
from sqlalchemy import select
from starlette.background import BackgroundTasks
from starlette.requests import Request
from starlette.templating import Jinja2Templates
from telegram.ext import Application
from telegram.helpers import escape_markdown

from shaas_common.model import Menu, MenuItem, Event, Order, EventState
from shaas_common.poll import close_poll_if_necessary
from shaas_common.settings import get_settings
from shaas_common.storage import Storage, get_db
from shaas_web.bot import get_bot

web_app_router = APIRouter()

templates = Jinja2Templates(directory="templates")

@web_app_router.get("/{menu_id}")
async def get_menu(
        menu_id: int,
        request: Request,
        s:Storage=Depends(get_db),
        user_id: Optional[int] = None,
        user_hash: Optional[str] = None,
        chat_id: Optional[int] = None
):
    menu = await s.menu_item.get_items(menu_id)

    current = await s.event.get_current(chat_id)
    active = current and current.state == EventState.collecting_orders

    data = {
        "request": request,
        "menu": menu,
        "user_id": user_id,
        "user_hash": user_hash,
        "active": "true" if active else "false",
        "event_id": current.id if current else 0
    }
    return templates.TemplateResponse("index.html", data)

@web_app_router.post("/order")
async def make_order(request: Request, background_tasks: BackgroundTasks, app = Depends(get_bot)):
    s = Storage()

    form = await request.form()

    user_id = int(form["user_id"])
    order_raw = json.loads(form["order_data"])
    event_id = int(form["event_id"])
    comment = form.get("comment")
    # comment = escape_markdown(comment)

    event = await s.event.get(event_id)

    if event.state != EventState.collecting_orders:
        await app.bot.send_message(
            chat_id=user_id,
            text="Это меню для события, которое уже закрыто.\n\nЗаказ не оформлен."
        )
        return {
            "ok": True
        }

    order_data = dict()
    for row in order_raw:
        key_id = int(row["id"])
        key = await s.menu_item.get(key_id)
        value = int(row["count"])
        order_data[key] = value

    await s.order.create_order(user_id, event.id, order_data, comment)
    await s.commit()

    items_str = []
    total_price = 0
    for item, count in order_data.items():
        price = count*item.price
        total_price += price
        items_str.append(
            f"{count}x {item.name} - {price} р."
        )

    msg = "Заказ принят!\n\n" + "\n".join(items_str) + f"\n\nИтого: {total_price} р."
    if comment:
        msg += f"\n\nКомментарий:\n{comment}"

    await app.bot.send_message(
        chat_id=user_id,
        text=msg
    )

    await close_poll_if_necessary(s, app.bot, event.id)
    # background_tasks.add_task(close_poll_if_necessary, s, app.bot, event_id)

    return {
        "ok": True
    }


"""Twitter URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.8/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Add an import:  from blog import urls as blog_urls
    2. Add a URL to urlpatterns:  url(r'^blog/', include(blog_urls))
"""
from django.conf.urls import include, url
from django.contrib import admin
from django.conf import settings
from main import views
from django.conf.urls.static import static

urlpatterns = [
    url(r'^admin/', include(admin.site.urls)),
    url('^registerForm$', views.register, name='register'),
    url('^$', views.main, name='main'),
    url('^addTweet/$', views.new_tweet, name='tweet'),
    url('^login$', views.login_page, name="login_page"),
    url('^profile/(?P<username>\w+)$', views.profile, name='profile'),
    url('^settings/$', views.settings, name='settings'),
    url('^profile_pic/$', views.profile_pic, name='profile_pic'),
    url(r'^logout/', views.log_out, name='log_out'),
    url(r'^follow/(?P<username>\w+)$', views.follow, name='follow'),
    url('^likes/(?P<username>\w+)/(?P<id>\d+)$', views.likes, name='likes'),
    url(r'^all_messages/(?P<username>\w+)$', views.all_messages, name='all_messages'),
    url(r'^show_message$', views.show_message, name='show_message'),
    url(r'^send_message$', views.send_message, name='send_message'),
    url(r'^delete_conversation', views.delete_conversation, name='delete_conversation'),
    url(r'^check_message', views.check_message, name='check_message'),
    ]

if settings.DEBUG:
    urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
    urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


# -*- coding: utf-8 -*-
from flask.ext.assets import Bundle, Environment

css = Bundle(
    "libs/bootstrap/dist/css/bootstrap.css",
    "css/style.css",
    "css/bootstrap-multiselect.css",
    "css/bootstrap-tagsinput.css",
    "css/selectize.css",
    filters="cssmin",
    output="public/css/common.css"
)

js = Bundle(
    "libs/jQuery/dist/jquery.js",
    "libs/bootstrap/dist/js/bootstrap.js",
    "libs/ajax/ckeditor/ckeditor.js",
    "js/plugins.js",
    "js/bootstrap-multiselect.js",
    "js/bootstrap-tagsinput.js",
    "js/selectize.js",
    "js/script.js",
    filters='jsmin',
    output="public/js/common.js"
)

assets = Environment()

assets.register("js_all", js)
assets.register("css_all", css)


'''def last(n):
    return n[-1]

def sort_list_last(tuples):
  return sorted(tuples, key=last)

print(sort_list_last([(2, 5), (1, 2), (4, 4), (2, 3), (2, 1)]))
'''

a=[(2, 5), (1, 2), (4, 4), (2, 3), (2, 1)]
print(len(a))
for j in range (1,len(a)) :
  for i in range(len(a)-j) :
    if a[i][1]>a[i+1][1] :
      temp=a[i]
      a[i]=a[i+1]
      a[i+1]=temp
for i in range(len(a)) :
  print(a[i])

#6.Write a Python program to get a list, sorted in increasing order by the last element in each tuple from a given list of non-empty tuples.py


##Given an array of integers and an integer k, you need to find the number of unique k-diff pairs in the array. Here a k-diff pair is defined as an integer pair (i, j), where i and j are both numbers in the array and their absolute difference is k.
##Example 1:
##Input: [3, 1, 4, 1, 5], k = 2
##Output: 2
##Explanation: There are two 2-diff pairs in the array, (1, 3) and (3, 5).
##Although we have two 1s in the input, we should only return the number of unique pairs.
##Example 2:
##Input:[1, 2, 3, 4, 5], k = 1
##Output: 4
##Explanation: There are four 1-diff pairs in the array, (1, 2), (2, 3), (3, 4) and (4, 5).
##Example 3:
##Input: [1, 3, 1, 5, 4], k = 0
##Output: 1
##Explanation: There is one 0-diff pair in the array, (1, 1).
##Note:
##The pairs (i, j) and (j, i) count as the same pair.
##The length of the array won't exceed 10,000.
##All the integers in the given input belong to the range: [-1e7, 1e7].

class Solution:
    def findPairs(self, nums, k):
        if not nums:
            return 0
        nums = sorted(nums)
        count = 0
        for i in range(0, len(nums), 1):
            if i > 0 and nums[i] == nums[i - 1]:
                i += 1
                continue
            j = i + 1
            while j < len(nums):
                if j > i + 1 and nums[j] == nums[j - 1]:
                    j += 1
                    continue
                if nums[j] - nums[i] < k:
                    j += 1
                    continue
                if nums[j] - nums[i] == k:
                    count += 1
                    break
                if nums[j] - nums[i] > k:
                    break
        return count

nums = [1, 1, 2, 2, 3]
k = 0
s = Solution()
print(s.findPairs(nums, k))


from datetime import datetime
from tensorboardX import SummaryWriter

import os
import logging

from utils.utils import create_logger, output_process, fix_random

class BaseTrainer(object):
    def __init__(self, experimental_name='debug', seed=None):
        # BASE
        self.current_time = datetime.now().strftime('%b.%d_%H.%M.%S')
        self.writer = None
        self.logger = None
        self.experimental_name = experimental_name
        self.seed = seed

        # SAVE PATH
        self.repo_path = os.getcwd()
        self.save_folder = f'{self.repo_path}/results/{experimental_name}'

        output_process(self.save_folder)  # create folder or not
        self._init_log()  # get log and writer
        if seed is not None:
            fix_random(seed)

    def _init_log(self):
        self.writer = SummaryWriter(log_dir=self.save_folder)
        self.logger = create_logger()
        fh = logging.FileHandler(filename=f'{self.save_folder}/log.txt')
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        fh.setFormatter(formatter)
        self.logger.addHandler(fh)


import numpy as np
import os
import random

nx = 8096
ny = 7888

fs = os.stat('scroll.dat').st_size
assert fs % (nx*ny*2) == 0
nz = fs//(nx*ny*2)

# "uint16" is little-endian, at least on my machine ...
fp = np.memmap('scroll.dat', dtype='uint16', mode='r', shape=(nx,ny,nz))

# just quickly check that endianness
assert fp[0][0][0] == 0x60a9

pp = []
cc = []

SIZE = 100

# pick an arbitrary point for the center
cx = random.randint(0, nx-SIZE-1)
cy = random.randint(0, ny-SIZE-1)
cz = random.randint(0, nz-SIZE-1) if nz > SIZE else 0

POINTS = 1000000

pp = np.zeros((POINTS,3), dtype='float')
cc = np.zeros((POINTS,3), dtype='float')
for n in range(0,POINTS):
    v = 0
    while v < 32768:
        xx = cx + random.random() * (SIZE-1)
        yy = cy + random.random() * (SIZE-1)
        zz = cz + random.random() * (min(SIZE,nz)-1)
        v = fp[int(xx)][int(yy)][int(zz)]
        #v = interp([xx,yy,zz])[0]

    pp[n] = [xx,yy,zz]
    vv = (v-32768)/32768
    cc[n] = [vv,1-vv,0]

import open3d as o3d

pcd = o3d.geometry.PointCloud()
pcd.points = o3d.utility.Vector3dVector(pp)
pcd.colors = o3d.utility.Vector3dVector(cc)
o3d.visualization.draw_geometries([pcd])


"""Using the human evaluation dataset to to sample code to test the effectiveness of this approach

Dataset from: https://github.com/openai/human-eval/tree/master
Paper: https://arxiv.org/pdf/2107.03374.pdf
"""

import glob
import json
from pathlib import Path

import tqdm
from typer import Typer

from prompt_to_code.agents.agents import run_agent
from prompt_to_code.config import PromptToCodeConfig

# requires human-eval is installed
# Also need to un-comment out the 'eval' function
try:
    from human_eval.data import HUMAN_EVAL, read_problems
    from human_eval.evaluation import evaluate_functional_correctness
except ImportError:
    print("=" * 80)
    print("Install human-eval from: https://github.com/openai/human-eval")
    print("$ git clone https://github.com/openai/human-eval")
    print("$ pip install -e human-eval")
    print("Then uncomment the import statement from /human_eval/execution.py")
    print("```#                         exec(check_program, exec_globals)```")
    print("=" * 80)
    raise ImportError("human-eval required")

def run_human_eval(
    agent="tdd",
    outdir: Path | str = "./examples/human_eval",
    num_samples_per_task=1,
    start_question: int = 0,
    start_ittr: int = 0,
):
    problems = read_problems(HUMAN_EVAL)
    outdir = Path(outdir)
    ittr = tqdm.tqdm(total=len(problems) * num_samples_per_task)
    problem_names = sorted(problems.keys())
    for i, example_id in enumerate(problem_names):
        if i < start_question:
            ittr.update(num_samples_per_task - start_ittr)
            continue
        example = problems[example_id]
        for loop_cnt in range(start_ittr, num_samples_per_task):
            filename = outdir / f"human_eval_{i:04}_{loop_cnt:04}.py"
            name = example["task_id"]
            prompt = example["prompt"]
            run_agent(agent, name, filename, prompt)
            ittr.update(1)

def aggregate_outputs(outdir: Path | str = "./examples/human_eval"):
    pattern = str(Path(outdir) / "human_eval_*.py")
    results = []
    task_ids = set()
    for fname in glob.glob(pattern):
        code = Path(fname).read_text()

        s = fname.split(str(outdir / "human_eval_"))[1].split(".py")[0]
        if "_" in s:
            task_number, *_ = s.split("_")
        else:
            task_number = s
        task_id = f"HumanEval/{int(task_number)}"
        task_ids.add(task_id)
        results.append({"task_id": task_id, "completion": code})
    with open(outdir / "human_eval_samples.jsonl", "w") as f:
        for problem in results:
            f.write(json.dumps(problem) + "\n")

    problems = read_problems(HUMAN_EVAL)
    with open(outdir / "human_eval_problems.jsonl", "w") as f:
        for problem in problems.values():
            if problem["task_id"] in task_ids:
                f.write(json.dumps(problem) + "\n")

def eval_tests(
    outdir="./examples/human_eval",
    k: list[int] | None = None,
    n_workers: int = 4,
    timeout: float = 3.0,
):
    if k is None:
        k = [1]
    outdir = Path(outdir)
    sample_file = outdir / "human_eval_samples.jsonl"
    problem_file = outdir / "human_eval_problems.jsonl"
    return evaluate_functional_correctness(
        str(sample_file), k, n_workers, timeout, str(problem_file)
    )

def score(num_samples_per_task, outdir):
    aggregate_outputs(outdir=outdir)
    # Eval
    k = [1, 2, 3, 4, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100]
    results = eval_tests(
        outdir=outdir,
        k=[i for i in k if i <= num_samples_per_task],
        n_workers=4,
        timeout=3,
    )
    print(results)
    with open(outdir / "human_eval_results.json", "w") as f:
        json.dump(results, f)

app = Typer(name="Prompt-to-code: Human Eval")

@app.command()
def human_eval(
    agent: str = "tdd",
    outdir_root: str = "./examples",
    num_samples_per_task: int = 1,
    just_score: bool = False,
    start_question: int = 0,
    start_ittr: int = 0,
):
    PromptToCodeConfig()
    outdir = Path(outdir_root) / f"./human_eval_{agent}"

    print("Running human eval for agent", agent)
    print(f"saving output to {outdir}")
    print("WARNING THIS WILL COST $$$, please monitor OPENAI bill")
    if not just_score:
        run_human_eval(
            agent=agent,
            outdir=outdir,
            num_samples_per_task=num_samples_per_task,
            start_question=start_question,
            start_ittr=start_ittr,
        )
    score(num_samples_per_task=num_samples_per_task, outdir=outdir)

if __name__ == "__main__":
    app()


"""
User interface
"""
import socket
from tkinter import ttk
import tkinter as tk
from tkinter.messagebox import showinfo
from tkinter.filedialog import askopenfile
import os
import configparser
from datetime import datetime, timedelta
from .client import Client
from .sample_files.sample_data import DATA

class UserInterface:
    """
    User interface for interacting with a server.

    Option 1: run with config file
    Option 2: run with tkinter GUI
    """

    def __init__(self, host: str = None, port: int = None):
        """
        :param host: The ip address or hostname of the server (do not pass if using config)
        :param port: The port of the server (do not pass if using config)
        """

        # Create client
        self.start_time = datetime.now()
        if host and port:
            self.client = Client(host, port)
            self.client.connection()

    def run(self, config_file=""):
        """
        Runs user interface, either by GUI or through configuration file

        :param config_file: path to config file (do not pass if not required)
        """

        if config_file:
            self._run_with_config(config_file)
        else:
            self._run_with_tkinter()

    @staticmethod
    def _run_with_config(config_file: str):
        """
        Runs connection through configuration file

        :param config_file: path to config file
        """

        config = configparser.ConfigParser()
        config.read(config_file)

        def send(client, config):
            if config["File"].getboolean("send"):
                file = config["File"]["filepath"].replace('"', "").replace("'", "")
                enc = config["File"].getboolean("encrypt")

                if not os.path.isfile(file):
                    print("Invalid file path")
                    return
                client.transfer_file(file, enc)

            if config["Object"].getboolean("send"):
                try:
                    obj = eval(config["Object"]["object"])
                except Exception as e:
                    print("[Object]object is not valid. Error:", e)
                    return
                enc = config["Object"].getboolean("encrypt")
                ser = config["Object"]["serialisation"]

                client.transfer_object(ser, obj, enc)

        if config["LocalServer"].getboolean("send"):
            host = config["LocalServer"]["host"]
            host = socket.gethostname() if host.lower() == "localhost" else host
            port = config["LocalServer"].getint("port")

            client = Client(host, port)
            client.connection()

            new_host = socket.gethostname()
            print(
                f"Could not connect to {config['LocalServer']['host']}, using {new_host} instead."
            )
            client = Client(new_host, port)
            client.connection()
            send(client, config)

        if config["AWS"].getboolean("send"):
            try:
                host = config["AWS"]["host"]
                port = int(config["AWS"]["port"])
                client = Client(host, port)
                client.connection()
                send(client, config)
            except OSError:
                print(f"Could not connect to {host}{port}")

    def _run_with_tkinter(self):
        """
        Runs connection using tkinter GUI
        """

        self.root = tk.Tk()
        self.root.resizable(False, False)
        self.root.title("Data Processing")
        self.set_geometry()
        self.outer_frame = tk.Frame(
            self.root, highlightbackground="white", highlightthickness=1
        )
        self.outer_frame.pack(padx=5, pady=5, expand=False)
        self.frame1 = tk.Frame(
            self.outer_frame, highlightbackground="white", highlightthickness=1
        )
        self.frame1.pack(side=tk.TOP, fill=tk.BOTH, expand=False)
        self.sub_frame1 = tk.Frame(self.frame1)
        self.sub_frame1.pack(padx=5, pady=5, anchor=tk.CENTER)
        self.frame2 = tk.Frame(
            self.outer_frame, highlightbackground="white", highlightthickness=1
        )
        self.frame2.pack(side=tk.TOP, fill=tk.BOTH, expand=False)
        self.sub_frame2 = tk.Frame(self.frame2)
        self.sub_frame2.pack(padx=5, pady=5, anchor=tk.CENTER)
        self.frame3 = tk.Frame(
            self.outer_frame, highlightbackground="white", highlightthickness=1
        )
        self.frame3.pack(side=tk.TOP, fill=tk.BOTH, expand=False)
        self.sub_frame3 = tk.Frame(self.frame3)
        self.sub_frame3.pack(padx=5, pady=5, anchor=tk.CENTER)
        self.frame4 = tk.Frame(
            self.outer_frame, highlightbackground="white", highlightthickness=1
        )
        self.frame4.pack(side=tk.TOP, fill=tk.BOTH, expand=False)
        self.sub_frame4 = tk.Frame(self.frame4)
        self.sub_frame4.pack(padx=5, pady=5, anchor=tk.CENTER)
        self.frame5 = tk.Frame(
            self.outer_frame, highlightbackground="white", highlightthickness=1
        )
        self.frame5.pack(side=tk.TOP, fill=tk.BOTH, expand=False)
        self.frame6 = tk.Frame(
            self.outer_frame, highlightbackground="white", highlightthickness=1
        )
        self.frame6.pack(side=tk.TOP, fill=tk.BOTH, expand=False)
        self.sub_frame6 = tk.Frame(self.frame6)
        self.sub_frame6.pack(padx=5, pady=5, anchor=tk.E)
        self.browse_file()
        self.obj_selection()
        self.serialisation_selection()
        self.encryption_option()
        self.execute_button()
        self.text_box()
        self.info_exit()
        self.selections = {}
        self.root.mainloop()

    def set_geometry(self):
        """
        Calculates geometry to center the window.
        """

        window_height = 670
        window_width = 600
        screen_width = self.root.winfo_screenwidth()
        screen_height = self.root.winfo_screenheight()
        x_cordinate = int((screen_width / 2) - (window_width / 2))
        y_cordinate = int((screen_height / 2) - (window_height / 2))
        self.root.geometry(
            "{}x{}+{}+{}".format(window_width, window_height, x_cordinate, y_cordinate)
        )

    def browse_file(self):
        """
        Creates button to upload files.
        """

        file_select = ttk.Label(
            self.sub_frame1, text="Select file or object for transfer:"
        )
        file_select.pack(pady=5)

        # Text for file select label
        file_select_label = ttk.Label(self.sub_frame1, text="Select file:")
        file_select_label.pack(padx=10, pady=5, side=tk.LEFT)

        # create a combobox
        selected_file = tk.StringVar()
        file_path = ttk.Combobox(self.sub_frame1, textvariable=selected_file)

        def select_file_callback():
            """
            Selects file path
            """

            # file path setup
            file_path = askopenfile(
                mode="r",
                filetypes=[
                    ("text files", ".txt"),
                    ("pdf files", ".pdf"),
                    ("xml files", ".xml"),
                    ("picture files", ".jpg"),
                    ("picture files", ".jpeg"),
                ],
            )
            if file_path is not None:
                self.file_path = file_path.name
                self.selections["File path"] = self.file_path
                self.update_text_box()

        # create a button
        file_path_button = tk.Button(
            self.sub_frame1, text="Browse", command=select_file_callback
        )
        file_path_button.config(width=10)
        file_path_button.pack(pady=5, side=tk.LEFT)

        # prevent typing a value
        file_path["state"] = "readonly"

    def serialisation_selection(self):
        """
        Creates drop down menu to select serialisation method.
        Methods: json, binary.
        """

        # Text for serialisation method label
        serialisation_label = ttk.Label(
            self.sub_frame2, text="Select object serialisation method:"
        )
        serialisation_label.pack(pady=5)

        def serialisation_callback(event):
            self.selections["Serialisation method"] = selected_method.get()
            self.update_text_box()
            return event

        selected_method = tk.StringVar()

        # create a combobox
        serialisation_method = ttk.Combobox(
            self.sub_frame2, textvariable=selected_method
        )
        serialisation_method.bind("<<ComboboxSelected>>", serialisation_callback)

        serialisation_method["values"] = ["Binary", "Json"]

        # prevent typing a value
        serialisation_method["state"] = "readonly"

        # place the widget
        serialisation_method.config(width=10)
        serialisation_method.pack(pady=5)

    def obj_selection(self):
        """
        Creates drop down menu to select from sample objects.
        Object types: dictionaries, lists, classes (for binary serialisation only)
        """

        # Text for select object label
        obj_selection_label = ttk.Label(self.sub_frame1, text="Select object:")
        obj_selection_label.pack(padx=10, pady=5, side=tk.LEFT)

        def obj_selection_callback(event):
            self.selections["Selected object"] = selected_object.get()
            self.update_text_box()
            return event

        # create a combobox
        selected_object = tk.StringVar()
        selection_obj = ttk.Combobox(self.sub_frame1, textvariable=selected_object)
        selection_obj.bind("<<ComboboxSelected>>", obj_selection_callback)

        selection_obj["values"] = list(DATA.keys())

        # prevent typing a value
        selection_obj["state"] = "readonly"

        # place the widget
        selection_obj.config(width=10)
        selection_obj.pack(pady=5, side=tk.LEFT)

    def encryption_option(self):
        """
        Creates tick box to select encryption method.
        Encryption options - Yes, No.
        """

        # Text for encrypt file label
        encryption_label = ttk.Label(self.sub_frame3, text="Encrypt file/object:")
        encryption_label.pack(pady=5, side=tk.TOP)

        # create a radio button
        def encryption_callback():
            """
            Updates selection and UI console
            """

            self.selections["Encryption"] = radio_var.get()
            self.update_text_box()

        radio_var = tk.StringVar()
        enc1 = tk.Radiobutton(
            self.sub_frame3,
            text="Yes",
            value=True,
            variable=radio_var,
            tristatevalue=" ",
            command=encryption_callback,
        )
        enc2 = tk.Radiobutton(
            self.sub_frame3,
            text="No",
            value=False,
            variable=radio_var,
            tristatevalue=" ",
            command=encryption_callback,
        )

        # place the widget
        enc1.pack(pady=5, side=tk.LEFT)
        enc2.pack(pady=5, side=tk.LEFT)

    def execute_button(self):
        """
        Executes transfer operation.
        """

        def file_transfer_callback():
            """
            Show file transfer status information.
            """

            if "File path" not in self.selections:
                showinfo(
                    title="Error",
                    message="No file path given, please select using the browse button",
                )
            elif "Encryption" not in self.selections:
                showinfo(
                    title="Error",
                    message="Please select encryption (yes or no) for file transfer",
                )
            else:
                showinfo(title="Information", message="Transfer initiated")
                self.client.transfer_file(self.file_path, encrypt=self.selections["Encryption"])
                self.selections["file_transfer"] = "True"
                self.update_text_box()

                try:
                    for line in self.get_log_output():
                        self.log_box.insert(tk.END, line)
                except (tk.TclError, AttributeError):
                    self.log_popup()

        def object_transfer_callback():
            """
            Show file transfer status information.
            """

            if "Selected object" not in self.selections:
                showinfo(title="Error", message="Please an object for transfer")
            elif "Serialisation method" not in self.selections:
                showinfo(
                    title="Error",
                    message="Please select serialisation method for object transfer",
                )
            elif "Encryption" not in self.selections:
                showinfo(
                    title="Error",
                    message="Please select encryption (yes or no) for object transfer",
                )
            else:
                showinfo(title="Information", message="Transfer initiated")
                data = DATA[self.selections["Selected object"]]
                self.client.transfer_object(
                    self.selections["Serialisation method"], data, self.selections["Encryption"]
                )
                try:
                    for line in self.get_log_output():
                        self.log_box.insert(tk.END, line)
                except (tk.TclError, AttributeError):
                    self.log_popup()

        file_transfer_button = tk.Button(
            self.sub_frame4, text="Upload file", command=file_transfer_callback
        )
        object_transfer_button = tk.Button(
            self.sub_frame4, text="Upload object", command=object_transfer_callback
        )

        # place the widget
        file_transfer_button.config(width=10)
        file_transfer_button.pack(side=tk.LEFT, pady=5)
        object_transfer_button.config(width=10)
        object_transfer_button.pack(side=tk.LEFT, pady=5)

    def text_box(self):
        """
        Message box with status information.
        """

        # Create a text widget
        self.text = tk.Text(self.frame5)

        # place the widget
        self.text.pack()

    def info_exit(self):
        """
        Creates information and exit buttons.
        """

        def info_button_callback():
            """
            Show instructions.
            """

            showinfo(
                title="Information",
                message="To transfer file:\n"
                "1) Select file using browse button\n"
                "2) Select encryption option\n"
                "3) Click Upload button to initiate transfer\n\n"
                "To transfer object:\n"
                "1) Select object from dropdown\n"
                "2) Select serialisation option\n"
                "3) Select encryption option\n"
                "4) Click Upload button to initiate transfer\n",
            )

        # create a button
        info_button = tk.Button(
            self.sub_frame6, text="Info", command=info_button_callback
        )
        exit_button = tk.Button(self.sub_frame6, text="Exit", command=self.root.destroy)
        info_button.pack(side=tk.LEFT, pady=5)
        exit_button.pack(side=tk.LEFT, pady=5)

    def update_text_box(self):
        """
        Updates UI console with user selections
        """

        self.text.delete("1.0", tk.END)
        text = "\n".join([f"{key}: {val}" for key, val in self.selections.items()])
        self.text.insert("1.0", text)
        if "Selected object" in self.selections:
            self.text.insert(
                tk.END, f"\nData selected: {DATA[self.selections['Selected object']]}"
            )

    def log_popup(self):
        """
        Provides user with popup showing file transfer log history
        """

        top = tk.Toplevel()
        top.resizable(True, True)
        top.geometry("500x400")
        top.title("Log History")

        # Create a text widget
        self.log_box = tk.Text(top)

        # place the widget
        self.log_box.pack(expand=True, fill=tk.BOTH)

        for line in self.get_log_output():
            self.log_box.insert(tk.END, line)

    def get_log_output(self):
        """
        Gets log history from log file

        :yield: each log line (only from current session)
        """

        filepath = "client_history.log"
        with open(filepath, "r") as file:
            for line in file.readlines():
                line_time = datetime.strptime(line[8:27], "%Y-%m-%d %H:%M:%S")
                if line_time >= (self.start_time - timedelta(seconds=1)):
                    yield line


import tweepy

class TwitterClient:
    def __init__(self,
                 consumer_key=None,
                 consumer_secret=None,
                 access_key=None,
                 access_secret=None):
        auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
        auth.set_access_token(access_key, access_secret)
        self.client = tweepy.API(auth)

    def send(self, message):
        self.client.update_status(message)


"""
Rotate by 90 degree
Given a square matrix mat[][] of size N x N. The task is to rotate it by 90 degrees in anti-clockwise direction without using any extra space.

Input:
The first line of input contains a single integer T denoting the number of test cases. Then T test cases follow. Each test case consist of two lines. The first line of each test case consists of an integer N, where N is the size of the square matrix. The second line of each test case contains N x N space separated values of the matrix mat.

Output:
Corresponding to each test case, in a new line, print the rotated matrix.

User Task:
The task is to complete the function rotateby90() which rotates the input matrix by 90 degree. You just need to rotate the matrix, printing the matrix will be automatically done the driver code.

Constraints:
1 ≤ T ≤ 50
1 ≤ N ≤ 50
1 <= mat[][] <= 100

Example:
Input:
3
3
1 2 3 4 5 6 7 8 9
2
1 2 3 4
1
80
Output:
3 6 9 2 5 8 1 4 7
2 4 1 3
80

Explanation:
Testcase 1: Matrix is as below:
1 2 3
4 5 6
7 8 9

Rotating it by 90 degrees in anticlockwise directions will result as below matrix:
3 6 9
2 5 8
1 4 7
Testcase 2: Matrix is as below:
1 2
3 4
After rotating by 90 degree , matrix will be
2 4
1 3
Testcase 3: Matrix is as below:
80
After rotating by 90 degree , matrix will be
80

** For More Input/Output Examples Use 'Expected Output' option **
"""
def transpose(mat,n):
    for x in range(n):
        for y in range(n):
            if(y>x):
                break
            temp = mat[x][y]
            mat[x][y]= mat[y][x]
            mat[y][x]=temp

def rotateby90(a, n):
    transpose(a,n)
    for x in range(n//2):
        temp = a[x]
        a[x] = a[n-x-1]
        a[n-x-1] = temp