# -*- coding: utf-8 -*-
"""
Created on Sun Mar 31 16:58:58 2019

@author: Ethan
"""

# -*- coding: utf-8 -*-
"""
Created on Thu Mar 21 11:25:12 2019

@author: Ethan
"""

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import statsmodels.api as sm
from sklearn import linear_model
import seaborn as sns
import pickle
import sys

#dataDir = '/dartfs-hpc/rc/lab/C/CMIG'
dataDir = 'e:/data/'

wxdata = 'era'

plotFigs = False

outageDaysOnly = False

dataset = 'all'

tempVar = 'txAvgSummer'

eData = {}
with open('eData.dat', 'rb') as f:
    eData = pickle.load(f)

entsoePlantData = eData['entsoePlantDataAll']
entsoeAgData = eData['entsoeAgDataAll']
nukeMatchData = eData['nukeMatchDataAll']
nukeAgData = eData['nukeAgDataAll']
nukePlantData = eData['nukePlantDataAll']

xtotal = []
if dataset == 'nuke' or dataset == 'all':
    xtotal.extend(nukeAgData[tempVar])

if dataset == 'entsoe' or dataset == 'all':
    xtotal.extend(entsoeAgData[tempVar])
xtotal = np.array(xtotal)

ytotal = []
if dataset == 'nuke' or dataset == 'all':
    ytotal.extend(nukeAgData['capacitySummer'])
if dataset == 'entsoe' or dataset == 'all':
    ytotal.extend(100*entsoeAgData['capacitySummer'])
ytotal = np.array(ytotal)

plantid = []
if dataset == 'nuke' or dataset == 'all':
    plantid.extend(nukeAgData['plantIds'])
if dataset == 'entsoe' or dataset == 'all':
    plantid.extend(entsoeAgData['plantIds'])
plantid = np.array(plantid)

plantMonths = []
if dataset == 'nuke' or dataset == 'all':
    plantMonths.extend(nukeAgData['plantMonths'])
if dataset == 'entsoe' or dataset == 'all':
    plantMonths.extend(entsoeAgData['plantMonths'])
plantMonths = np.array(plantMonths)

plantDays = []
if dataset == 'nuke' or dataset == 'all':
    plantDays.extend(nukeAgData['plantDays'])
if dataset == 'entsoe' or dataset == 'all':
    plantDays.extend(entsoeAgData['plantDays'])
plantDays = np.array(plantDays)

xtotalBool = []
if dataset == 'nuke' or dataset == 'all':
    xtotalBool.extend(nukeAgData[tempVar])
if dataset == 'entsoe' or dataset == 'all':
    xtotalBool.extend(entsoeAgData[tempVar])
xtotalBool = np.array(xtotalBool)

ytotalBool = []
if dataset == 'nuke' or dataset == 'all':
    ytotalBool.extend(nukeAgData['outagesBoolSummer'])
if dataset == 'entsoe' or dataset == 'all':
    ytotalBool.extend(entsoeAgData['outagesBoolSummer'])
ytotalBool = np.array(ytotalBool)

ind = np.where((ytotal <= 100.1) & (xtotal > 20))[0]
xtotal = xtotal[ind]
ytotal = ytotal[ind]

if outageDaysOnly:
    indOutages = np.where(ytotal<100)
    df = pd.DataFrame({'x':xtotal[indOutages], 'y':ytotal[indOutages]})

else:
    df = pd.DataFrame({'x':xtotal, 'y':ytotal})

plt.figure(figsize=(4,4))
plt.xlim([20,44])
plt.ylim([-25, 105])
sns.regplot(x='x', y='y', data=df, order=3, \
            scatter_kws={"color": [.5, .5, .5], "facecolor":[.75, .75, .75], "s":10, 'alpha':.25}, \
            line_kws={"color": [234/255., 49/255., 49/255.]})

sns.regplot(x='x', y='y', data=df, order=1, scatter=False, \
            line_kws={"color": [244/255., 153/255., 34/255.]})

#if outageDaysOnly:
#    plt.xlim([20, 44])
#    sns.regplot(x='x', y='y', data=df, lowess=True, scatter=False, \
#                line_kws={"color": [34/255., 171/255., 244/255.]})

plt.xlim([20,45])
plt.ylim([-25, 105])

for tick in plt.gca().xaxis.get_major_ticks():
    tick.label.set_fontname('Helvetica')
    tick.label.set_fontsize(14)
for tick in plt.gca().yaxis.get_major_ticks():
    tick.label.set_fontname('Helvetica')
    tick.label.set_fontsize(14)

plt.xlabel('Daily max temperature ($\degree$C)', fontname = 'Helvetica', fontsize=16)
plt.ylabel('Plant capacity (%)', fontname = 'Helvetica', fontsize=16)

plt.gca().set_xticks(range(20,45,4))
plt.gca().set_yticks(range(0,101,20))

x0,x1 = plt.gca().get_xlim()
y0,y1 = plt.gca().get_ylim()
plt.gca().set_aspect(abs(x1-x0)/abs(y1-y0))

#plt.title('July - August', fontname = 'Helvetica', fontsize=16)
if plotFigs:
    if outageDaysOnly:
        plt.savefig('nuke-eu-cap-reduction-outage-days.png', format='png', dpi=300, bbox_inches = 'tight', pad_inches = 0)
    else:
        plt.savefig('nuke-eu-cap-reduction-all-days.png', format='png', dpi=300, bbox_inches = 'tight', pad_inches = 0)

binstep = 4
bin_x1 = 20
bin_x2 = 44

bincounts = []
for t in range(bin_x1, bin_x2, binstep):
    if outageDaysOnly:
        bincounts.append(len(xtotal[(ytotal < 100) & (xtotal >= t) & (xtotal < t+binstep)]))
    else:
        bincounts.append(len(xtotal[(xtotal >= t) & (xtotal < t+binstep)]))

# plot hist of days in each temp bin
plt.figure(figsize=(6,1))
plt.xlim([19, 45])
#plt.ylim([0, 1])

plt.bar(range(bin_x1, bin_x2, binstep), bincounts, \
        facecolor = [.75, .75, .75], \
        edgecolor = [0, 0, 0], width = binstep, align = 'edge', \
        zorder=0)

plt.gca().set_xticks([])
plt.gca().set_xticklabels([])
if outageDaysOnly:
    plt.gca().set_yticks([0, 1500, 3000])
    plt.gca().set_yticklabels(['0', '1.5K', '3K'])
else:
    plt.gca().set_yticks([0, 10000, 20000])
    plt.gca().set_yticklabels(['0', '10K', '20K'])

plt.gca().set_yticks([])
plt.gca().set_yticklabels([])

for tick in plt.gca().xaxis.get_major_ticks():
    tick.label.set_fontname('Helvetica')
    tick.label.set_fontsize(14)
for tick in plt.gca().yaxis.get_major_ticks():
    tick.label.set_fontname('Helvetica')
    tick.label.set_fontsize(14)

if plotFigs:
    if outageDaysOnly:
        plt.savefig('temp-day-hist-only-outages-era.png', format='png', dpi=1000, bbox_inches = 'tight', pad_inches = 0)
    else:
        plt.savefig('temp-day-hist-all-days-era.png', format='png', dpi=1000, bbox_inches = 'tight', pad_inches = 0)

binstep = 20
bin_y1 = 0
bin_y2 = 105

bincounts = []
for t in range(bin_y1, bin_y2, binstep):
    if outageDaysOnly:
        bincounts.append(len(ytotal[(ytotal < 100) & (ytotal >= t) & (ytotal < t+binstep)]))
    else:
        bincounts.append(len(ytotal[(ytotal >= t) & (ytotal <= t+binstep)]))

# plot hist of days in each temp bin
plt.figure(figsize=(6,1))
plt.xlim([0, 100])
#plt.ylim([0, 1])

plt.bar(range(bin_y1, bin_y2, binstep), bincounts, \
        facecolor = [.75, .75, .75], \
        edgecolor = [0, 0, 0], width = binstep, align = 'edge', \
        zorder=0)

plt.gca().set_xticks([])
plt.gca().set_xticklabels([])
if outageDaysOnly:
    plt.gca().set_yticks([0, 2500, 5000])
    plt.gca().set_yticklabels(['0', '2.5K', '5K'])
else:
    plt.gca().set_yticks([0, 10000, 20000])
    plt.gca().set_yticklabels(['0', '10K', '20K'])

plt.gca().set_yticks([])
plt.gca().set_yticklabels([])

for tick in plt.gca().xaxis.get_major_ticks():
    tick.label.set_fontname('Helvetica')
    tick.label.set_fontsize(14)
for tick in plt.gca().yaxis.get_major_ticks():
    tick.label.set_fontname('Helvetica')
    tick.label.set_fontsize(14)

if plotFigs:
    if outageDaysOnly:
        plt.savefig('outage-day-hist-only-outages-era.png', format='png', dpi=1000, bbox_inches = 'tight', pad_inches = 0)
    else:
        plt.savefig('outage-day-hist-all-days-era.png', format='png', dpi=1000, bbox_inches = 'tight', pad_inches = 0)

dfLogistic = pd.DataFrame({'x':xtotalBool, 'y':ytotalBool})

binnedOutageData = []
binnedTx = []

binstep = 4
bin_x1 = 20
bin_x2 = 44

for i in range(entsoePlantData[tempVar].shape[0]):
    binnedOutageData.append([])
    for t in range(bin_x1, bin_x2, binstep):
        tempInds = np.where((entsoePlantData[tempVar][i] >= t) & (entsoePlantData[tempVar][i] < t+binstep))[0]
        if len(tempInds) > 0:
            binnedOutageData[i].append(np.nanmean(100 * entsoePlantData['outagesBoolSummer'][i, tempInds]))
        else:
            binnedOutageData[i].append(np.nan)

for i in range(nukeAgData['percCapacity'].shape[0]):
    binnedOutageData.append([])

    y = nukeAgData['percCapacity'][i]
    x = nukeMatchData['tx'][i,1:]

    if len(y) == len(x):
        y = y[nukeAgData['summerInds']]
        x = x[nukeAgData['summerInds']]
        nn = np.where((~np.isnan(y)) & (~np.isnan(x)))[0]

        # convert to boolean: 0 = no outage, > 0 = outage
        y = 100-y[nn]
        x = x[nn]
        y[y>0] = 100

        for t in range(bin_x1, bin_x2, binstep):
            tempInds = np.where((x >= t) & (x < t+binstep))[0]
            if len(tempInds) > 0:
                binnedOutageData[-1].append(np.nanmean(y[tempInds]))
            else:
                binnedOutageData[-1].append(np.nan)

binnedOutageData = np.array(binnedOutageData)
binnedOutageData = binnedOutageData / 100

boxy = []
boxyMeans = []
for c in range(binnedOutageData.shape[1]):
    boxy.append([])
    for i in range(binnedOutageData.shape[0]):
        if not np.isnan(binnedOutageData[i, c]):
            boxy[-1].append(binnedOutageData[i, c])
    boxyMeans.append(np.mean(boxy[-1]))

z = np.polyfit(range(bin_x1, bin_x2, binstep), np.nanmean(binnedOutageData, axis=0), 3)
p = np.poly1d(z)

plt.figure(figsize=(4,4))
#plt.bar(range(bin_x1, bin_x2, binstep), np.nanmean(binnedOutageData, axis=0),\
#        yerr = np.nanstd(binnedOutageData, axis=0)/2, \
#        facecolor = [.75, .75, .75], \
#        edgecolor = [0, 0, 0], width = 2, align = 'edge', \
#        error_kw=dict(lw=2, capsize=3, capthick=2), ecolor = [.25, .25, .25], zorder=0)

medianprops = dict(linestyle='-', linewidth=2, color='black')
meanpointprops = dict(marker='D', markeredgecolor='black',
                      markerfacecolor='red')
bplot = plt.boxplot(boxy, positions = range(22, 44, 4), showmeans=True, widths=4, sym='.', patch_artist=True, \
                    medianprops=medianprops, meanprops=meanpointprops, zorder=0)

colors = ['lightgray']
for patch in bplot['boxes']:
    patch.set_facecolor([.75, .75, .75])

plt.xlim([20, 44])
sns.regplot(x='x', y='y', data=dfLogistic, logistic=True, scatter=False, \
            line_kws={"color": [234/255., 49/255., 49/255.]})

#plt.plot(range(21, 44), p(range(20, 43)), "--", linewidth = 2, color = [234/255., 49/255., 49/255.])
plt.xlim([19, 45])
plt.ylim([0, .55])

plt.gca().set_xticks(range(20,45,4))
plt.gca().set_xticklabels(range(20,45,4))
plt.gca().set_yticks(np.arange(0,1.1,.20))
plt.gca().set_yticklabels(range(0,110,20))

for tick in plt.gca().xaxis.get_major_ticks():
    tick.label.set_fontname('Helvetica')
    tick.label.set_fontsize(14)
for tick in plt.gca().yaxis.get_major_ticks():
    tick.label.set_fontname('Helvetica')
    tick.label.set_fontsize(14)

plt.xlabel('Daily max temperature ($\degree$C)', fontname = 'Helvetica', fontsize=16)
plt.ylabel('Chance of outage (%)', fontname = 'Helvetica', fontsize=16)

x0,x1 = plt.gca().get_xlim()
y0,y1 = plt.gca().get_ylim()
plt.gca().set_aspect(abs(x1-x0)/abs(y1-y0))
if plotFigs:
    plt.savefig('nuke-eu-perc-plants-with-outages-%s.png'%wxdata, format='png', dpi=1000, bbox_inches = 'tight', pad_inches = 0)


import numpy as np
import cv2
import random
import math
import os
import threading
import queue
import sys
import h5py
import copy

FETCH_BATCH_SIZE = 32
BATCH_SIZE = 32
HEIGHT = 192
WIDTH = 256
POINTCLOUDSIZE = 16384
OUTPUTPOINTS = 1024
REEBSIZE = 1024

def get_filelist(lst_dir, maxnverts, minsurbinvox, cats, cats_info, type):
    for cat in cats:
        cat_id = cats_info[cat]
    inputlistfile = os.path.join(lst_dir, cat_id + type + ".lst")
    with open(inputlistfile, 'r') as f:
        lines = f.read().splitlines()
        file_lst = [[cat_id, line.strip()] for line in lines]
    return file_lst

class Pt_sdf_img(threading.Thread):
    def __init__(self, FLAGS, listinfo=None, info=None, qsize=64, cats_limit=None, shuffle=True):
        super(Pt_sdf_img, self).__init__()
        self.queue = queue.Queue(qsize)
        self.stopped = False
        self.bno = 0
        self.listinfo = listinfo
        self.num_points = FLAGS.num_points
        self.gen_num_pt = FLAGS.num_sample_points
        self.batch_size = FLAGS.batch_size
        self.img_dir = info['rendered_dir']
        self.sdf_dir = info['sdf_dir']
        self.cache = {}  # from index to (point_set, cls, seg) tuple
        self.cache_size = 60000
        self.data_num = len(self.listinfo)
        self.FLAGS = FLAGS
        self.shuffle = shuffle
        self.num_batches = int(self.data_num / self.FLAGS.batch_size)
        self.cats_limit, self.epoch_amount = self.set_cat_limit(cats_limit)
        self.data_order = list(range(len(listinfo)))
        self.order = self.data_order

    # def get_img_dir_old(self, cat_id, obj):
    #     img_dir = os.path.join(self.img_dir, cat_id, obj, "rendering")
    #
    #     with open(img_dir + "/renderings.txt", 'r') as f:
    #         lines = f.read().splitlines()
    #         file_lst = [line.strip() for line in lines]
    #     return img_dir, file_lst
    def set_cat_limit(self, cats_limit):
        epoch_amount = 0
        for cat, amount in cats_limit.items():
            cats_limit[cat] = min(self.FLAGS.cat_limit, amount)
            epoch_amount += cats_limit[cat]
        print("epoch_amount ", epoch_amount)
        print("cats_limit ", cats_limit)
        return cats_limit, epoch_amount

    def get_img_dir(self, cat_id, obj):
        img_dir = os.path.join(self.img_dir, cat_id, obj)
        return img_dir, None

    def get_sdf_h5_filenm(self, cat_id, obj):
        return os.path.join(self.sdf_dir, cat_id, obj, "ori_sample.h5")

    def pc_normalize(self, pc, centroid=None):

        """ pc: NxC, return NxC """
        l = pc.shape[0]

        if centroid is None:
            centroid = np.mean(pc, axis=0)

        pc = pc - centroid
        # m = np.max(pc, axis=0)
        m = np.max(np.sqrt(np.sum(pc ** 2, axis=1)))

        pc = pc / m

        return pc, centroid, m

    def __len__(self):
        return self.epoch_amount

    def memory(self):
        """
        Get node total memory and memory usage
        """
        with open('/proc/meminfo', 'r') as mem:
            ret = {}
            tmp = 0
            for i in mem:
                sline = i.split()
                if str(sline[0]) == 'MemTotal:':
                    ret['total'] = int(sline[1])
                elif str(sline[0]) in ('MemFree:', 'Buffers:', 'Cached:'):
                    tmp += int(sline[1])
            ret['free'] = tmp
            ret['used'] = int(ret['total']) - int(ret['free'])
        return ret

    def getitem(self, index):
        cat_id, obj, num = self.listinfo[index]
        sdf_file = self.get_sdf_h5_filenm(cat_id, obj)
        ori_pt, ori_sdf_val, sample_pt, sample_sdf_val, norm_params, sdf_params\
            = self.get_sdf_h5(sdf_file, cat_id, obj)
        img_dir, img_file_lst = self.get_img_dir(cat_id, obj)
        return ori_pt, ori_sdf_val, sample_pt, sample_sdf_val, norm_params,\
               sdf_params, img_dir, img_file_lst, cat_id, obj, num

    def get_sdf_h5(self, sdf_h5_file, cat_id, obj):
        h5_f = h5py.File(sdf_h5_file, 'r')
        try:
            if ('pc_sdf_original' in h5_f.keys()
                    and 'pc_sdf_sample' in h5_f.keys()
                    and 'norm_params' in h5_f.keys()):
                ori_sdf = h5_f['pc_sdf_original'][:].astype(np.float32)
                # sample_sdf = np.reshape(h5_f['pc_sdf_sample'][:],(ori_sdf.shape[0], -1 ,4)).astype(np.float32)
                sample_sdf = h5_f['pc_sdf_sample'][:].astype(np.float32)
                ori_pt = ori_sdf[:,:3]#, ori_sdf[:,3]
                ori_sdf_val = None
                if sample_sdf.shape[1] == 4:
                    sample_pt, sample_sdf_val = sample_sdf[:, :3], sample_sdf[:, 3]
                else:
                    sample_pt, sample_sdf_val = None, sample_sdf[:, 0]
                norm_params = h5_f['norm_params'][:]
                sdf_params = h5_f['sdf_params'][:]
            else:
                raise Exception(cat_id, obj, "no sdf and sample")
        finally:
            h5_f.close()
        return ori_pt, ori_sdf_val, sample_pt, sample_sdf_val, norm_params, sdf_params

    def get_img_old(self, img_dir, num, file_lst):
        params = np.loadtxt(img_dir + "/rendering_metadata.txt")
        img_file = os.path.join(img_dir, file_lst[num])
        # azimuth, elevation, in-plane rotation, distance, the field of view.
        param = params[num, :].astype(np.float32)
        cam_mat, cam_pos = self.camera_info(self.degree2rad(param))
        img_arr = cv2.imread(img_file, cv2.IMREAD_UNCHANGED)[:,:,:3].astype(np.float32) / 255.
        return img_arr, cam_mat, cam_pos

    def get_img(self, img_dir, num):
        img_h5 = os.path.join(img_dir, "%02d.h5"%num)
        cam_mat, cam_pos, trans_mat, obj_rot_mat, regress_mat = None, None, None, None, None
        with h5py.File(img_h5, 'r') as h5_f:
            if self.FLAGS.img_feat_onestream or self.FLAGS.img_feat_twostream:
                trans_mat = h5_f["trans_mat"][:].astype(np.float32)
                obj_rot_mat = h5_f["obj_rot_mat"][:].astype(np.float32)
                regress_mat = h5_f["regress_mat"][:].astype(np.float32)
            else:
                cam_mat, cam_pos = h5_f["cam_mat"][:].astype(np.float32), h5_f["cam_pos"][:].astype(np.float32)
            if self.FLAGS.alpha:
                img_arr = h5_f["img_arr"][:].astype(np.float32)
                img_arr[:, :, :4] = img_arr[:,:,:4] / 255.
            else:
                img_raw = h5_f["img_arr"][:]
                img_arr = img_raw[:, :, :3]
                if self.FLAGS.augcolorfore or self.FLAGS.augcolorback:
                    r_aug = 60 * np.random.rand() - 30
                    g_aug = 60 * np.random.rand() - 30
                    b_aug = 60 * np.random.rand() - 30
                if self.FLAGS.augcolorfore:
                    img_arr[img_raw[:, :, 3] != 0, 0] + r_aug
                    img_arr[img_raw[:, :, 3] != 0, 1] + g_aug
                    img_arr[img_raw[:, :, 3] != 0, 2] + b_aug
                if self.FLAGS.backcolorwhite:
                    img_arr[img_raw[:, :, 3] == 0] = [255, 255, 255]
                if self.FLAGS.augcolorback:
                    img_arr[img_raw[:, :, 3] == 0, 0] + r_aug
                    img_arr[img_raw[:, :, 3] == 0, 1] + g_aug
                    img_arr[img_raw[:, :, 3] == 0, 2] + b_aug
                img_arr = np.clip(img_arr, 0, 255)
                img_arr = img_arr.astype(np.float32) / 255.

            return img_arr, cam_mat, cam_pos, trans_mat, obj_rot_mat, regress_mat

    def degree2rad(self, params):
        params[0] = np.deg2rad(params[0] + 180.0)
        params[1] = np.deg2rad(params[1])
        params[2] = np.deg2rad(params[2])
        return params

    def unit(self, v):
        norm = np.linalg.norm(v)
        if norm == 0:
            return v
        return v / norm

    def camera_info(self, param):
        az_mat = self.get_az(param[0])
        el_mat = self.get_el(param[1])
        inl_mat = self.get_inl(param[2])
        cam_mat = np.transpose(np.matmul(np.matmul(inl_mat, el_mat), az_mat))
        cam_pos = self.get_cam_pos(param)
        return cam_mat, cam_pos

    def get_cam_pos(self, param):
        camX = 0
        camY = 0
        camZ = param[3]
        cam_pos = np.array([camX, camY, camZ])
        return -1 * cam_pos

    def get_az(self, az):
        cos = np.cos(az)
        sin = np.sin(az)
        mat = np.asarray([cos, 0.0, sin, 0.0, 1.0, 0.0, -1.0*sin, 0.0, cos], dtype=np.float32)
        mat = np.reshape(mat, [3,3])
        return mat
    #
    def get_el(self, el):
        cos = np.cos(el)
        sin = np.sin(el)
        mat = np.asarray([1.0, 0.0, 0.0, 0.0, cos, -1.0*sin, 0.0, sin, cos], dtype=np.float32)
        mat = np.reshape(mat, [3,3])
        return mat
    #
    def get_inl(self, inl):
        cos = np.cos(inl)
        sin = np.sin(inl)
        # zeros = np.zeros_like(inl)
        # ones = np.ones_like(inl)
        mat = np.asarray([cos, -1.0*sin, 0.0, sin, cos, 0.0, 0.0, 0.0, 1.0], dtype=np.float32)
        mat = np.reshape(mat, [3,3])
        return mat

    def get_batch(self, index):
        if index + self.batch_size > self.epoch_amount:
            index = index + self.batch_size - self.epoch_amount
        batch_pc = np.zeros((self.batch_size, self.num_points, 3)).astype(np.float32)
        batch_sdf_pt = np.zeros((self.batch_size, self.gen_num_pt, 3)).astype(np.float32)
        batch_sdf_pt_rot = np.zeros((self.batch_size, self.gen_num_pt, 3)).astype(np.float32)
        batch_sdf_val = np.zeros((self.batch_size, self.gen_num_pt, 1)).astype(np.float32)
        batch_norm_params = np.zeros((self.batch_size, 4)).astype(np.float32)
        batch_sdf_params = np.zeros((self.batch_size, 6)).astype(np.float32)
        if self.FLAGS.alpha:
            batch_img = np.zeros((self.batch_size, self.FLAGS.img_h, self.FLAGS.img_w, 4), dtype=np.float32)
        else:
            batch_img = np.zeros((self.batch_size, self.FLAGS.img_h, self.FLAGS.img_w, 3), dtype=np.float32)
        batch_regress_mat = np.zeros((self.batch_size, 4, 3), dtype=np.float32)
        batch_trans_mat = np.zeros((self.batch_size, 4, 3), dtype=np.float32)
        batch_cat_id = []
        batch_obj_nm = []
        batch_view_id = []
        cnt = 0
        for i in range(index, index + self.batch_size):
            single_obj = self.getitem(self.order[i])
            if single_obj == None:
                raise Exception("single mesh is None!")
            ori_pt, ori_sdf_val, sample_pt, sample_sdf_val, norm_params, sdf_params, img_dir, img_file_lst, cat_id, obj, num = single_obj
            img, cam_mat, cam_pos, trans_mat, obj_rot_mat, regress_mat = self.get_img(img_dir, num)
            if ori_pt is not None:
                cf_ref_choice = np.random.randint(ori_pt.shape[0], size=self.num_points)
                batch_pc[cnt, :, :] = ori_pt[cf_ref_choice, :]
                if self.FLAGS.threedcnn:
                    batch_sdf_val[cnt, :, 0] = sample_sdf_val
                else:
                    if self.gen_num_pt > sample_pt.shape[0]:
                        choice = np.random.randint(sample_pt.shape[0], size=self.gen_num_pt)
                    else:
                        choice = np.asarray(random.sample(range(sample_pt.shape[0]), self.gen_num_pt), dtype=np.int32)
                    batch_sdf_pt[cnt, ...] = sample_pt[choice, :]
                    batch_sdf_val[cnt, :, 0] = sample_sdf_val[choice]
                    if self.FLAGS.rot:
                        batch_sdf_pt_rot[cnt, ...] = np.dot(sample_pt[choice, :], obj_rot_mat)
                    else:
                        batch_sdf_pt_rot[cnt, ...] = sample_pt[choice, :]
                batch_norm_params[cnt, ...] = norm_params
                batch_sdf_params[cnt, ...] = sdf_params
            else:
                raise Exception("no verts or binvox")
            # img, cam_mat, cam_pos = self.get_img_old(img_dir, num, img_file_lst)
            batch_img[cnt, ...] = img.astype(np.float32)
            batch_regress_mat[cnt, ...] = regress_mat
            batch_trans_mat[cnt, ...] = trans_mat
            batch_cat_id.append(cat_id)
            batch_obj_nm.append(obj)
            batch_view_id.append(num)
            cnt += 1
        batch_data = {}
        batch_data['pc'] = batch_pc
        batch_data['sdf_pt'] = batch_sdf_pt
        batch_data['sdf_pt_rot'] = batch_sdf_pt_rot
        batch_data['sdf_val'] = batch_sdf_val
        batch_data['norm_params'] = batch_norm_params
        batch_data['sdf_params'] = batch_sdf_params
        batch_data['img'] = batch_img
        batch_data['trans_mat'] = batch_trans_mat
        batch_data['cat_id'] = batch_cat_id
        batch_data['obj_nm'] = batch_obj_nm
        batch_data['view_id'] = batch_view_id
        return batch_data

    def refill_data_order(self):
        temp_order = copy.deepcopy(self.data_order)
        cats_quota = {key: value for key, value in self.cats_limit.items()}
        np.random.shuffle(temp_order)
        pointer = 0
        epoch_order=[]
        while len(epoch_order) < self.epoch_amount:
            cat_id, _, _ = self.listinfo[temp_order[pointer]]
            if cats_quota[cat_id] > 0:
                epoch_order.append(temp_order[pointer])
                cats_quota[cat_id]-=1
            pointer+=1
        return epoch_order

    def work(self, epoch, index):
        if index == 0 and self.shuffle:
            self.order = self.refill_data_order()
            print("data order reordered!")
        return self.get_batch(index)

    def run(self):
        while (self.bno // (self.num_batches* self.batch_size)) < self.FLAGS.max_epoch and not self.stopped:
            self.queue.put(self.work(self.bno // (self.num_batches* self.batch_size),
                                     self.bno % (self.num_batches * self.batch_size)))
            self.bno += self.batch_size

    def fetch(self):
        if self.stopped:
            return None
        # else:
        #     print("queue length", self.queue.qsize())
        return self.queue.get()

    def shutdown(self):
        self.stopped = True
        while not self.queue.empty():
            self.queue.get()

if __name__ == '__main__':

    sys.path.append('../preprocessing/')
    import create_file_lst as create

    data = Pt_sdf_img(res=256, expr=1.5,
                      listinfo=[["03001627", "ff3581996365bdddc3bd24f986301745"],
                                ["03001627", "ff3581996365bdddc3bd24f986301745"]],
                      info=create.get_all_info(), maxnverts=6000, maxntris=50000,
                      minsurbinvox=4096, num_points=2048, batch_size=2, normalize=False, norm_color=True)
    batch1 = data.get_batch(0)
    print(batch1.keys())
    print(batch1["verts"].shape)
    print(batch1["nverts"])
    print(batch1["tris"].shape)
    print(batch1["ntris"])
    print(batch1["surfacebinvoxpc"].shape)
    print(batch1["sdf"].shape)
    print(batch1["sdf_params"])
    print(batch1["img"].shape, batch1["img"][0, 64, 64, :])
    print(batch1["img_cam"])

    # (2048, 3)
    cloud1 = batch1["surfacebinvoxpc"][0, ...]
    trans1 = batch1["img_cam"][0, ...]
    az1 = float(trans1[0] + 180) * math.pi / 180.0
    el1 = float(trans1[1]) * math.pi / 180.0
    in1 = float(trans1[2]) * math.pi / 180.0
    transmatrix_az1 = [[math.cos(az1), 0, math.sin(az1)],
                       [0, 1, 0],
                       [-math.sin(az1), 0, math.cos(az1)]]
    transmatrix_az1 = np.asarray(transmatrix_az1).astype(np.float32)
    transmatrix_el1 = [[1, 0, 0],
                       [0, math.cos(el1), -math.sin(el1)],
                       [0, math.sin(el1), math.cos(el1)]]
    transmatrix_el1 = np.asarray(transmatrix_el1).astype(np.float32)
    transmatrix_in1 = [[math.cos(in1), -math.sin(in1), 0],
                       [math.sin(in1), math.cos(in1), 0],
                       [0, 0, 1]]
    transmatrix_in1 = np.asarray(transmatrix_in1).astype(np.float32)

    trans = np.matmul(np.matmul(transmatrix_in1, transmatrix_el1), transmatrix_az1)
    translate1 = np.tile(np.expand_dims(np.asarray([-trans1[2], 0, 0]).astype(np.float32), axis=0), (2048, 1))
    points = np.matmul(cloud1, trans.T)
    np.savetxt("ff_rotate.xyz", points)
    np.savetxt("ff.xyz", cloud1)


# # # # -*- coding: utf-8 -*-
class Solution(object):

    def wordBreak(self, s, wordDict):
        """
        :type s: str
        :type wordDict: List[str]
        :rtype: bool
        """
        return self.helper(s, wordDict, 0, {})

    def helper(self, s, wordDict, start, temp):
        if start == len(s):
            return True
        if start in temp:
            return temp[start]
        for i in range(start+1, len(s)+1):
            sub_string = s[start:i]
            if sub_string in wordDict and self.helper(s, wordDict, i, temp):
                temp[start] = True
                return True
        return False

if __name__ == '__main__':
    s = Solution()
    word = "applepenapple"
    wordDict = ["apple", "pen"]
    # word = "catsandog"
    # wordDict = ["cats", "dog", "sand", "and", "cat"]
    # word = "a"
    # wordDict = ["b"]
    print(s.wordBreak(word, wordDict))


# -*- coding: utf-8 -*-

from aoc import *

inp = input_as_grid("twentytwo.txt", str)
#inp = input_as_values("ex.txt", int)

# state where only infected nodes are part of the set
def init_state_set():
    state = set()
    for ydx, y in enumerate(inp):
        for xdx, x in enumerate(y):
            if x == '#':
                state.add((xdx, ydx))

    return state

# state where only non-clean nodes are part of the dict
# states will be I, W or F
def init_state_dict():
    state = {}
    for ydx, y in enumerate(inp):
        for xdx, x in enumerate(y):
            if x == '#':
                state[(xdx, ydx)] = 'I'

    return state

# 5261
def part1():
    # starting move
    move_idx = 0
    # moves (x, y) U R D L
    moves = [(0, -1), (1, 0), (0, 1), (-1, 0)]
    # init state holding (x, y) tuples
    state = init_state_set()
    # starting at center of grid
    pos_x = int(len(inp[0]) / 2)
    pos_y = int(len(inp) / 2)

    infections = 0
    bursts = 10000
    for i in range(bursts):
        # current
        current = (pos_x, pos_y)

        # check node
        infected = current in state

        # infect / disinfect
        if infected:
            state.remove(current)
        else:
            infections += 1
            state.add(current)

        # turn
        turn = 1 if infected else -1
        move_idx = (move_idx + turn) % 4

        # move
        pos_x += moves[move_idx][0]
        pos_y += moves[move_idx][1]

    print("Part1 infections", infections)

# 2511927
def part2():
    # starting move
    move_idx = 0
    # moves (x, y) U R D L
    moves = [(0, -1), (1, 0), (0, 1), (-1, 0)]
    # init state holding (x, y) tuples
    state = init_state_dict()
    # starting at center of grid
    pos_x = int(len(inp[0]) / 2)
    pos_y = int(len(inp) / 2)

    infections = 0
    bursts = 10000000
    for i in range(bursts):
        # current
        current = (pos_x, pos_y)

        # check node
        node_status = 'clean'
        try:
            node_status = state[current]
        except:
            pass

        # infect / disinfect
        if node_status == 'clean':
            state[current] = 'W'
        elif node_status == 'W':
            infections += 1
            state[current] = 'I'
        elif node_status == 'I':
            state[current] = 'F'
        elif node_status == 'F':
            del state[current]

        # turn
        if node_status == 'clean':
            turn = -1
        elif node_status == 'W':
            turn = 0 # do not turn
        elif node_status == 'I':
            turn = 1
        elif node_status == 'F':
            turn = 2

        # only change move idx if we're changing direction
        if turn != 0:
            move_idx = (move_idx + turn) % 4

        # move
        pos_x += moves[move_idx][0]
        pos_y += moves[move_idx][1]

    print("Part2 infections", infections)

def main():
    time_it(part1)
    time_it(part2)

if __name__ == "__main__":
    main()


import numpy as np
from sklearn import metrics
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV

from sklearn import svm

data = pd.read_csv("cardio_train_clean_featureselection.csv")

# Features = xfeat, Target Variable = Y
xfeat = data[['Age', 'Gender', 'Height', 'Weight', "Systolic BP",
              "Chlosterol ", "Glucose", "Smoke", "Alcohol", "Active"]]

y = data['Cardio']

x_train, x_test, y_train, y_test = train_test_split(xfeat, y, test_size=0.2, random_state=0)

# SVM

svm_model = svm.SVC()
svm_model.fit(x_train, y_train)
y_pred_svm = svm_model.predict(x_test)

confusion_matrix_svm = pd.crosstab(y_test, y_pred_svm, rownames=['Actual'], colnames=['Predicted'])
sns.heatmap(confusion_matrix_svm, annot=True)

print('SVM Metrics:')
print('Accuracy: ', metrics.accuracy_score(y_test, y_pred_svm))
print('Precision: ', metrics.precision_score(y_test, y_pred_svm))
print('Recall: ', metrics.recall_score(y_test, y_pred_svm))
print()
plt.show()

# Tuning
'''
parameters = {'kernel': ('linear', 'rbf'), 'C': [1, 10]}

search = GridSearchCV(svm_model, parameters, scoring = 'recall')

results = search.fit(x_test, y_test)

print('Mean Recall: ', results.best_score_)
print('Config: ', results.best_params_)

'''
svm_model = svm.SVC(kernel='linear', C=10)
svm_model.fit(x_train, y_train)
y_pred_svm = svm_model.predict(x_test)

confusion_matrix_svm = pd.crosstab(y_test, y_pred_svm, rownames=['Actual'], colnames=['Predicted'])
sns.heatmap(confusion_matrix_svm, annot=True)

print('Tuned SVM Metrics:')
print('Accuracy: ', metrics.accuracy_score(y_test, y_pred_svm))
print('Precision: ', metrics.precision_score(y_test, y_pred_svm))
print('Recall: ', metrics.recall_score(y_test, y_pred_svm))
print()
plt.show()

Data Exploration, Training, Evaluation/SVM_Tuned.py


import logging
import os
from argparse import ArgumentParser
from typing import Optional

from bsf.analysis import Analysis, Stage
from bsf.executables.collection import RunnableStepCollectionPruneFastq
from bsf.ngs import Collection, Sample, PairedReads, Reads
from bsf.procedure import FilePath, ConsecutiveRunnable
from bsf.process import Command, RunnableStep, RunnableStepJava, RunnableStepMakeDirectory
from bsf.standards import Configuration, JavaArchive

module_logger = logging.getLogger(name=__name__)

class FilePathTrimmomaticReadGroup(FilePath):
    """The :py:class:`bsf.analyses.trimmomatic.FilePathTrimmomaticReadGroup` class models
    read group-specific Trimmomatic files.

    :ivar output_directory: An output directory path.
    :type output_directory: str
    :ivar trim_log_tsv: A Trimmomatic trim log :emphasis:`Tab-Separated Value` (TSV) file path.
    :type trim_log_tsv: str
    :ivar summary_tsv: A summary :emphasis:`Tab-Separated Value` (TSV) file path.
    :type summary_tsv: str
    :ivar coverage_png: A coverage :emphasis:`Portable Network Graphics` (PNG) file path.
    :type coverage_png: str
    :ivar frequency_png: A frequency :emphasis:`Portable Network Graphics` (PNG) file path.
    :type frequency_png: str
    :ivar surviving_png: A surviving :emphasis:`Portable Network Graphics` (PNG) file path.
    :type surviving_png: str
    :ivar reads_1p: A First Reads paired file path.
    :type reads_1p: str | None
    :ivar reads_1u: First Reads unpaired file path.
    :type reads_1u: str | None
    :ivar reads_2p: Second Reads paired file path.
    :type reads_2p: str | None
    :ivar reads_2u: Second Reads unpaired file path.
    :type reads_2u: str | None
    """

    def __init__(self, prefix: str) -> None:
        """Initialise a :py:class:`bsf.analyses.trimmomatic.FilePathTrimmomaticReadGroup` object.

        :param prefix: A Python :py:class:`str` prefix representing a :py:attr:`bsf.procedure.Runnable.name` attribute.
        :type prefix: str
        """
        super(FilePathTrimmomaticReadGroup, self).__init__(prefix=prefix)

        self.output_directory = prefix
        # Automatic GNU Zip-compression of trim log files does not work.
        self.trim_log_tsv = prefix + '_trim_log.tsv'
        self.trim_log_tsv_gz = prefix + '_trim_log.tsv.gz'
        self.summary_tsv = prefix + '_summary.tsv'  # Defined by the R script.
        self.coverage_png = prefix + '_coverage.png'  # Defined by the R script.
        self.frequency_png = prefix + '_frequency.png'  # Defined by the R script.
        self.surviving_png = prefix + '_surviving.png'  # Defined by the R script.
        self.reads_1p = None
        self.reads_1u = None
        self.reads_2p = None
        self.reads_2u = None

        return

class FilePathTrimmomaticProject(FilePath):
    """The :py:class:`bsf.analyses.trimmomatic.FilePathTrimmomaticProject` class models
    project-specific Trimmomatic files.

    :ivar output_directory: An output directory path.
    :type output_directory: str
    :ivar sas_path_old: An old Sample Annotation Sheet file path.
    :type sas_path_old: str
    :ivar sas_path_new: A new Sample Annotation Sheet file path.
    :type sas_path_new: str
    """

    def __init__(self, prefix: str, prefix_analysis: str, project_name: str) -> None:
        """Initialise a :py:class:`bsf.analyses.trimmomatic.FilePathTrimmomaticProject` object.

        :param prefix: A Python :py:class:`str` prefix representing a :py:attr:`bsf.procedure.Runnable.name` attribute.
        :type prefix: str
        :param prefix_analysis: A :py:attr:`bsf.analysis.Analysis.prefix` attribute.
        :type prefix_analysis: str
        :param project_name: A project name.
        :type project_name: str
        """
        super(FilePathTrimmomaticProject, self).__init__(prefix=prefix)

        self.output_directory = prefix
        self.sas_path_old = '_'.join((project_name, prefix_analysis, 'original.csv'))
        self.sas_path_new = '_'.join((project_name, prefix_analysis, 'samples.csv'))

        return

class Trimmomatic(Analysis):
    """The :py:class:`bsf.analyses.trimmomatic.Trimmomatic` class represents the logic to run the Trimmomatic analysis.

    :ivar adapter_path: An adapter definition file path.
    :type adapter_path: str | None
    :ivar trimming_step_pe_list: A colon-separated list of Trimmomatic steps for paired-end data.
    :type trimming_step_pe_list: list[str] | None
    :ivar trimming_step_se_list: A colon-separated list of Trimmomatic steps for single-end data.
    :type trimming_step_se_list: list[str] | None
    :ivar java_archive_trimmomatic: A Trimmomatic tool :emphasis:`Java Archive` (JAR) file path.
    :type java_archive_trimmomatic: str | None
    """

    name = 'Trimmomatic Analysis'
    prefix = 'trimmomatic'

    @classmethod
    def get_stage_name_read_group(cls) -> str:
        """Get a particular :py:attr:`bsf.analysis.Stage.name` attribute.

        :return: A :py:attr:`bsf.analysis.Stage.name` attribute.
        :rtype: str
        """
        return '_'.join((cls.prefix, 'read_group'))

    @classmethod
    def get_stage_name_summary(cls) -> str:
        """Get a particular :py:attr:`bsf.analysis.Stage.name` attribute.

        :return: A :py:attr:`bsf.analysis.Stage.name` attribute.
        :rtype: str
        """
        return '_'.join((cls.prefix, 'summary'))

    @classmethod
    def get_stage_name_project(cls) -> str:
        """Get a particular :py:attr:`bsf.analysis.Stage.name` attribute.

        :return: A :py:attr:`bsf.analysis.Stage.name` attribute.
        :rtype: str
        """
        return '_'.join((cls.prefix, 'project'))

    @classmethod
    def get_prefix_read_group(cls, read_group_name: str) -> str:
        """Get a Python :py:class:`str` prefix representing a :py:class:`bsf.procedure.Runnable` object.

        :param read_group_name: A read group name.
        :type read_group_name: str
        :return: A Python :py:class:`str` (prefix)  object representing a :py:class:`bsf.procedure.Runnable` object.
        :rtype: str
        """
        return '_'.join((cls.get_stage_name_read_group(), read_group_name))

    @classmethod
    def get_prefix_summary(cls) -> str:
        """Get a Python :py:class:`str` prefix representing a :py:class:`bsf.procedure.Runnable` object.

        :return: A Python :py:class:`str` (prefix)  object representing a :py:class:`bsf.procedure.Runnable` object.
        :rtype: str
        """
        return cls.get_stage_name_summary()

    @classmethod
    def get_prefix_project(cls) -> str:
        """Get a Python :py:class:`str` prefix representing a :py:class:`bsf.procedure.Runnable` object.

        :return: A Python :py:class:`str` (prefix)  object representing a :py:class:`bsf.procedure.Runnable` object.
        :rtype: str
        """
        return cls.get_stage_name_project()

    @classmethod
    def get_file_path_read_group(cls, read_group_name: str) -> FilePathTrimmomaticReadGroup:
        """Get a :py:class:`bsf.analyses.trimmomatic.FilePathTrimmomaticReadGroup` object.

        :param read_group_name: A read group name.
        :type read_group_name: str
        :return: A :py:class:`bsf.analyses.trimmomatic.FilePathTrimmomaticReadGroup` object.
        :rtype: FilePathTrimmomaticReadGroup
        """
        return FilePathTrimmomaticReadGroup(
            prefix=cls.get_prefix_read_group(read_group_name=read_group_name))

    @classmethod
    def get_file_path_project(cls, project_name: str, prefix_analysis: str) -> FilePathTrimmomaticProject:
        """Get a :py:class:`bsf.analyses.trimmomatic.FilePathTrimmomaticProject` object.

        :param project_name: A project name.
        :type project_name: str
        :param prefix_analysis: A :py:attr:`bsf.analysis.Analysis.prefix` attribute.
        :type prefix_analysis: str
        :return: A :py:class:`bsf.analyses.trimmomatic.FilePathTrimmomaticProject` object.
        :rtype: FilePathTrimmomaticProject
        """
        return FilePathTrimmomaticProject(
            prefix=cls.get_prefix_project(),
            prefix_analysis=prefix_analysis,
            project_name=project_name)

    def __init__(
            self,
            configuration: Optional[Configuration] = None,
            project_name: Optional[str] = None,
            genome_version: Optional[str] = None,
            input_directory: Optional[str] = None,
            output_directory: Optional[str] = None,
            project_directory: Optional[str] = None,
            genome_directory: Optional[str] = None,
            report_style_path: Optional[str] = None,
            report_header_path: Optional[str] = None,
            report_footer_path: Optional[str] = None,
            e_mail: Optional[str] = None,
            stage_list: Optional[list[Stage]] = None,
            collection: Optional[Collection] = None,
            sample_list: Optional[list[Sample]] = None,
            adapter_path: Optional[str] = None,
            trimming_step_pe_list: Optional[list[str]] = None,
            trimming_step_se_list: Optional[list[str]] = None,
            java_archive_trimmomatic: Optional[str] = None) -> None:
        """Initialise a :py:class:`bsf.analyses.trimmomatic.Trimmomatic` object.

        :param configuration: A :py:class:`bsf.standards.Configuration` object.
        :type configuration: Configuration | None
        :param project_name: A project name.
        :type project_name: str | None
        :param genome_version: A genome assembly version.
        :type genome_version: str | None
        :param input_directory: An input directory path.
        :type input_directory: str | None
        :param output_directory: An output directory path.
        :type output_directory: str | None
        :param project_directory: A project directory path, normally under the output directory path.
        :type project_directory: str | None
        :param genome_directory: A genome directory path, normally under the project directory path.
        :type genome_directory: str | None
        :param report_style_path: A report style :literal:`CSS` file path.
        :type report_style_path: str | None
        :param report_header_path: A report header :literal:`XHTML 1.0` file path.
        :type report_header_path: str | None
        :param report_footer_path: A report footer :literal:`XHTML 1.0` file path.
        :type report_footer_path: str | None
        :param e_mail: An e-mail address for a :emphasis:`UCSC Genome Browser Track Hub`.
        :type e_mail: str | None
        :param stage_list: A Python :py:class:`list` object of :py:class:`bsf.analysis.Stage` objects.
        :type stage_list: list[Stage] | None
        :param collection: A :py:class:`bsf.ngs.Collection` object.
        :type collection: Collection | None
        :param sample_list: A Python :py:class:`list` object of :py:class:`bsf.ngs.Sample` objects.
        :type sample_list: list[Sample] | None
        :param adapter_path: An adapter definition file path.
        :type adapter_path: str | None
        :param trimming_step_pe_list: A colon-separated list of Trimmomatic steps for paired-end data.
        :type trimming_step_pe_list: list[str] | None
        :param trimming_step_se_list: A colon-separated list of Trimmomatic steps for single-end data.
        :type trimming_step_se_list: list[str] | None
        :param java_archive_trimmomatic: A Trimmomatic tool :emphasis:`Java Archive` (JAR) file path.
        :type java_archive_trimmomatic: str | None
        """
        super(Trimmomatic, self).__init__(
            configuration=configuration,
            project_name=project_name,
            genome_version=genome_version,
            input_directory=input_directory,
            output_directory=output_directory,
            project_directory=project_directory,
            genome_directory=genome_directory,
            report_style_path=report_style_path,
            report_header_path=report_header_path,
            report_footer_path=report_footer_path,
            e_mail=e_mail,
            stage_list=stage_list,
            collection=collection,
            sample_list=sample_list)

        self.adapter_path = adapter_path
        self.trimming_step_pe_list = trimming_step_pe_list
        self.trimming_step_se_list = trimming_step_se_list
        self.java_archive_trimmomatic = java_archive_trimmomatic

        return

    def set_configuration(self, configuration: Configuration, section: str) -> None:
        """Set instance variables of a :py:class:`bsf.analyses.trimmomatic.Trimmomatic` object
        via a section of a :py:class:`bsf.standards.Configuration` object.

        Instance variables without a configuration option remain unchanged.

        :param configuration: A :py:class:`bsf.standards.Configuration` object.
        :type configuration: Configuration
        :param section: A configuration file section.
        :type section: str
        """
        super(Trimmomatic, self).set_configuration(configuration=configuration, section=section)

        # Sub-class specific ...

        option = 'adapter_path'
        if configuration.config_parser.has_option(section=section, option=option):
            self.adapter_path = configuration.config_parser.get(section=section, option=option)

        # Get the list of default trimming steps for paired-end data mode.

        option = 'trimming_steps_pe'
        if configuration.config_parser.has_option(section=section, option=option):
            self.trimming_step_pe_list = configuration.get_list_from_csv(section=section, option=option)

        # Get the list of default trimming steps for single-end data mode.

        option = 'trimming_steps_se'
        if configuration.config_parser.has_option(section=section, option=option):
            self.trimming_step_se_list = configuration.get_list_from_csv(section=section, option=option)

        # Get the Trimmomatic tool Java Archive (JAR) file path.

        option = 'java_archive_trimmomatic'
        if configuration.config_parser.has_option(section=section, option=option):
            self.java_archive_trimmomatic = configuration.config_parser.get(section=section, option=option)

        return

    def run(self) -> None:
        """Run a :py:class:`bsf.analyses.trimmomatic.Trimmomatic` object.

        This method changes the :py:class:`bsf.analyses.trimmomatic.Trimmomatic.collection` attribute
        (:py:class:`bsf.ngs.Collection`) to update with FASTQ file paths.
        """

        def run_adjust_illumina_clip_path(trimming_step_list: list[str]) -> None:
            """Private function to adjust the adapter :emphasis:`FASTA` file path of
            :literal:`ILLUMINACLIP` trimming steps.

            If the file path is not absolute, prepend the value of the adapter_path
            instance variable.

            :param trimming_step_list: A Python :py:class:`list` object of
                Python :py:class:`str` (trimming step) objects.
            :type trimming_step_list: list[str]
            """
            for i in range(0, len(trimming_step_list)):
                if trimming_step_list[i].startswith('ILLUMINACLIP'):
                    component_list = trimming_step_list[i].split(':')
                    if not os.path.isabs(component_list[1]):
                        component_list[1] = os.path.join(self.adapter_path, component_list[1])
                    trimming_step_list[i] = ':'.join(component_list)
            return

        def run_read_comparisons() -> None:
            """Private function to read a :py:class:`bsf.annotation.AnnotationSheet` specifying comparisons
            from a CSV file path.

            This implementation just adds all :py:class:`bsf.ngs.Sample` objects from the
            :py:attr:`bsf.analysis.Analysis.collection` instance variable
            (i.e., :py:class:`bsf.ngs.Collection` object) to the
            :py:attr:`bsf.analysis.Analysis.sample_list` instance variable.
            """

            self.sample_list.extend(self.collection.get_all_samples())

            return

        # Start of the run() method body.

        super(Trimmomatic, self).run()

        # Get the Trimmomatic tool Java Archive (JAR) file path.

        if not self.java_archive_trimmomatic:
            self.java_archive_trimmomatic = JavaArchive.get_trimmomatic()
            if not self.java_archive_trimmomatic:
                raise Exception(f"A {self.name!s} requires a 'java_archive_trimmomatic' configuration option.")

        if not (self.adapter_path and os.path.isabs(self.adapter_path)):
            self.adapter_path = os.path.join(os.path.dirname(self.java_archive_trimmomatic), 'adapters')

        if self.trimming_step_pe_list is None:
            raise Exception(f"A {self.name!s} requires a 'trimming_steps_pe' configuration option.")

        if self.trimming_step_se_list is None:
            raise Exception(f"A {self.name!s} requires a 'trimming_steps_se' configuration option.")

        run_adjust_illumina_clip_path(trimming_step_list=self.trimming_step_pe_list)
        run_adjust_illumina_clip_path(trimming_step_list=self.trimming_step_se_list)

        run_read_comparisons()

        # Trimmomatic

        stage_read_group = self.get_stage(name=self.get_stage_name_read_group())
        stage_summary = self.get_stage(name=self.get_stage_name_summary())
        stage_project = self.get_stage(name=self.get_stage_name_project())

        project_dependency_list: list[str] = list()

        # Sort the Python list of Sample objects by Sample.name.

        self.sample_list.sort(key=lambda item: item.name)

        for sample in self.sample_list:
            module_logger.debug('Sample name: %r', sample.name)
            module_logger.log(logging.DEBUG - 2, 'Sample: %r', sample)

            sample_step_list: list[str] = list()
            if 'Trimmomatic Steps' in sample.annotation_dict:
                for trimming_step in sample.annotation_dict['Trimmomatic Steps']:
                    sample_step_list.extend(self.configuration.list_from_csv(csv_string=trimming_step))
                run_adjust_illumina_clip_path(trimming_step_list=sample_step_list)

            # The Trimmomatic analysis does not obey excluded PairedReads objects,
            # more high-level analyses generally do.
            paired_reads_dict = sample.get_all_paired_reads(replicate_grouping=False, exclude=False)

            for paired_reads_name in sorted(paired_reads_dict):
                for paired_reads in paired_reads_dict[paired_reads_name]:
                    module_logger.debug('PairedReads name: %r', paired_reads.get_name())
                    module_logger.log(logging.DEBUG - 2, 'PairedReads: %r', paired_reads)

                    paired_reads_step_list = list()
                    if 'Trimmomatic Steps' in paired_reads.annotation_dict:
                        for trimming_step in paired_reads.annotation_dict['Trimmomatic Steps']:
                            paired_reads_step_list.extend(self.configuration.list_from_csv(csv_string=trimming_step))
                        run_adjust_illumina_clip_path(trimming_step_list=paired_reads_step_list)

                    # Apply some sanity checks.

                    # Maybe this case should be allowed after Trimmomatic trimming,
                    # where only the second Read survives.
                    if paired_reads.reads_1 is None and paired_reads.reads_2 is not None:
                        raise Exception(f"The PairedReads object {paired_reads.get_name()!r} has a 'reads_2', "
                                        f"but no reads_1 object.")

                    file_path_read_group = self.get_file_path_read_group(read_group_name=paired_reads_name)

                    # Create a Runnable and an Executable for running the Trimmomatic analysis.

                    runnable_read_group = self.add_runnable(
                        runnable=ConsecutiveRunnable(
                            name=self.get_prefix_read_group(read_group_name=paired_reads_name),
                            working_directory=self.project_directory))
                    self.set_stage_runnable(stage=stage_read_group, runnable=runnable_read_group)

                    # Record the Executable.name for the project dependency.

                    project_dependency_list.append(runnable_read_group.name)

                    # Create a new RunnableStepMakeDirectory in preparation of the Trimmomatic program.

                    runnable_step_read_group = RunnableStepMakeDirectory(
                        name='mkdir',
                        directory_path=file_path_read_group.output_directory)
                    runnable_read_group.add_runnable_step(runnable_step=runnable_step_read_group)

                    # Create a RunnableStep for the Trimmomatic program.

                    runnable_step_read_group = RunnableStepJava(
                        name='trimmomatic',
                        java_temporary_path=runnable_read_group.temporary_directory_path(absolute=False),
                        java_heap_maximum='Xmx4G',
                        java_jar_path=self.java_archive_trimmomatic)
                    runnable_read_group.add_runnable_step(runnable_step=runnable_step_read_group)

                    if paired_reads.reads_2 is None:
                        runnable_step_read_group.sub_command.sub_command = Command(program='SE')
                    else:
                        runnable_step_read_group.sub_command.sub_command = Command(program='PE')

                    # Add options to the sub command.
                    sub_command = runnable_step_read_group.sub_command.sub_command
                    sub_command.add_option_short(key='trimlog', value=file_path_read_group.trim_log_tsv)

                    if paired_reads.reads_2 is None:
                        file_path_read_group.reads_1u = os.path.join(
                            file_path_read_group.output_directory,
                            paired_reads.reads_1.name + 'U.fastq.gz')

                        sub_command.arguments.append(paired_reads.reads_1.file_path)
                        sub_command.arguments.append(file_path_read_group.reads_1u)

                        # Update unpaired Reads information.

                        paired_reads.reads_1.name += 'U'
                        paired_reads.reads_1.file_path = os.path.join(
                            self.genome_directory,
                            file_path_read_group.reads_1u)
                    else:
                        file_path_read_group.reads_1p = os.path.join(
                            file_path_read_group.output_directory,
                            paired_reads.reads_1.name + 'P.fastq.gz')
                        file_path_read_group.reads_1u = os.path.join(
                            file_path_read_group.output_directory,
                            paired_reads.reads_1.name + 'U.fastq.gz')
                        file_path_read_group.reads_2p = os.path.join(
                            file_path_read_group.output_directory,
                            paired_reads.reads_2.name + 'P.fastq.gz')
                        file_path_read_group.reads_2u = os.path.join(
                            file_path_read_group.output_directory,
                            paired_reads.reads_2.name + 'U.fastq.gz')

                        sub_command.arguments.append(paired_reads.reads_1.file_path)
                        sub_command.arguments.append(paired_reads.reads_2.file_path)
                        sub_command.arguments.append(file_path_read_group.reads_1p)
                        sub_command.arguments.append(file_path_read_group.reads_1u)
                        sub_command.arguments.append(file_path_read_group.reads_2p)
                        sub_command.arguments.append(file_path_read_group.reads_2u)

                        # Update paired Reads information.

                        paired_reads.reads_1.name += 'P'
                        paired_reads.reads_1.file_path = os.path.join(
                            self.genome_directory,
                            file_path_read_group.reads_1p)
                        paired_reads.reads_2.name += 'P'
                        paired_reads.reads_2.file_path = os.path.join(
                            self.genome_directory,
                            file_path_read_group.reads_2p)

                        # Add unpaired Reads 1 and 2 as separate PairedReads objects to this sample.

                        sample.add_paired_reads(
                            paired_reads=PairedReads(
                                annotation_dict=paired_reads.annotation_dict,
                                reads_1=Reads(
                                    name=paired_reads.reads_1.name[:-1] + 'U',
                                    file_path=os.path.join(
                                        self.genome_directory,
                                        file_path_read_group.reads_1u)),
                                exclude=paired_reads.exclude,
                                index_1=paired_reads.index_1,
                                index_2=paired_reads.index_2,
                                read_group=paired_reads.read_group))

                        sample.add_paired_reads(
                            paired_reads=PairedReads(
                                annotation_dict=paired_reads.annotation_dict,
                                reads_1=Reads(
                                    name=paired_reads.reads_2.name[:-1] + 'U',
                                    file_path=os.path.join(
                                        self.genome_directory,
                                        file_path_read_group.reads_2u)),
                                exclude=paired_reads.exclude,
                                index_1=paired_reads.index_1,
                                index_2=paired_reads.index_2,
                                read_group=paired_reads.read_group))

                    # Append trimming steps in order of specificity read group, sample and default.

                    if len(paired_reads_step_list):
                        sub_command.arguments.extend(paired_reads_step_list)
                    elif len(sample_step_list):
                        sub_command.arguments.extend(sample_step_list)
                    else:
                        if paired_reads.reads_2 is None:
                            sub_command.arguments.extend(self.trimming_step_se_list)
                        else:
                            sub_command.arguments.extend(self.trimming_step_pe_list)

                    # Create a new RunnableStep to compress the trim log file.

                    runnable_step_read_group = RunnableStep(
                        name='compress_logs',
                        program='pigz')
                    runnable_read_group.add_runnable_step(runnable_step=runnable_step_read_group)

                    runnable_step_read_group.add_switch_long(key='best')
                    runnable_step_read_group.add_option_long(key='processes', value=str(stage_summary.threads))
                    runnable_step_read_group.arguments.append(file_path_read_group.trim_log_tsv)

                    # Create a Runnable for the bsf_trimmomatic_summary.R analysis.

                    prefix_summary = '_'.join((stage_summary.name, paired_reads_name))

                    runnable_summary = self.add_runnable(
                        runnable=ConsecutiveRunnable(
                            name=prefix_summary,
                            working_directory=self.project_directory))
                    executable_summary = self.set_stage_runnable(stage=stage_summary, runnable=runnable_summary)
                    executable_summary.dependencies.append(runnable_read_group.name)

                    # Create a new RunnableStep to aggregate the trim log file.

                    runnable_step_summary = RunnableStep(
                        name='trimmomatic_summary',
                        program='bsf_trimmomatic_summary.R',
                        obsolete_file_path_list=[
                            # file_path_read_group.trim_log_tsv_gz,
                        ])
                    runnable_summary.add_runnable_step(runnable_step=runnable_step_summary)

                    runnable_step_summary.add_option_long(
                        key='file-path',
                        value=file_path_read_group.trim_log_tsv_gz)

        # Create a Runnable for pruning the sample annotation sheet.

        prefix_project = '_'.join((stage_project.name, self.project_name))

        file_path_project = self.get_file_path_project(project_name=self.project_name, prefix_analysis=self.prefix)

        # Convert the (modified) Collection object into a SampleAnnotationSheet object and write it to disk.

        annotation_sheet = self.collection.to_sas(
            file_path=os.path.join(self.project_directory, file_path_project.sas_path_old),
            name=prefix_project)
        annotation_sheet.to_file_path()

        runnable_project = self.add_runnable(
            runnable=ConsecutiveRunnable(
                name=prefix_project,
                working_directory=self.project_directory))
        executable_project = self.set_stage_runnable(
            stage=stage_project,
            runnable=runnable_project)
        executable_project.dependencies.extend(project_dependency_list)

        # Create a new RunnableStep.

        runnable_step_project = RunnableStepCollectionPruneFastq(
            name='prune_sample_annotation_sheet',
            obsolete_file_path_list=[
                # file_path_project.sas_path_old,
            ],
            file_path_old=file_path_project.sas_path_old,
            file_path_new=file_path_project.sas_path_new,
            minimum_size=1024)
        runnable_project.add_runnable_step(runnable_step=runnable_step_project)

        return

    def report(self) -> None:
        """Create a :literal:`XHTML 1.0` report and a :emphasis:`UCSC Genome Browser Track Hub`.
        """

        # Create a symbolic link containing the project name and a UUID.
        self.create_public_project_link()

        # Write a HTML document.

        report_list: list[str] = list()

        report_list.append('<h1 id="' + self.prefix + '_analysis">' + self.project_name + ' ' + self.name + '</h1>\n')
        report_list.append('\n')

        report_list.append('<h2 id="aliquot_and_sample_level">Aliquot and Sample Level</h2>\n')
        report_list.append('\n')
        report_list.append('<table id="aliquot_and_sample_table">\n')
        report_list.append('<thead>\n')
        report_list.append('<tr>\n')
        report_list.append('<th>Sample</th>\n')
        report_list.append('<th>Aliquot</th>\n')
        report_list.append('<th>Coverage</th>\n')
        report_list.append('<th>Frequency</th>\n')
        report_list.append('<th>Summary</th>\n')
        report_list.append('</tr>\n')
        report_list.append('</thead>\n')
        report_list.append('<tbody>\n')

        for sample in self.sample_list:
            # The Trimmomatic analysis does not obey excluded PairedReads objects,
            # more high-level analyses generally do.
            paired_reads_dict = sample.get_all_paired_reads(replicate_grouping=False, exclude=False)

            if not paired_reads_dict:
                # Skip Sample objects, which PairedReads objects have all been excluded.
                continue

            report_list.append('<tr>\n')
            report_list.append('<td class="left">' + sample.name + '</td>\n')
            report_list.append('<td class="left"></td>\n')  # Aliquot
            report_list.append('<td class="center"></td>\n')  # Coverage PNG
            report_list.append('<td class="center"></td>\n')  # Frequency PNG
            report_list.append('<td class="center"></td>\n')  # Summary TSV
            report_list.append('</tr>\n')

            # This analysis is special in that read group names carry 'P' or 'U' suffices and samples carry additional
            # read groups after trimming that do no longer correspond to the initial Runnable objects. Sigh.
            # Transiently create a Python dict without the suffix and sort its keys (bsf.ngs.PairedReads.name).

            for paired_reads_name in sorted(dict(map(lambda item: (item[:-1], True), paired_reads_dict.keys()))):
                prefix_read_group = self.get_prefix_read_group(read_group_name=paired_reads_name)

                # The second read may still not be there.
                if prefix_read_group not in self.runnable_dict:
                    continue

                runnable_read_group = self.runnable_dict[prefix_read_group]
                file_path_read_group = self.get_file_path_read_group(read_group_name=paired_reads_name)

                report_list.append('<tr>\n')
                # Sample
                report_list.append('<td class="left"></td>\n')
                # Aliquot
                report_list.append('<td class="left">' + paired_reads_name + '</td>\n')
                # Coverage
                report_list.append('<td class="center">')
                report_list.append('<a href="' + file_path_read_group.coverage_png + '">')
                report_list.append('<img alt="Coverage ' + runnable_read_group.name + '"')
                report_list.append(' src="' + file_path_read_group.coverage_png + '"')
                report_list.append(' height="100" width="100" />')
                report_list.append('</a>')
                report_list.append('</td>\n')
                # Frequency
                report_list.append('<td class="center">')
                report_list.append('<a href="' + file_path_read_group.frequency_png + '">PNG</a>')
                report_list.append('</td>\n')
                # The frequency plots provide little information that does not necessarily justify
                # adding another set of images onto the HTML report.
                report_list.append('<td class="center">')
                report_list.append('<a href="' + file_path_read_group.summary_tsv + '">TSV</a>')
                report_list.append('</td>\n')
                report_list.append('</tr>\n')

        report_list.append('</tbody>\n')
        report_list.append('</table>\n')
        report_list.append('\n')

        self.report_to_file(content=report_list)

        return

    @classmethod
    def console_submit(
            cls,
            configuration_path: str,
            stage_name: Optional[str] = None,
            drms_submit: Optional[bool] = None,
            project_name: Optional[str] = None,
            sas_path: Optional[str] = None,
            *args, **kwargs) -> int:
        """Console function to submit a :py:class:`bsf.analyses.trimmomatic.Trimmomatic` analysis.

        This analysis requires either a :literal:`configuration` argument or
        :literal:`project_name` and :literal:`sas_path` arguments.

        :param configuration_path: A configuration `INI <https://en.wikipedia.org/wiki/INI_file>`_ file path.
        :type configuration_path: str
        :param stage_name: A :py:class:`bsf.analysis.Stage` name.
        :type stage_name: str | None
        :param drms_submit: Request submitting into the DRMS.
        :type drms_submit: bool | None
        :param project_name: A project name.
        :type project_name: str
        :param sas_path: A sample annotation sheet file path.
        :type sas_path: str
        :return: A :py:class:`SystemExit` status value.
        :rtype: int
        """
        if configuration_path == Configuration.get_global_file_path():
            if not project_name:
                raise Exception('The --project-name argument is required if configuration is not set.')
            if not sas_path:
                raise Exception('The --sas-path argument is required if configuration is not set.')

        analysis: Trimmomatic = cls.from_config_file_path(config_path=configuration_path)

        if project_name:
            if project_name.endswith('.ini'):
                raise Exception('The --project-name option should not be a configuration (INI) file.')

            analysis.project_name = project_name

        if sas_path:
            if not sas_path.endswith('.csv'):
                raise Exception('The --sas-path option requires a comma-separated value (CSV) file.')

            analysis.sas_file = sas_path

        analysis.run()
        analysis.check_state()
        analysis.submit(name=stage_name, drms_submit=drms_submit)

        print(analysis.name)
        print('Project name:      ', analysis.project_name)
        print('Input directory:   ', analysis.input_directory)
        print('Project directory: ', analysis.project_directory)
        print('Genome directory:  ', analysis.genome_directory)
        if stage_name == 'report':
            print('Report URL:        ', analysis.get_html_report_url())

        return 0

    @classmethod
    def entry_point_submit(cls) -> int:
        """Console entry point to submit a :py:class:`bsf.analyses.trimmomatic.Trimmomatic` analysis.

        This analysis requires either a positional :literal:`configuration` argument or
        :literal:`--project-name` and :literal:`--sas-path` arguments.

        :return: A :py:class:`SystemExit` status value.
        :rtype: int
        """
        argument_parser = ArgumentParser(
            description=cls.name + ' submission script.')

        argument_parser.add_argument(
            '--dry-run',
            action='store_false',
            help='dry run',
            dest='drms_submit')

        argument_parser.add_argument(
            '--logging-level',
            default='WARNING',
            choices=('CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG', 'DEBUG1', 'DEBUG2'),
            help='logging level [WARNING]')

        argument_parser.add_argument(
            '--stage-name',
            help='limit job submission to a particular analysis stage')

        argument_parser.add_argument(
            '--project-name',
            help='project name')

        argument_parser.add_argument(
            '--sas-path',
            help='sample annotation sheet (CSV) file path')

        argument_parser.add_argument(
            'configuration',
            nargs='?',
            default=Configuration.get_global_file_path(),
            help=f'configuration (INI) file path [{Configuration.get_global_file_path()!s}]')

        name_space = argument_parser.parse_args()

        if name_space.logging_level:
            logging.addLevelName(level=logging.DEBUG - 1, levelName='DEBUG1')
            logging.addLevelName(level=logging.DEBUG - 2, levelName='DEBUG2')

            logging.basicConfig(level=name_space.logging_level)

        return cls.console_submit(
            configuration_path=name_space.configuration,
            stage_name=name_space.stage_name,
            drms_submit=name_space.drms_submit,
            project_name=name_space.project_name,
            sas_path=name_space.sas_path)


import numpy as np
import os

from ReCal.TransformedRunner import TransformedRunner

class LogitsFileExport:
  def __init__(self, loader, loader_str, transformation_specs, n_trans, normalization_params, model, device, transformations):
    self.loader = loader
    self.loader_str = loader_str

    self.transformed_runner = TransformedRunner(self.loader, transformation_specs, n_trans, normalization_params, model, device, transformations)

  def export(self, output_dir):
    if not os.path.exists(output_dir):
      os.makedirs(output_dir)

    fn_base = "{}_{}_{}_{}.npy"  # [F_TYPE]_[F_ARG]_[VAL_TEST]_[TYPE].npy

    for idx, (tran_type, tran_arg) in enumerate([('identity', 1.0)] + self.transformed_runner.transformations):
      print(idx + 1, tran_type, tran_arg)

      logits_all, y_all = self.transformed_runner.run_on_tran(tran_type, tran_arg)

      logits_fn = os.path.join(output_dir, fn_base.format(tran_type,
                                                          tran_arg,
                                                          self.loader_str,
                                                          "logits"))
      np.save(logits_fn, logits_all.numpy())
      print("\tstored {}.".format(logits_fn))

      if tran_type == 'identity':
        y_fn = os.path.join(output_dir, fn_base.format(tran_type,
                                                       tran_arg,
                                                       self.loader_str,
                                                       "ys"))
        np.save(y_fn, y_all.numpy())
        print("\tstored {}.".format(y_fn))


"""Module for building a logger.

The custom logger allows to add extra info when available,
when you run for particular use cases to be easier to debug.
"""

import coloredlogs
import functools
import logging
import numpy as np
import psutil
import time
from typing import Any, Callable, Optional, Tuple, Union

# our other modules
import configs.settings as settings

class RequestLogger(logging.LoggerAdapter):
    """Custom logging adapter to add name when available."""

    request_id_ = None
    option_name_ = None

    def set_kwargs(self, request_id: str, option_name: str) -> None:
        """Set request_id and name for all logs for this request."""
        self.request_id_ = request_id
        self.option_name_ = option_name

    @property
    def request_id(cls) -> Optional[str]:
        """Get current request ID."""
        return cls.request_id_

    @property
    def option_name(cls) -> Optional[str]:
        """Get current city name."""
        return cls.option_name_

    def process(self, msg: Any, kwargs: Any) -> Tuple[str, Any]:
        """Add request_id, name and code context to logs."""
        if isinstance(msg, dict):
            pass
        # check if we pass either str or dict; but not convert str message to dict
        elif isinstance(msg, str):
            # msg = {
            #     "message": msg,
            # }
            pass
        else:
            raise TypeError("Logging message must be either str or dict.")

        # add request_id and asset_name to every log message
        if self.request_id is not None:
            msg["request_id"] = self.request_id
        if self.option_name is not None:
            msg["option_name"] = self.option_name

        # parse message to have city_name
        # at the beginning of every log message
        msg = "%s" % (msg)
        return msg, kwargs

logger = logging.getLogger(__name__)
coloredlogs.install(
    level=settings.LOGGING_LEVEL,
    logger=logger,
    fmt="%(asctime)s %(name)s [%(process)d] %(levelname)s %(message)s",
)
request_logger = RequestLogger(logger, dict())

def time_and_log(
    add_func_args: Union[str, list] = [],
    level: str = settings.LOGGING_LEVEL,
) -> Callable:
    """Decorate any function to time and log start and end.

    Including also the RAM memory usage.
    """

    def _get_args_logged(kwargs: Any, add_func_args: Union[str, list]) -> dict:
        if add_func_args == "all":
            args_logged = kwargs
        elif add_func_args == []:
            args_logged = "removed"
        else:
            args_logged = {kw: val for kw, val in kwargs.items() if kw in add_func_args}
        return args_logged

    def decorator(func: Callable) -> Callable:
        @functools.wraps(func)
        def wrapper(*args: Any, **kwargs: Any) -> None:
            logging_method = getattr(request_logger, level.lower())

            try:
                # process started
                ram_usage_before = psutil.virtual_memory().percent
                logging_method(
                    {
                        "process": func.__qualname__,
                        "message": "Started.",
                        "current_ram_usage": ram_usage_before,
                        "args": _get_args_logged(kwargs, add_func_args),
                    }
                )
                start_time = time.time()
                result = func(*args, **kwargs)

                # process finished
                ram_usage_after = psutil.virtual_memory().percent
                logging_method(
                    {
                        "process": func.__qualname__,
                        "message": "Success.",
                        "elapsed_seconds": np.round(time.time() - start_time, 4),
                        "current_ram_usage": ram_usage_after,
                    }
                )
                return result

            # process error
            except Exception as e:
                request_logger.exception(
                    {"process": func.__qualname__, "error_msg": str(e)}
                )

                raise e

        return wrapper

    return decorator


from predictionserver.futureconventions.activityconventions import (
    Activity, ActivityContext
)
from predictionserver.futureconventions.memoconventions import Memo
from predictionserver.serverhabits.sortedsethabits import (
    SortedSetType, SortedSetNameGranularity, SortedSetKeyGranularity
)
from predictionserver.servermixins.ownershipserver import OwnershipServer
from predictionserver.servermixins.memoserver import MemoServer
from collections import OrderedDict
from typing import Union
from copy import deepcopy

class SortedSetServer(MemoServer, OwnershipServer):

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    def _authorize_sortedset_change(
            self,
            sortedset_type: SortedSetType,
            name_granularity: SortedSetNameGranularity,
            **kwargs
    ):
        if name_granularity in [
                SortedSetNameGranularity.name,
                SortedSetNameGranularity.name_and_delay]:
            return self._authorize(name=kwargs['name'], write_key=kwargs['write_key'])
        elif name_granularity == SortedSetNameGranularity.write_key:
            return self.is_valid_key(write_key=kwargs['write_key'])
        elif name_granularity in [
            SortedSetNameGranularity.code, SortedSetNameGranularity.code_and_genus
        ]:
            return self.shash(write_key=kwargs['write_key']) == kwargs['code']
        elif name_granularity in [SortedSetNameGranularity.memory]:
            return False  # System only
        else:
            return True

    def get_sortedset_values(
            self,
            sortedset_type: SortedSetType,
            name_granularity: SortedSetNameGranularity,
            **name_kwargs
    ):
        """ Retrieve entire sortedset_type """
        location = self.sortedset_location(
            sortedset_type=sortedset_type,
            name_granularity=name_granularity,
            **name_kwargs
        )
        raw = list(reversed(
            self.client.zrange(
                name=location,
                start=0,
                end=-1,
                withscores=True)
        ))
        return OrderedDict([(k, float(v) if v is not None else 0.0) for k, v in raw])

    def add_sortedset_value(
            self, sortedset_type: SortedSetType,
            name_granularity: SortedSetNameGranularity,
            key_granularity: SortedSetKeyGranularity,
            name_kwargs: dict,
            key_kwargs: dict,
            value: float,
            write_key=None,
            verbose=False
    ):
        return self._add_sortedset_value_implementation(
            sortedset_type=sortedset_type,
            name_granularity=name_granularity,
            key_granularity=key_granularity,
            name_kwargs=name_kwargs,
            write_key=write_key,
            key_kwargs=key_kwargs,
            value=value,
            verbose=verbose
        )

    def multiply_sortedset(
            self,
            sortedset_type: SortedSetType,
            name_granularity: SortedSetNameGranularity,
            name_kwargs: dict,
            weight: float,
            write_key=None,
            verbose=False
    ):
        return self._multiply_sortedset_implementation(
            sortedset_type=sortedset_type,
            name_granularity=name_granularity,
            name_kwargs=name_kwargs,
            weight=weight,
            write_key=write_key,
            verbose=verbose
        )

    def delete_sortedset(
            self,
            sortedset_type: SortedSetType,
            name_granularity: SortedSetNameGranularity,
            name_kwargs: dict,
            write_key: str,
            verbose=False
    ):
        """ If permission, confirmed delete of sortedset_type by user """
        return self._delete_sortedset_implementation(
            sortedset_type=sortedset_type,
            name_granularity=name_granularity,
            name_kwargs=name_kwargs,
            write_key=write_key,
            verbose=verbose
        )

    # ------------------ #
    #   Implementation   #
    # ------------------ #

    def _delete_sortedset_implementation(
            self,
            sortedset_type: SortedSetType,
            name_granularity: SortedSetNameGranularity,
            name_kwargs: dict,
            verbose: Union[bool, int],
            write_key=None
    ):
        """ If permission, confirmed delete of sortedset_type by user """
        write_key = write_key or name_kwargs.get('write_key')
        allowed = self._authorize_sortedset_change(
            sortedset_type=sortedset_type,
            name_granularity=name_granularity,
            write_key=write_key,
            code=name_kwargs.get('code'),
            name=name_kwargs.get('name')
        )
        location = self.sortedset_location(
            sortedset_type=sortedset_type,
            name_granularity=name_granularity,
            **name_kwargs
        )
        memo_kwargs = deepcopy(name_kwargs)
        memo_kwargs.update({'write_key': write_key})
        memo = Memo(
            activity=Activity.delete,
            sortedset_type=sortedset_type,
            allowed=int(allowed),
            success=int(allowed),
            **memo_kwargs
        )
        if allowed:
            execution = self.client.delete(location)
            assert self.add_memo_as_owner_confirm(
                memo=memo, success=1, execution=execution
            )
        else:
            assert self.add_memo_as_owner_error(memo=memo, success=0, execution=0)
        return allowed if not verbose else memo.as_dict()

    def _add_sortedset_value_implementation(
            self,
            sortedset_type: SortedSetType,
            name_granularity: SortedSetNameGranularity,
            key_granularity: SortedSetKeyGranularity,
            name_kwargs: dict,
            key_kwargs: dict,
            verbose: Union[bool, int],
            value: float,
            write_key=None,
            context: ActivityContext = None
    ):
        """ If permission, confirmed setting of sortedset_type value by user """
        write_key = write_key or name_kwargs.get('write_key') or key_kwargs.get(
            'write_key')
        allowed = self._authorize_sortedset_change(
            sortedset_type=sortedset_type,
            name_granularity=name_granularity,
            write_key=write_key,
            code=name_kwargs.get('code'),
            name=name_kwargs.get('name')
        )
        location = self.sortedset_location(
            sortedset_type=sortedset_type,
            name_granularity=name_granularity,
            **name_kwargs
        )

        key = self.sortedset_key(key_granularity=key_granularity, **key_kwargs)
        memo_kwargs = deepcopy(name_kwargs)
        memo_kwargs.update({'write_key': write_key})
        memo = Memo(
            activity=Activity.add,
            sortedset_type=sortedset_type,
            context=context,
            allowed=int(allowed),
            success=int(allowed),
            **memo_kwargs
        )
        if allowed:
            mapping = {key: value}
            execution = self.client.zadd(name=location, mapping=mapping)
            assert self.add_memo_as_owner_confirm(
                memo=memo, success=1, execution=execution
            )
        else:
            assert self.add_memo_as_owner_error(memo=memo, success=0, execution=0)
        return allowed if not verbose else memo.as_dict()

    def _multiply_sortedset_implementation(
            self,
            sortedset_type: SortedSetType,
            name_granularity: SortedSetNameGranularity,
            name_kwargs: dict,
            weight: float,
            write_key=None,
            verbose=False
    ):
        """ If permission, multiply all values in sortedset_type by float multiplier """
        write_key = write_key or name_kwargs.get('write_key')
        allowed = self._authorize_sortedset_change(
            sortedset_type=sortedset_type,
            name_granularity=name_granularity,
            write_key=write_key,
            code=name_kwargs.get('code'),
            name=name_kwargs.get('name')
        )
        location = self.sortedset_location(
            sortedset_type=sortedset_type,
            name_granularity=name_granularity,
            **name_kwargs
        )
        memo_kwargs = deepcopy(name_kwargs)
        memo_kwargs.update({'write_key': write_key})
        memo = Memo(
            activity=Activity.multiply,
            sortedset_type=sortedset_type,
            allowed=int(allowed),
            success=int(allowed),
            value=weight,
            **memo_kwargs
        )
        if allowed:
            import random
            temporary_key = 'temporary_' + \
                ''.join([random.choice(['a', 'b', 'c']) for _ in range(20)])
            shrink_pipe = self.client.pipeline(transaction=True)
            shrink_pipe.zunionstore(dest=temporary_key, keys={location: weight})
            shrink_pipe.zunionstore(dest=location, keys={temporary_key: 1})
            shrink_pipe.expire(name=temporary_key, time=1)
            _ = shrink_pipe.execute()
            assert self.add_memo_as_owner_confirm(memo=memo, success=1, execution=1)
        else:
            assert self.add_memo_as_owner_error(memo=memo, success=0, execution=0)
        return allowed if not verbose else memo.as_dict()

    def _pipe_incr_sortedset_value_implementation(
            self,
            pipe,
            sortedset_type: SortedSetType,
            name_granularity: SortedSetNameGranularity,
            key_granularity: SortedSetKeyGranularity,
            name_kwargs: dict,
            key_kwargs: dict,
            amount=float
    ):
        """ System use only """
        location = self.sortedset_location(
            sortedset_type=sortedset_type,
            name_granularity=name_granularity,
            **name_kwargs
        )
        key = self.sortedset_key(key_granularity=key_granularity, **key_kwargs)
        pipe = pipe.zincrby(name=location, value=key, amount=amount)
        return pipe


# Import required modules
import cv2 as cv
import math
import time
import argparse

MODEL_MEAN_VALUES = (78.4263377603, 87.7689143744, 114.895847746)
ageList = ['(0-2)', '(4-6)', '(8-12)', '(15-20)', '(25-32)', '(38-43)', '(48-53)', '(60-100)']
genderList = ['Male', 'Female']

genderNet = None
ageNet = None

def init_gender_net(model_path, proto_path):
	global genderNet
	if (genderNet is None):
		genderNet = cv.dnn.readNet(model_path, proto_path)

def init_age_net(model_path, proto_path):
	global ageNet
	if (ageNet is None):
		ageNet = cv.dnn.readNet(model_path, proto_path)

def get_gender(face, model_path, proto_path):
	init_gender_net(model_path, proto_path)
	blob = cv.dnn.blobFromImage(face, 1.0, (227, 227), MODEL_MEAN_VALUES, swapRB=False)
	genderNet.setInput(blob)
	genderPreds = genderNet.forward()
	gender = genderList[genderPreds[0].argmax()]

	return {"gender": gender, "conf": genderPreds[0].max()}
	#print("Gender : {}, conf = {:.3f}".format(gender, genderPreds[0].max()))

def get_age(face, model_path, proto_path):
	init_age_net(model_path, proto_path)
	blob = cv.dnn.blobFromImage(face, 1.0, (227, 227), MODEL_MEAN_VALUES, swapRB=False)
	ageNet.setInput(blob)
	agePreds = ageNet.forward()
	age = ageList[agePreds[0].argmax()]
	print("Age : {}, conf = {:.3f}".format(age, agePreds[0].max()))


numeroUm = int(input("Digite o primeiro número: "))
numeroDois = int(input("Digite o segundo número: "))
numeroTres = int(input("Digite o terceiro número: "))

if numeroUm > numeroDois and numeroUm > numeroTres:
    print ("O maior número é: " + str(numeroUm))

if numeroDois > numeroUm and numeroDois > numeroTres:
    print ("O maior número é: " + str(numeroDois))

if numeroTres > numeroUm and numeroTres > numeroDois:
    print ("O maior número é: " + str(numeroTres))


import numpy as np
import scipy
from tqdm import tqdm
import pickle
from sklearn.metrics import confusion_matrix
from keras.preprocessing.image import img_to_array

def get_psf_airy(n, nr):
    """
    Calculates Point Spread Function.

    :param n:
    :param nr:
    :return: Numpy array of Point Spread Function
    """
    xpsf = np.linspace(-1, 1, n)
    xg, yg = np.meshgrid(xpsf, xpsf)
    r = np.sqrt(xg**2+yg**2)*np.pi*nr
    psf = (scipy.special.j1(r)/r)**2
    psf = psf/psf.sum()
    return psf

def image_augmentation(images, image_data_generator, num_of_augumentations,
                       disable=False):
    """
    Form augmented images for input array of images

    :param images: numpy array of Images.
    :param image_data_generator: Keras image generator object.
    :param num_of_augumentations: Number of augmentations of each image.
    :param disable: Bool. Disable/enable tqdm progress bar.
    :return: Numpy array of augmented images.
    """
    images_aug = []
    for image in tqdm(images, disable=disable):
        img_dim = image.shape
        img_array = img_to_array(image)
        img_array = img_array.reshape((1,) + img_array.shape)
        i = 0
        for batch in image_data_generator.flow(img_array, batch_size=1):
            i += 1
            img = batch[0]
            img = img.reshape(img_dim)
            images_aug.append(img)

            if i >= num_of_augumentations:
                break

    images_aug = np.array(images_aug)
    return images_aug

def psnr(img1, img2):
    """
    Calculate Peak Signal to Noise Ratio value.

    :param img1: Float. Array of first image.
    :param img2: Float.Array of second image.
    :return: Float. PSNR value of x and y.
    """
    mse = np.mean((img1 - img2) ** 2)
    return -10 * np.log10(mse)

def append_one_to_shape(x):
    """
    Reshapes input.

    :param x: Array input.
    :return: Reshaped array.
    """
    x_shape = x.shape
    x = x.reshape((len(x), np.prod(x.shape[1:])))
    x = np.reshape(x, (*x_shape, 1))
    return x

def unpickle(file):
    """
    Unpickle and read file.

    :param file: Pickle file to read.
    :return: Data loaded from pickle file.
    """
    with open(file, 'rb') as fo:
        data = pickle.load(fo, encoding='bytes')
    return data

def to_numeric(y):
    """
    Convert numpy array of array of probabilities to numeric array.

    :param y: Numpy array.
    :return: Numpy array of classes.
    """
    return np.array([np.argmax(value) for value in y])

def accuracy_per_class(y_true, y_pred):
    """
    Computes accuracy per class.

    :param y_true: True class.
    :param y_pred: Predicted class.
    :return:
    """
    y_true = to_numeric(y_true)
    y_pred = to_numeric(y_pred)

    cm = confusion_matrix(y_true, y_pred)
    return cm.diagonal() / cm.sum(axis=1)


import argparse
import sys
import numpy as np
import torch
import torch.nn as nn
from torch.autograd import Variable
import torch.utils.data as Data
from torch.nn.utils.rnn import pad_packed_sequence

from torch.nn.utils.rnn import pack_padded_sequence
import preprocess
from model import EncoderRNN, DecoderRNN, VanillaDecoderRNN, BAttnDecoderRNN, LAttnDecoderRNN
import dataloader
from gensim.models import word2vec
from gensim import models

def getDataLoader(data, batch_size=100, num_workers=4):
    dataset = dataloader.TestDataset(data)
    loader = Data.DataLoader(
        dataset=dataset,  # torch TensorDataset format
        batch_size=batch_size,  # mini batch size
        shuffle=False,
        num_workers=num_workers,
        collate_fn=dataloader.test_collate_fn
    )
    return loader

def to_var(x, volatile=False):
    if torch.cuda.is_available():
        x = x.cuda()
    return Variable(x, volatile=volatile)

def getwv():
    i2v = preprocess.load_obj('index2vocab')
    model = models.Word2Vec.load('w2v200.model.bin')
    embedding = np.zeros((len(i2v), model.wv.vector_size))
    j = 0
    for i in range(len(i2v)):
        try:
            embedding[i] = model.wv[i2v[i]]
        except:
            print(i2v[i], 'not in w2v')
            j += 1
            continue
    print(j)
    return embedding

def main(args):
    fout = open(args.output, 'w')
    maxlen = 20
    num_layers = 1
    dim = 512
    quotes_set = set(['[',']','{','}','!','?','。'])
    sentences, v_size = preprocess.label2onehot(args.input)
    data = getDataLoader(sentences)
    v = preprocess.load_obj('vocabindex')
    i2v = preprocess.load_obj('index2vocab')
    print(sentences)
    v_size = len(v)
    print('sentences shape =', sentences.shape)
    we = getwv()
    encoder = EncoderRNN(v_size, dim, we, num_layers=num_layers, bidirectional=False)
    #decoder = VanillaDecoderRNN(dim, v_size, num_layers=num_layers)
    #decoder = BAttnDecoderRNN(dim, v_size, num_layers=num_layers)
    decoder = LAttnDecoderRNN(dim, v_size, we, num_layers=num_layers)
    encoder.load_state_dict(torch.load(args.encoder))
    decoder.load_state_dict(torch.load(args.decoder))
    if torch.cuda.is_available():
        encoder.cuda()
        decoder.cuda()
    print(encoder)
    print(decoder)
    i = 0
    for sentence in sentences:
        print(sentence)
        s = to_var(torch.LongTensor(sentence).view(1, -1))
        encoder_hidden = encoder.initHidden(num_layers, 1)
        encoder_o, encoder_hidden = encoder(s, [s.size(1)], encoder_hidden)
        encoder_o = pad_packed_sequence(encoder_o, batch_first=True)[0]
        decoder_hidden = encoder_hidden[:num_layers]
        inputword = v['<start>']
        flag = True
        sentence = ''
        words = []
        length = 0
        while flag:
            inputword = to_var(torch.LongTensor([inputword]).view(1,-1))
            output, decoder_hidden = decoder(inputword,decoder_hidden, encoder_o)
            maxkey = np.argmax(output[0].data)
            inputword = maxkey
            word = i2v[maxkey.item()]
            length += 1
            if length > maxlen:
                flag = False
            if word == '<end>' or word == '<pad>':
                flag = False
            elif word == '<unk>':
                continue
            else:
                if word == '.' or word == '。':
                    #sentence = sentence[:-1]
                    #sentence += word
                    flag = False
                else:
                    if word in words and word in quotes_set:
                        continue
                    if (len(words) == 0 or word != words[-1]):
                        words.append(word)
        sentence = ' '.join(words)
        print(sentence)
        fout.write(sentence+'\n')
        i += 1

    fout.close()

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--input", type=str, required=True, help='input file')
    parser.add_argument("--encoder", type=str, required=True, help='encoder model')
    parser.add_argument("--decoder", type=str, required=True, help='decoder model')
    parser.add_argument("--output", type=str, required=True, help='output file')
    args = parser.parse_args()
    main(args)


bicycles = ['trekingowy', 'szosowy', 'miejski', 'górski']
print(bicycles[1])
message = f'moim rowerem jest rower {bicycles[1].title()}'
print(message)
motocykle = []
motocykle.append('honda')
motocykle.append('suzuki')
motocykle.append('yamaha')
print(motocykle)
pop_motocykle = motocykle.pop()
print(pop_motocykle)
magicians = ['alicja', 'dawid', 'karolina']
for magician in magicians:
    print(f"{magician.title()}, to była doskonała sztuczka!")
    print(f"Nie mogę się doczekać Twojej kolejnej sztuczki, {magician.title()}.\n")

print("Dziękuję wszystkim. To był naprawdę wspaniały występ!")
numbers = list(range(2, 11, 2))
print(numbers)
squares = []
for value in range(1, 11):
    square = value ** 2
    squares.append(square)
print(squares)


import matplotlib.pyplot as plt
import numpy as np

arr1 = np.loadtxt("../experiment3_cleaned/Pretrained_aug/augVGGpreacc.csv",delimiter = ',')
arr2 = np.loadtxt("../experiment3_cleaned/Pretrained_aug/augRESNETpreacc.csv",delimiter = ',')
arr3 = np.loadtxt("../experiment3_cleaned/Pretrained_aug/augATNNpreacc.csv",delimiter = ',')
arr11 = np.loadtxt("../experiment2_cleaned_v2/Pretrained_aug/augVGGpreacc.csv",delimiter = ',')
arr22 = np.loadtxt("../experiment2_cleaned_v2/Pretrained_aug/augRESNETpreacc.csv",delimiter = ',')
arr33 = np.loadtxt("../experiment2_cleaned_v2/Pretrained_aug/augATNNpreacc.csv",delimiter = ',')

fig, axs = plt.subplots(1,2, figsize = (15,5))

axs[0].plot(arr1[:,0], label = 'VGG16 - original')
axs[0].plot(arr2[:,0], label = 'Resnet18 - original')
axs[0].plot(arr3[:,0], label = 'ViT - original')
axs[0].plot(arr11[:,0], label = 'VGG16 - fisheye')
axs[0].plot(arr22[:,0], label = 'Resnet18 - fisheye')
axs[0].plot(arr33[:,0], label = 'ViT - fisheye')
#axs[0].plot(arr44[:,0], color = 'orange', linestyle = '--', label = 'ViT')
axs[0].set_xlabel("Epochs")
axs[0].set_ylabel("Accuracy (%)")
axs[0].grid()
axs[0].legend()

axs[1].plot(arr1[:,1], label = 'VGG16 - original')
axs[1].plot(arr2[:,1], label = 'Resnet18 - original')
axs[1].plot(arr3[:,1], label = 'ViT - original')
axs[1].plot(arr11[:,1], label = 'VGG16 - fisheye')
axs[1].plot(arr22[:,1], label = 'Resnet18 - fisheye')
axs[1].plot(arr33[:,1], label = 'ViT - fisheye')
axs[1].set_xlabel("Epochs")
axs[1].set_ylabel("Accuracy (%)")
axs[1].grid()
axs[1].legend()
plt.savefig("experiment3_res.png", bbox_inches = "tight")