import numbers
import time
from itertools import product

import numpy as np
import torch

try:
    from tqdm import tqdm
except ImportError:

    def tqdm(x):
        return x

def product1d(inrange):
    for ii in inrange:
        yield ii

def slice_to_start_stop(s, size):
    """For a single dimension with a given size, normalize slice to size.
     Returns slice(None, 0) if slice is invalid."""
    if s.step not in (None, 1):
        raise ValueError('Nontrivial steps are not supported')

    if s.start is None:
        start = 0
    elif -size <= s.start < 0:
        start = size + s.start
    elif s.start < -size or s.start >= size:
        return slice(None, 0)
    else:
        start = s.start

    if s.stop is None or s.stop > size:
        stop = size
    elif s.stop < 0:
        stop = (size + s.stop)
    else:
        stop = s.stop

    if stop < 1:
        return slice(None, 0)

    return slice(start, stop)

def int_to_start_stop(i, size):
    """For a single dimension with a given size, turn an int into slice(start, stop)
    pair."""
    if -size < i < 0:
        start = i + size
    elif i >= size or i < -size:
        raise ValueError('Index ({}) out of range (0-{})'.format(i, size - 1))
    else:
        start = i
    return slice(start, start + 1)

def normalize_slices(slices, shape):
    """ Normalize slices to shape.

    Normalize input, which can be a slice or a tuple of slices / ellipsis to
    be of same length as shape and be in bounds of shape.

    Args:
        slices (int or slice or ellipsis or tuple[int or slice or ellipsis]): slices to be normalized

    Returns:
        tuple[slice]: normalized slices (start and stop are both non-None)
        tuple[int]: which singleton dimensions should be squeezed out
    """
    type_msg = 'Advanced selection inappropriate. ' \
               'Only numbers, slices (`:`), and ellipsis (`...`) are valid indices (or tuples thereof)'

    if isinstance(slices, tuple):
        slices_lst = list(slices)
    elif isinstance(slices, (numbers.Number, slice, type(Ellipsis))):
        slices_lst = [slices]
    else:
        raise TypeError(type_msg)

    ndim = len(shape)
    if len([item for item in slices_lst if item != Ellipsis]) > ndim:
        raise TypeError("Argument sequence too long")
    elif len(slices_lst) < ndim and Ellipsis not in slices_lst:
        slices_lst.append(Ellipsis)

    normalized = []
    found_ellipsis = False
    squeeze = []
    for item in slices_lst:
        d = len(normalized)
        if isinstance(item, slice):
            normalized.append(slice_to_start_stop(item, shape[d]))
        elif isinstance(item, numbers.Number):
            squeeze.append(d)
            normalized.append(int_to_start_stop(int(item), shape[d]))
        elif isinstance(item, type(Ellipsis)):
            if found_ellipsis:
                raise ValueError("Only one ellipsis may be used")
            found_ellipsis = True
            while len(normalized) + (len(slices_lst) - d - 1) < ndim:
                normalized.append(slice(0, shape[len(normalized)]))
        else:
            raise TypeError(type_msg)
    return tuple(normalized), tuple(squeeze)

def blocking(shape, block_shape, roi=None, center_blocks_at_roi=False):
    """ Generator for nd blocking.

    Args:
        shape (tuple): nd shape
        block_shape (tuple): nd block shape
        roi (tuple[slice]): region of interest (default: None)
        center_blocks_at_roi (bool): if given a roi,
            whether to center the blocks being generated
            at the roi's origin (default: False)
    """
    assert len(shape) == len(block_shape), "Invalid number of dimensions."

    if roi is None:
        # compute the ranges for the full shape
        ranges = [range(sha // bsha if sha % bsha == 0 else sha // bsha + 1)
                  for sha, bsha in zip(shape, block_shape)]
        min_coords = [0] * len(shape)
        max_coords = shape
    else:
        # make sure that the roi is valid
        roi, _ = normalize_slices(roi, shape)
        ranges = [range(rr.start // bsha,
                        rr.stop // bsha if rr.stop % bsha == 0 else rr.stop // bsha + 1)
                  for rr, bsha in zip(roi, block_shape)]
        min_coords = [rr.start for rr in roi]
        max_coords = [rr.stop for rr in roi]

    need_shift = False
    if roi is not None and center_blocks_at_roi:
        shift = [rr.start % bsha for rr, bsha in zip(roi, block_shape)]
        need_shift = sum(shift) > 0

    # product raises memory error for too large ranges,
    # because input iterators are cast to tuple
    # so far I have only seen this for 1d "open-ended" datasets
    # and hence just implemented a workaround for this case,
    # but it should be fairly easy to implement an nd version of product
    # without casting to tuple for our use case using the imglib loop trick, see also
    # https://stackoverflow.com/questions/8695422/why-do-i-get-a-memoryerror-with-itertools-product
    try:
        start_points = product(*ranges)
    except MemoryError:
        assert len(ranges) == 1
        start_points = product1d(ranges)

    for start_point in start_points:
        positions = [sp * bshape for sp, bshape in zip(start_point, block_shape)]
        if need_shift:
            positions = [pos + sh for pos, sh in zip(positions, shift)]
            if any(pos > maxc for pos, maxc in zip(positions, max_coords)):
                continue
        yield tuple(slice(max(pos, minc), min(pos + bsha, maxc))
                    for pos, bsha, minc, maxc in zip(positions, block_shape,
                                                     min_coords, max_coords))

def ensure_5d(tensor):
    if tensor.ndim == 3:
        tensor = tensor[None, None]
    elif tensor.ndim == 4:
        tensor = tensor[None]
    elif tensor.ndim == 5:
        pass
    return tensor

# we don't save any output, because this is just for benchmarking purposes
def run_inference(input_dataset, model,
                  block_shape, halo,
                  preprocess,
                  precision):
    dtype = torch.float32 if precision == 'single' else torch.float16

    device = torch.device('cuda')
    model.to(device, dtype=dtype)
    model.eval()
    shape = input_dataset.shape

    full_block_shape = tuple(bs + 2 * ha for bs, ha in zip(block_shape, halo))
    local_bb = tuple(slice(ha, bsh - ha)
                     for bsh, ha in zip(block_shape, halo))

    def grow_bounding_box(bb):
        grown_bb = tuple(slice(max(b.start - ha, 0), min(sh, b.stop + ha))
                         for b, ha, sh in zip(bb, halo, shape))
        return grown_bb

    def ensure_block_shape(input_):
        if input_.shape != full_block_shape:
            pad_shape = [(0, bsh - sh)
                         for bsh, sh in zip(full_block_shape, input_.shape)]
            input_ = np.pad(input_, pad_shape)
        return input_

    blocks = list(blocking(shape, block_shape))
    per_block_times = []

    t_tot = time.time()
    with torch.no_grad():
        for bb in tqdm(blocks):
            bb = grow_bounding_box(bb)

            input_ = input_dataset[bb]
            input_ = ensure_block_shape(input_)

            input_ = preprocess(input_)
            input_ = ensure_5d(input_)

            t0 = time.time()
            input_ = torch.from_numpy(input_).to(device, dtype=dtype)
            output = model(input_)
            output = output.cpu().to(dtype=torch.float32).numpy()
            per_block_times.append(time.time() - t0)

            # this is where we would save the output ...
            output = output[0]
            output = output[(slice(None),) + local_bb]
    t_tot = time.time() - t_tot

    return t_tot, per_block_times


import pygame, sys, operator, random, time
from pygame.locals import *

# Global variables
WIDTH = 800
HEIGHT = 500
SUB_SPEED = 3
BUBBLE_MAX_SPEED = 1
TIME_LIMIT = 30
BONUS_SCORE = 1500

BLACK = (0, 0, 0)
BLUE = (12,34,56)
RED = (255,0,0)
WHITE = (255,255,255)

x_sub = 40
y_sub = 250
score = 0
game_end = time.time() + TIME_LIMIT
bonus = 0

# bubbles_id = list()
bubbles_pos = list()
bubbles_speed = list()
bubbles_state = list()
bubbles_size = list()

# Quit the game
def leave_game():
    pygame.display.quit()
    pygame.quit()
    sys.exit()

# Update the screen display
def update_screen ():
    screen.blit(background_image, (0,0))
    screen.blit(sub, (x_sub, y_sub))

    for i in range(len(bubbles_pos) - 1, -1, -1):
        if bubbles_state[i] == "Good":
            screen.blit(pygame.transform.scale(blue_bubble, (bubbles_size[i], bubbles_size[i])), bubbles_pos[i])
        else:
            screen.blit(pygame.transform.scale(bad_bubble, (bubbles_size[i], bubbles_size[i])), bubbles_pos[i])

    message = "Score : " +  str(score)
    display_text (message, BLACK, 'Calibri', 20, 10, 15)

    # print ("Time : ", int(game_end - time.time()))

    message = "Time : " +  str(int(game_end - time.time()))
    display_text (message, BLACK, 'Calibri', 20, 700, 15)

    pygame.display.flip()

# Move the submarine on the scene
def sub_control():
    global x_sub, y_sub

    key = pygame.key.get_pressed()

    if key[pygame.K_RIGHT]:
        x_sub += SUB_SPEED

    if key[pygame.K_LEFT]:
        x_sub -= SUB_SPEED

    if key[pygame.K_UP]:
        y_sub -= SUB_SPEED

    if key[pygame.K_DOWN]:
        y_sub += SUB_SPEED

    sub_in_scene()

# Check if the sub is still on the visible part of the screen
def sub_in_scene():
   global x_sub, y_sub

   if x_sub < 0:
      x_sub = 0

   if y_sub < 0:
      y_sub = 0

   if x_sub + sub.get_width() > WIDTH:
      x_sub = WIDTH - sub.get_width()

   if y_sub + sub.get_height() > HEIGHT:
      y_sub = HEIGHT - sub.get_height()

# Create many bubbles
def create_bubbles(state) :

    x_bubble = WIDTH
    y_bubble = random.randint(0, HEIGHT)

    if state == "Good":
        #bubble = pygame.image.load("Ressources/bulle.png")
        size_bubble = random.randint(blue_bubble.get_width() / 3, blue_bubble.get_width() * 2)
    else:
        #bubble = pygame.image.load("Ressources/red_bulle.png")
        size_bubble = random.randint(bad_bubble.get_width(), bad_bubble.get_width() * 3)

    # bubble = pygame.transform.scale (bubble, (size_bubble, size_bubble))

    # bubbles_id.append(bubble)
    bubbles_pos.append((x_bubble, y_bubble))
    bubbles_speed.append(random.randint(1, BUBBLE_MAX_SPEED))
    bubbles_state.append(state)
    bubbles_size.append(size_bubble)

# Move the bubble on the screen at set speed
def move_bubbles():
    for i in range (len(bubbles_pos) - 1, -1, -1) :
        bubbles_pos[i] = tuple(map(operator.sub, bubbles_pos[i], (bubbles_speed[i], 0)))

# Update bubble position
def update_game():

    global bonus, game_end

    if (random.randint(1, 20) == 1):
        create_bubbles("Good")

    if (random.randint(1, 60) == 1):
        create_bubbles("Bad")

    collision()

    if (int(score / BONUS_SCORE)) > bonus:
        bonus += 1
        game_end += TIME_LIMIT

    move_bubbles()
    clean_bubbles()

# Collision between the sub and the bubbles
def collision () :

    global score, game_end

    for bubble in range(len(bubbles_pos) -1, -1, -1):
      if (x_sub < bubbles_pos[bubble][0] + bubbles_size[bubble]
          and x_sub + sub.get_width() > bubbles_pos[bubble][0]
          and y_sub < bubbles_pos[bubble][1] + bubbles_size[bubble]
          and y_sub + sub.get_height() > bubbles_pos[bubble][1]) :
        # print ("La bulle ", bubble, "se superpose au sous-marin")
        print("etat de la bulle : ", bubbles_state[bubble])
        if bubbles_state[bubble] == "Good":
            score += bubbles_size[bubble] + bubbles_speed[bubble]
        else:
            game_end -= 5

        # print ("points : ", score)
        pop_sound.play(0)
        delete_bubble (bubble)

# Delete Bubble when it collides with the submarine
def delete_bubble (bubble):
    del bubbles_state[bubble]
    del bubbles_speed[bubble]
    del bubbles_pos[bubble]
    del bubbles_size[bubble]
    # del bubbles_id[bubble]

# Remove bubbles who leave the screen
def clean_bubbles ():
    for i in range (len(bubbles_pos) - 1, -1, -1) :
        if (bubbles_pos[i][0] + bubbles_size[i] < 0) :
            delete_bubble(i)

# Display colored text in position X and Y
def display_text(text, color, font, font_size, x, y):

    myfont = pygame.font.SysFont(font, font_size, True)
    message = myfont.render(text, True, color)

    screen.blit(message, (x,y))

# Game Over Screen
def game_over_message():

    pygame.mixer.stop()

    lose_sound.play(0)

    screen.fill(BLUE)

    display_text("GAME OVER !", RED, 'Calibri', 40, WIDTH * 0.4, HEIGHT * 0.2 )

    message = "Ton Score : " +  str(score)
    display_text(message, RED, 'Calibri', 40, WIDTH * 0.37, HEIGHT * 0.4 )

    display_text("Appuie sur R pour rejouer !", WHITE, 'Calibri', 30, WIDTH * 0.33, HEIGHT * 0.6)

# Initialize game variables when restart
def init_game():
    global score, x_sub, y_sub, game_end, bubbles_pos, bubbles_size, bubbles_speed, bubbles_state

    game_end = time.time() + TIME_LIMIT
    score = 0
    x_sub = 40
    y_sub = 250

    # bubbles_id = list()
    bubbles_pos = list()
    bubbles_size = list()
    bubbles_speed = list()
    bubbles_state = list()

# Window Init
pygame.init()

# Display creation
screen = pygame.display.set_mode ((WIDTH, HEIGHT))

# Set the repetition rate of the key
pygame.key.set_repeat(1, 1)

# Window Name
pygame.display.set_caption("Bubble Blaster")

# The Background image
background_image = pygame.image.load("Ressources/ocean.jpg")

# The submarine
sub = pygame.image.load("Ressources/submarine.png")

# The bubble
blue_bubble = pygame.image.load("Ressources/blue_bubble.png")
bad_bubble = pygame.image.load("Ressources/red_bubble.png")

pop_sound = pygame.mixer.Sound("Ressources/collect.wav")
ambient_sound = pygame.mixer.Sound("Ressources/ambient_music.wav")
lose_sound = pygame.mixer.Sound("Ressources/lose.wav")
ambient_sound.set_volume(0.05)

#create_bubble()

# Main loop
while True:

    pygame.mixer.stop()

    ambient_sound.play(-1)

    # Time loop
    while time.time() < game_end:
    #    move_bubble()
        update_game()
        update_screen()

        # Main event loop
        for event in pygame.event.get() :
            if event.type == pygame.QUIT:
                leave_game()
        sub_control()

    game_over_message()
    pygame.display.flip()

    restart = False

    while not restart:

        # Event Manager Loop
        for event in pygame.event.get() :
            if event.type == pygame.QUIT:
                leave_game()
            if not hasattr (event, 'key'):
                continue
            if event.key == K_r:
                restart = True
                init_game()
##            if event.key == K_ESCAPE:
##                leave_game()


import xarray as xr
import numpy as np

#Este script baixa os dados do hycom para os períodos selecionados para o experimento GLBv0.08/expt_53.X

#Importante: Por conta da estruturas dos servidores OpenDAP, e preciso baixar o dado por cada passo de tempo para postriormente concaternar
#Para concatenar, selecionar os arquivos desejados e utilizar o CDO, portando, este processamento é melhor realizado numa máquina Linux.

#Comando: cdo cat <*.nc> <saidamodeloteste.nc>
expt = ['http://tds.hycom.org/thredds/dodsC/GLBv0.08/expt_56.3',
        'http://tds.hycom.org/thredds/dodsC/GLBv0.08/expt_57.2',
        'http://tds.hycom.org/thredds/dodsC/GLBv0.08/expt_57.7',
        'http://tds.hycom.org/thredds/dodsC/GLBv0.08/expt_92.8',
        'http://tds.hycom.org/thredds/dodsC/GLBv0.08/expt_92.9',
        'http://tds.hycom.org/thredds/dodsC/GLBv0.08/expt_93.0',
        ]

#Parametros de entrada - Lembrando que as coordenadas deve ser passadas em WGS84 graus decimais
x = -73.575979
y = 11.552520

prof_ini = 0
prof_max = 1000

#Opcao para exportar area ao redor do ponto
#celulas ao redor. 0 para extrair apenas a localização mais proxima ao ponto
cell = 2
area = 0 + cell

for ex in expt:

  hycom = xr.open_dataset(ex,decode_times=False,decode_cf=False)

  if '_9' in ex:
    hycom['lon'] = hycom.lon-360

  #extraindo area ou pontos do HYCOM
  if area ==0:
    hycom = hycom.sel(lon=x, lat=y,method='nearest')
    hycom = hycom.sel(depth = slice(prof_ini,prof_max))

  if area >0:
    #matriz de distancias
    dist = ((hycom.lon-x)**2 + (hycom.lat-y)**2)**0.5
    #procurar pelo indice do modelo com as coordenadas mais proximas ao dado
    ind = np.unravel_index(np.argmin(dist, axis=None), dist.shape)
    hycom = hycom.isel(lon=slice(ind[0]-area,ind[0]+area), lat=slice(ind[1]-area,ind[1]+area))
    hycom = hycom.sel(depth = slice(prof_ini,prof_max))

  #dropando informações nao necessarias
  hycom = hycom.drop(['tau','surf_el','water_temp_bottom','salinity_bottom','water_u_bottom','water_v_bottom'])

  for i in list(range(0,len(hycom.time))):

    try:
      hyc = hycom.isel(time = i)
      hyc = hyc.load()
      hyc.to_netcdf('Hycom_Expt{}_{}.nc'.format(ex[-4:],i))

    except:
      pass


#########  import statements for sample_models.py ###########
from keras import backend as K
from keras.models import Model
from keras.layers import (BatchNormalization, Conv1D, Dense, Input,
    TimeDistributed, Activation, Bidirectional, SimpleRNN, GRU, LSTM)

################################

###########  import statements for train_utils.py #############
# from data_generator import AudioGenerator   ## Now codes of data_generator.py are pasted here. So I think that this import is useless
import _pickle as pickle

from keras import backend as K
from keras.models import Model
from keras.layers import (Input, Lambda, BatchNormalization)
from keras.optimizers import SGD, RMSprop
from keras.callbacks import ModelCheckpoint
import os
#####################################################

############ import and variable definitions for data_generator.py #############
import json
import numpy as np
import random
from python_speech_features import mfcc
import librosa
import scipy.io.wavfile as wav
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import make_axes_locatable

from utils import calc_feat_dim, spectrogram_from_file, text_to_int_sequence
from utils import conv_output_length

RNG_SEED = 123
######################################################################

##################### all codes of data_generator.py starts here ############################3

class AudioGenerator():
    def __init__(self, step=10, window=20, max_freq=8000, mfcc_dim=13,
        minibatch_size=20, desc_file=None, spectrogram=True, max_duration=10.0,
        sort_by_duration=False):
        """
        Params:
            step (int): Step size in milliseconds between windows (for spectrogram ONLY)
            window (int): FFT window size in milliseconds (for spectrogram ONLY)
            max_freq (int): Only FFT bins corresponding to frequencies between
                [0, max_freq] are returned (for spectrogram ONLY)
            desc_file (str, optional): Path to a JSON-line file that contains
                labels and paths to the audio files. If this is None, then
                load metadata right away
        """

        self.feat_dim = calc_feat_dim(window, max_freq)   # spectogram
        self.mfcc_dim = mfcc_dim
        self.feats_mean = np.zeros((self.feat_dim,))
        self.feats_std = np.ones((self.feat_dim,))
        self.rng = random.Random(RNG_SEED)
        if desc_file is not None:
            self.load_metadata_from_desc_file(desc_file)
        self.step = step
        self.window = window
        self.max_freq = max_freq
        self.cur_train_index = 0
        self.cur_valid_index = 0
        self.cur_test_index = 0
        self.max_duration=max_duration
        self.minibatch_size = minibatch_size
        self.spectrogram = spectrogram
        self.sort_by_duration = sort_by_duration

    def get_batch(self, partition):
        """ Obtain a batch of train, validation, or test data
        """
        if partition == 'train':
            audio_paths = self.train_audio_paths
            cur_index = self.cur_train_index
            texts = self.train_texts
        elif partition == 'valid':
            audio_paths = self.valid_audio_paths
            cur_index = self.cur_valid_index
            texts = self.valid_texts
        elif partition == 'test':
            audio_paths = self.test_audio_paths
            cur_index = self.test_valid_index
            texts = self.test_texts
        else:
            raise Exception("Invalid partition. "
                "Must be train/validation")

        features = [self.normalize(self.featurize(a)) for a in
            audio_paths[cur_index:cur_index+self.minibatch_size]]

        # calculate necessary sizes
        max_length = max([features[i].shape[0]
            for i in range(0, self.minibatch_size)])
        max_string_length = max([len(texts[cur_index+i])
            for i in range(0, self.minibatch_size)])

        # initialize the arrays
        X_data = np.zeros([self.minibatch_size, max_length,
            self.feat_dim*self.spectrogram + self.mfcc_dim*(not self.spectrogram)])
        labels = np.ones([self.minibatch_size, max_string_length]) * 28  # blanks
        input_length = np.zeros([self.minibatch_size, 1])
        label_length = np.zeros([self.minibatch_size, 1])

        for i in range(0, self.minibatch_size):
            # calculate X_data & input_length
            feat = features[i]
            input_length[i] = feat.shape[0]
            X_data[i, :feat.shape[0], :] = feat

            # calculate labels & label_length
            label = np.array(text_to_int_sequence(texts[cur_index+i]))
            labels[i, :len(label)] = label
            label_length[i] = len(label)

        # return the arrays
        outputs = {'ctc': np.zeros([self.minibatch_size])}
        inputs = {'the_input': X_data,
                  'the_labels': labels,
                  'input_length': input_length,
                  'label_length': label_length
                 }
        return (inputs, outputs)

    def shuffle_data_by_partition(self, partition):
        """ Shuffle the training or validation data
        """
        if partition == 'train':
            self.train_audio_paths, self.train_durations, self.train_texts = shuffle_data(
                self.train_audio_paths, self.train_durations, self.train_texts)
        elif partition == 'valid':
            self.valid_audio_paths, self.valid_durations, self.valid_texts = shuffle_data(
                self.valid_audio_paths, self.valid_durations, self.valid_texts)
        else:
            raise Exception("Invalid partition. "
                "Must be train/validation")

    def sort_data_by_duration(self, partition):
        """ Sort the training or validation sets by (increasing) duration
        """
        if partition == 'train':
            self.train_audio_paths, self.train_durations, self.train_texts = sort_data(
                self.train_audio_paths, self.train_durations, self.train_texts)
        elif partition == 'valid':
            self.valid_audio_paths, self.valid_durations, self.valid_texts = sort_data(
                self.valid_audio_paths, self.valid_durations, self.valid_texts)
        else:
            raise Exception("Invalid partition. "
                "Must be train/validation")

    def next_train(self):
        """ Obtain a batch of training data
        """
        while True:
            ret = self.get_batch('train')
            self.cur_train_index += self.minibatch_size
            if self.cur_train_index >= len(self.train_texts) - self.minibatch_size:
                self.cur_train_index = 0
                self.shuffle_data_by_partition('train')
            yield ret

    def next_valid(self):
        """ Obtain a batch of validation data
        """
        while True:
            ret = self.get_batch('valid')
            self.cur_valid_index += self.minibatch_size
            if self.cur_valid_index >= len(self.valid_texts) - self.minibatch_size:
                self.cur_valid_index = 0
                self.shuffle_data_by_partition('valid')
            yield ret

    def next_test(self):
        """ Obtain a batch of test data
        """
        while True:
            ret = self.get_batch('test')
            self.cur_test_index += self.minibatch_size
            if self.cur_test_index >= len(self.test_texts) - self.minibatch_size:
                self.cur_test_index = 0
            yield ret

    def load_train_data(self, desc_file='train_corpus.json'):
        self.load_metadata_from_desc_file(desc_file, 'train')
        self.fit_train()
        if self.sort_by_duration:
            self.sort_data_by_duration('train')

    def load_validation_data(self, desc_file='valid_corpus.json'):
        self.load_metadata_from_desc_file(desc_file, 'validation')
        if self.sort_by_duration:
            self.sort_data_by_duration('valid')

    def load_test_data(self, desc_file='test_corpus.json'):
        self.load_metadata_from_desc_file(desc_file, 'test')

    def load_metadata_from_desc_file(self, desc_file, partition):
        """ Read metadata from a JSON-line file
            (possibly takes long, depending on the filesize)
        Params:
            desc_file (str):  Path to a JSON-line file that contains labels and
                paths to the audio files
            partition (str): One of 'train', 'validation' or 'test'
        """
        audio_paths, durations, texts = [], [], []
        with open(desc_file) as json_line_file:
            for line_num, json_line in enumerate(json_line_file):
                try:
                    spec = json.loads(json_line)
                    if float(spec['duration']) > self.max_duration:
                        continue
                    audio_paths.append(spec['key'])
                    durations.append(float(spec['duration']))
                    texts.append(spec['text'])
                except Exception as e:
                    # Change to (KeyError, ValueError) or
                    # (KeyError,json.decoder.JSONDecodeError), depending on
                    # json module version
                    print('Error reading line #{}: {}'
                                .format(line_num, json_line))
        if partition == 'train':
            self.train_audio_paths = audio_paths
            self.train_audio_paths = self.train_audio_paths[:500]   # changed
            self.train_durations = durations
            self.train_durations = self.train_durations[:500]  # changed
            self.train_texts = texts
            self.train_texts = self.train_texts[:500]  # changed
        elif partition == 'validation':
            self.valid_audio_paths = audio_paths
            self.valid_audio_paths = self.valid_audio_paths[:50]  # changed
            self.valid_durations = durations
            self.valid_durations = self.valid_durations[:50]  # changed
            self.valid_texts = texts
            self.valid_texts = self.valid_texts[:50]  # changed
        elif partition == 'test':
            self.test_audio_paths = audio_paths
            self.test_durations = durations
            self.test_texts = texts
        else:
            raise Exception("Invalid partition to load metadata. "
             "Must be train/validation/test")

    def fit_train(self, k_samples=100):
        """ Estimate the mean and std of the features from the training set
        Params:
            k_samples (int): Use this number of samples for estimation
        """
        k_samples = min(k_samples, len(self.train_audio_paths))
        samples = self.rng.sample(self.train_audio_paths, k_samples)
        feats = [self.featurize(s) for s in samples]
        feats = np.vstack(feats)
        self.feats_mean = np.mean(feats, axis=0)
        self.feats_std = np.std(feats, axis=0)

    def featurize(self, audio_clip):
        """ For a given audio clip, calculate the corresponding feature
        Params:
            audio_clip (str): Path to the audio clip
        """
        if self.spectrogram:
            return spectrogram_from_file(
                audio_clip, step=self.step, window=self.window,
                max_freq=self.max_freq)
        else:
            (rate, sig) = wav.read(audio_clip)
            return mfcc(sig, rate, numcep=self.mfcc_dim)

    def normalize(self, feature, eps=1e-14):
        """ Center a feature using the mean and std
        Params:
            feature (numpy.ndarray): Feature to normalize
        """
        return (feature - self.feats_mean) / (self.feats_std + eps)

def shuffle_data(audio_paths, durations, texts):
    """ Shuffle the data (called after making a complete pass through
        training or validation data during the training process)
    Params:
        audio_paths (list): Paths to audio clips
        durations (list): Durations of utterances for each audio clip
        texts (list): Sentences uttered in each audio clip
    """
    p = np.random.permutation(len(audio_paths))
    audio_paths = [audio_paths[i] for i in p]
    durations = [durations[i] for i in p]
    texts = [texts[i] for i in p]
    return audio_paths, durations, texts

def sort_data(audio_paths, durations, texts):
    """ Sort the data by duration
    Params:
        audio_paths (list): Paths to audio clips
        durations (list): Durations of utterances for each audio clip
        texts (list): Sentences uttered in each audio clip
    """
    p = np.argsort(durations).tolist()
    audio_paths = [audio_paths[i] for i in p]
    durations = [durations[i] for i in p]
    texts = [texts[i] for i in p]
    return audio_paths, durations, texts

def vis_train_features(index=0):
    """ Visualizing the data point in the training set at the supplied index
    """
    # obtain spectrogram
    audio_gen = AudioGenerator(spectrogram=True)
    audio_gen.load_train_data()
    vis_audio_path = audio_gen.train_audio_paths[index]
    vis_spectrogram_feature = audio_gen.normalize(audio_gen.featurize(vis_audio_path))
    # obtain mfcc
    audio_gen = AudioGenerator(spectrogram=False)
    audio_gen.load_train_data()
    vis_mfcc_feature = audio_gen.normalize(audio_gen.featurize(vis_audio_path))
    # obtain text label
    vis_text = audio_gen.train_texts[index]
    # obtain raw audio
    vis_raw_audio, _ = librosa.load(vis_audio_path)
    # print total number of training examples
    print('There are %d total training examples.' % len(audio_gen.train_audio_paths))
    # return labels for plotting
    return vis_text, vis_raw_audio, vis_mfcc_feature, vis_spectrogram_feature, vis_audio_path

def plot_raw_audio(vis_raw_audio):
    # plot the raw audio signal
    fig = plt.figure(figsize=(12,3))
    ax = fig.add_subplot(111)
    steps = len(vis_raw_audio)
    ax.plot(np.linspace(1, steps, steps), vis_raw_audio)
    plt.title('Audio Signal')
    plt.xlabel('Time')
    plt.ylabel('Amplitude')
    plt.show()

def plot_mfcc_feature(vis_mfcc_feature):
    # plot the MFCC feature
    fig = plt.figure(figsize=(12,5))
    ax = fig.add_subplot(111)
    im = ax.imshow(vis_mfcc_feature, cmap=plt.cm.jet, aspect='auto')
    plt.title('Normalized MFCC')
    plt.ylabel('Time')
    plt.xlabel('MFCC Coefficient')
    divider = make_axes_locatable(ax)
    cax = divider.append_axes("right", size="5%", pad=0.05)
    plt.colorbar(im, cax=cax)
    ax.set_xticks(np.arange(0, 13, 2), minor=False);
    plt.show()

def plot_spectrogram_feature(vis_spectrogram_feature):
    # plot the normalized spectrogram
    fig = plt.figure(figsize=(12,5))
    ax = fig.add_subplot(111)
    im = ax.imshow(vis_spectrogram_feature, cmap=plt.cm.jet, aspect='auto')
    plt.title('Normalized Spectrogram')
    plt.ylabel('Time')
    plt.xlabel('Frequency')
    divider = make_axes_locatable(ax)
    cax = divider.append_axes("right", size="5%", pad=0.05)
    plt.colorbar(im, cax=cax)
    plt.show()

################################# all codes of data_generator.py ends here ###########################3

# from data_generator import vis_train_features  # ## Now codes of data_generator.py are pasted here. So I think that this import is useless

# extract label and audio features for a single training example
vis_text, vis_raw_audio, vis_mfcc_feature, vis_spectrogram_feature, vis_audio_path = vis_train_features()

# allocate 50% of GPU memory (if you like, feel free to change this)
from keras.backend.tensorflow_backend import set_session
from keras.optimizers import RMSprop, SGD
import tensorflow as tf
"""
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.5
set_session(tf.Session(config=config))
"""

# watch for any changes in the sample_models module, and reload it automatically
#%load_ext autoreload
#%autoreload 2
# import NN architectures for speech recognition
# from sample_models import *  # I have pasted code of sample_models in this file. So no need to import this
# import function for training acoustic model
# from train_utils import train_model  # I have pasted code of train_utils in this file. So no need to import this

import numpy as np
# from data_generator import AudioGenerator  ## Now codes of data_generator.py are pasted here. So I think that this import is useless
from keras import backend as K
from utils import int_sequence_to_text
from IPython.display import Audio

######################  All codes / model defined in sample_models.py start here ################

def simple_rnn_model(input_dim, output_dim=29):
    """ Build a recurrent network for speech
    """
    # Main acoustic input
    input_data = Input(name='the_input', shape=(None, input_dim))
    # Add recurrent layer
    simp_rnn = GRU(output_dim, return_sequences=True,
                 implementation=2, name='rnn')(input_data)
    # Add softmax activation layer
    y_pred = Activation('softmax', name='softmax')(simp_rnn)
    # Specify the model
    model = Model(inputs=input_data, outputs=y_pred)
    model.output_length = lambda x: x
    print(model.summary())
    return model

def rnn_model(input_dim, units, activation, output_dim=29):
    """ Build a recurrent network for speech
    """
    # Main acoustic input
    input_data = Input(name='the_input', shape=(None, input_dim))
    # Add recurrent layer
    simp_rnn = LSTM(units, activation=activation,
        return_sequences=True, implementation=2, name='rnn')(input_data)
    # TODO: Add batch normalization
    bn_rnn = BatchNormalization(name='bn_rnn_1d')(simp_rnn)
    # TODO: Add a TimeDistributed(Dense(output_dim)) layer
    time_dense = TimeDistributed(Dense(output_dim))(bn_rnn)
    # Add softmax activation layer
    y_pred = Activation('softmax', name='softmax')(time_dense)
    # Specify the model
    model = Model(inputs=input_data, outputs=y_pred)
    model.output_length = lambda x: x
    print(model.summary())
    return model

def cnn_rnn_model(input_dim, filters, kernel_size, conv_stride,
    conv_border_mode, units, output_dim=29):
    """ Build a recurrent + convolutional network for speech
    """
    # Main acoustic input
    input_data = Input(name='the_input', shape=(None, input_dim))
    # Add convolutional layer
    conv_1d = Conv1D(filters, kernel_size,
                     strides=conv_stride,
                     padding=conv_border_mode,
                     activation='relu',
                     name='conv1d')(input_data)
    # Add batch normalization
    bn_cnn = BatchNormalization(name='bn_conv_1d')(conv_1d)
    # Add a recurrent layer
    simp_rnn = GRU(units, activation='relu',
        return_sequences=True, implementation=2, name='rnn')(bn_cnn)
    # TODO: Add batch normalization
    bn_rnn = BatchNormalization(name='bn_rnn_1d')(simp_rnn)
    # TODO: Add a TimeDistributed(Dense(output_dim)) layer
    time_dense = TimeDistributed(Dense(output_dim))(bn_rnn)
    # Add softmax activation layer
    y_pred = Activation('softmax', name='softmax')(time_dense)
    # Specify the model
    model = Model(inputs=input_data, outputs=y_pred)
    model.output_length = lambda x: cnn_output_length(
        x, kernel_size, conv_border_mode, conv_stride)
    print(model.summary())
    return model

def cnn_output_length(input_length, filter_size, border_mode, stride,
                       dilation=1):
    """ Compute the length of the output sequence after 1D convolution along
        time. Note that this function is in line with the function used in
        Convolution1D class from Keras.
    Params:
        input_length (int): Length of the input sequence.
        filter_size (int): Width of the convolution kernel.
        border_mode (str): Only support `same` or `valid`.
        stride (int): Stride size used in 1D convolution.
        dilation (int)
    """
    if input_length is None:
        return None
    assert border_mode in {'same', 'valid'}
    dilated_filter_size = filter_size + (filter_size - 1) * (dilation - 1)
    if border_mode == 'same':
        output_length = input_length
    elif border_mode == 'valid':
        output_length = input_length - dilated_filter_size + 1
    return (output_length + stride - 1) // stride

def deep_rnn_model(input_dim, units, recur_layers, output_dim=29):
    """ Build a deep recurrent network for speech
    """
    # Main acoustic input
    input_data = Input(name='the_input', shape=(None, input_dim))
    # TODO: Add recurrent layers, each with batch normalization
    if recur_layers == 1:
        layer = LSTM(units, return_sequences=True, activation='relu')(input_data)
        layer = BatchNormalization(name='bt_rnn_1')(layer)
    else:
        layer = LSTM(units, return_sequences=True, activation='relu')(input_data)
        layer = BatchNormalization(name='bt_rnn_1')(layer)

        for i in range(recur_layers - 2):
            layer = LSTM(units, return_sequences=True, activation='relu')(layer)
            layer = BatchNormalization(name='bt_rnn_{}'.format(2+i))(layer)

        layer = LSTM(units, return_sequences=True, activation='relu')(layer)
        layer = BatchNormalization(name='bt_rnn_last_rnn')(layer)

    # TODO: Add a TimeDistributed(Dense(output_dim)) layer
    time_dense = TimeDistributed(Dense(output_dim))(layer)
    # Add softmax activation layer
    y_pred = Activation('softmax', name='softmax')(time_dense)
    # Specify the model
    model = Model(inputs=input_data, outputs=y_pred)
    model.output_length = lambda x: x
    print(model.summary())
    return model

def bidirectional_rnn_model(input_dim, units, output_dim=29):
    """ Build a bidirectional recurrent network for speech
    """
    # Main acoustic input
    input_data = Input(name='the_input', shape=(None, input_dim))
    # TODO: Add bidirectional recurrent layer
    bidir_rnn = Bidirectional(LSTM(units, return_sequences=True, activation='relu'), merge_mode='concat')(input_data)
    # TODO: Add a TimeDistributed(Dense(output_dim)) layer
    time_dense = TimeDistributed(Dense(output_dim))(bidir_rnn)
    # Add softmax activation layer
    y_pred = Activation('softmax', name='softmax')(time_dense)
    # Specify the model
    model = Model(inputs=input_data, outputs=y_pred)
    model.output_length = lambda x: x
    print(model.summary())
    return model

def final_model(input_dim, filters, kernel_size, conv_stride,
    conv_border_mode, units, output_dim=29, dropout_rate=0.5, number_of_layers=2,
    cell=GRU, activation='tanh'):
    """ Build a deep network for speech
    """
    # Main acoustic input
    input_data = Input(name='the_input', shape=(None, input_dim))
    # TODO: Specify the layers in your network
    conv_1d = Conv1D(filters, kernel_size,
                     strides=conv_stride,
                     padding=conv_border_mode,
                     activation='relu',
                     name='layer_1_conv',
                     dilation_rate=1)(input_data)
    conv_bn = BatchNormalization(name='conv_batch_norm')(conv_1d)

    if number_of_layers == 1:
        layer = cell(units, activation=activation,
            return_sequences=True, implementation=2, name='rnn_1', dropout=dropout_rate)(conv_bn)
        layer = BatchNormalization(name='bt_rnn_1')(layer)
    else:
        layer = cell(units, activation=activation,
                    return_sequences=True, implementation=2, name='rnn_1', dropout=dropout_rate)(conv_bn)
        layer = BatchNormalization(name='bt_rnn_1')(layer)

        for i in range(number_of_layers - 2):
            layer = cell(units, activation=activation,
                        return_sequences=True, implementation=2, name='rnn_{}'.format(i+2), dropout=dropout_rate)(layer)
            layer = BatchNormalization(name='bt_rnn_{}'.format(i+2))(layer)

        layer = cell(units, activation=activation,
                    return_sequences=True, implementation=2, name='final_layer_of_rnn')(layer)
        layer = BatchNormalization(name='bt_rnn_final')(layer)

    time_dense = TimeDistributed(Dense(output_dim))(layer)
    # TODO: Add softmax activation layer
    y_pred = Activation('softmax', name='softmax')(time_dense)
    # Specify the model
    model = Model(inputs=input_data, outputs=y_pred)
    # TODO: Specify model.output_length
    model.output_length = lambda x: cnn_output_length(
        x, kernel_size, conv_border_mode, conv_stride)
    print(model.summary())
    return model

##################################### code / model defined in sample_models.py ends here ##############################

########################## all codes of train_utils.py starts here #########################

def ctc_lambda_func(args):
    y_pred, labels, input_length, label_length = args
    #print("y_pred.shape = " + str(y_pred.shape))
    #print("labels.shape = " + str(labels.shape))
    #print("input_length.shape = " + str(input_length.shape))
    #print("label_length.shape = " + str(label_length.shape))
    return K.ctc_batch_cost(labels, y_pred, input_length, label_length)  # input_length= seq length of each item in y_pred
	# label_length is the seq length of each item in labels

def add_ctc_loss(input_to_softmax):
    the_labels = Input(name='the_labels', shape=(None,), dtype='float32')
    input_lengths = Input(name='input_length', shape=(1,), dtype='int64')
    label_lengths = Input(name='label_length', shape=(1,), dtype='int64')
    output_lengths = Lambda(input_to_softmax.output_length)(input_lengths)
    # output_length = BatchNormalization()(input_lengths)
    # CTC loss is implemented in a lambda layer

    loss_out = Lambda(ctc_lambda_func, output_shape=(1,), name='ctc')(
        [input_to_softmax.output, the_labels, output_lengths, label_lengths])
    model = Model(
        inputs=[input_to_softmax.input, the_labels, input_lengths, label_lengths],
        outputs=loss_out)
    return model

def train_model(input_to_softmax,
                pickle_path,
                save_model_path,
                train_json='train_corpus.json',
                valid_json='valid_corpus.json',
                minibatch_size=20,
                spectrogram=True,
                mfcc_dim=13,
                optimizer=SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=5),
                epochs=20,
                verbose=1,
                sort_by_duration=False,
                max_duration=10.0):

    # create a class instance for obtaining batches of data
    audio_gen = AudioGenerator(minibatch_size=minibatch_size,
        spectrogram=spectrogram, mfcc_dim=mfcc_dim, max_duration=max_duration,
        sort_by_duration=sort_by_duration)
    # add the training data to the generator
    audio_gen.load_train_data(train_json)
    audio_gen.load_validation_data(valid_json)
    # calculate steps_per_epoch
    num_train_examples=len(audio_gen.train_audio_paths)
    steps_per_epoch = num_train_examples//minibatch_size
    # calculate validation_steps
    num_valid_samples = len(audio_gen.valid_audio_paths)
    validation_steps = num_valid_samples//minibatch_size

    # add CTC loss to the NN specified in input_to_softmax
    model = add_ctc_loss(input_to_softmax)

    # CTC loss is implemented elsewhere, so use a dummy lambda function for the loss
    model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=optimizer)

    # make results/ directory, if necessary
    if not os.path.exists('results'):
        os.makedirs('results')

    # add checkpointer
    checkpointer = ModelCheckpoint(filepath='results/'+save_model_path, verbose=0)

    # train the model
    hist = model.fit_generator(generator=audio_gen.next_train(), steps_per_epoch=steps_per_epoch,
        epochs=epochs, validation_data=audio_gen.next_valid(), validation_steps=validation_steps,
        callbacks=[checkpointer], verbose=verbose)

    # save model loss
    with open('results/'+pickle_path, 'wb') as f:
        pickle.dump(hist.history, f)

################################ all codes of train_utils.py ends here #######################################

"""
model_0 = simple_rnn_model(input_dim=13) # change to 13 if you would like to use MFCC features
"""

"""
train_model(input_to_softmax=model_0,
            pickle_path='model_0.pickle',
            save_model_path='model_0.h5',
            spectrogram=False) # change to False if you would like to use MFCC features

"""

model_end = final_model(input_dim=13,
                        filters=200,
                        kernel_size=11,
                        conv_stride=2,
                        conv_border_mode='valid',
                        units=200,
                        activation='relu',
                        cell=GRU,
                        dropout_rate=1,
                        number_of_layers=2)

train_model(input_to_softmax=model_end,
            pickle_path='model_end.pickle',
            save_model_path='model_end.h5',
            epochs=5,
            spectrogram=False)

"""

model_4 = bidirectional_rnn_model(input_dim=13, # change to 13 if you would like to use MFCC features
                                  units=200)

train_model(input_to_softmax=model_4,
            pickle_path='model_4.pickle',
            save_model_path='model_4.h5',
            epochs=5,
                optimizer=SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True, clipnorm=2),
            spectrogram=False) # change to False if you would like to use MFCC features

"""

def get_predictions(index, partition, input_to_softmax, model_path):
    """ Print a model's decoded predictions
    Params:
        index (int): The example you would like to visualize
        partition (str): One of 'train' or 'validation'
        input_to_softmax (Model): The acoustic model
        model_path (str): Path to saved acoustic model's weights
    """
    # load the train and test data
    data_gen = AudioGenerator(spectrogram=False)
    data_gen.load_train_data()
    data_gen.load_validation_data()

    # obtain the true transcription and the audio features
    if partition == 'validation':
        transcr = data_gen.valid_texts[index]
        audio_path = data_gen.valid_audio_paths[index]
        data_point = data_gen.normalize(data_gen.featurize(audio_path))
    elif partition == 'train':
        transcr = data_gen.train_texts[index]
        audio_path = data_gen.train_audio_paths[index]
        data_point = data_gen.normalize(data_gen.featurize(audio_path))
    else:
        raise Exception('Invalid partition!  Must be "train" or "validation"')

    # obtain and decode the acoustic model's predictions
    input_to_softmax.load_weights(model_path)
    prediction = input_to_softmax.predict(np.expand_dims(data_point, axis=0))
    print("prediction.shape: " + str(prediction.shape))
    output_length = [input_to_softmax.output_length(data_point.shape[0])]
    pred_ints = (K.eval(K.ctc_decode(
                prediction, output_length)[0][0])+1).flatten().tolist()
    print("pred_ints: " + str(pred_ints))
    print("len(pred_ints): " + str(len(pred_ints)))

    # play the audio file, and display the true and predicted transcriptions
    print('-'*80)
    Audio(audio_path)
    print('True transcription:\n' + '\n' + transcr)
    print('-'*80)
    print('Predicted transcription:\n' + '\n' + ''.join(int_sequence_to_text(pred_ints)))
    print('-'*80)

"""
get_predictions(index=2,
                partition='validation',
                input_to_softmax=model_end,
                model_path='results/model_end.h5')

"""

"""

get_predictions(index=1,
                partition='validation',
                input_to_softmax=model_0,
                model_path='results/model_0.h5')

"""


from kubeflow.fairing.cloud.docker import get_docker_secret
from kubeflow.fairing.constants import constants
import json
import os

def test_docker_secret_spec():
    os.environ["DOCKER_CONFIG"] = "/tmp"
    config_dir = os.environ.get('DOCKER_CONFIG')
    config_file_name = 'config.json'
    config_file = os.path.join(config_dir, config_file_name)
    with open(config_file, 'w+') as f:
        json.dump({'config': "config"}, f)
    docker_secret = get_docker_secret()
    assert docker_secret.metadata.name == constants.DOCKER_CREDS_SECRET_NAME
    os.remove(config_file)


import logging
import os
from typing import List
from plumbum import cmd, local
from pathlib import Path

import doit
from doit.action import CmdAction

from constants import DEFAULT_DB, DB_USERNAME, DB_PASSWORD, VERBOSITY_DEFAULT

logging.basicConfig()
logger = logging.getLogger("dodo")
logger.setLevel(logging.DEBUG)

NOISEPAGE_PATH = Path.joinpath(Path.home(), "noisepage-pilot").absolute()
ARTIFACTS_PATH = Path.joinpath(NOISEPAGE_PATH, "artifacts/benchbase")
PROJECT_PATH = Path.joinpath(NOISEPAGE_PATH, "artifacts/project")
POSTGRES_PATH = str(Path.joinpath(Path.home(), "postgres/build/bin"))
POSTGRES_DATA_PATH = str(Path.joinpath(Path.home(), "postgresql/data"))
ARTIFACT_benchbase = Path.joinpath(ARTIFACTS_PATH, "benchbase.jar")
ARTIFACT_benchbase_results = ARTIFACT_benchbase / "results"

PSQL = "/home/kramana2/postgres/build/bin/psql"

BENCHBASE_CONFIG_TAGS = {
    "scalefactor": "/parameters/scalefactor",
    "time": "/parameters/works/work/time",
    "rate": "/parameters/works/work/rate",
    "terminals": "/parameters/terminals",
}

def task_hello():
    return {"actions": ["echo 'Hello world!'"], "verbosity": VERBOSITY_DEFAULT}

def get_config_path(benchmark, config=None) -> str:
    """
    Fetches the path to the config file of the given benchmark.
    """
    if config is None:
        config = PROJECT_PATH / f"{benchmark}_config.xml"
    elif not config.startswith("/"):
        config = Path(NOISEPAGE_PATH / config).absolute()

    return str(config)

def task_update_log_collection():
    sql_list = [
        "ALTER SYSTEM SET log_destination='csvlog'",
        "ALTER SYSTEM SET logging_collector='on'",
        "ALTER SYSTEM SET log_statement='all'",
        "ALTER SYSTEM SET log_connections='on'",
        "ALTER SYSTEM SET log_disconnections='on'",
        "ALTER SYSTEM SET log_directory='%(log_directory)s'",
    ]

    return {
        "actions": [
            f"mkdir -p {POSTGRES_DATA_PATH}/%(log_directory)s",
            *[
                f'PGPASSWORD={DB_PASSWORD} {PSQL} --host=localhost --dbname={DEFAULT_DB} --username={DB_USERNAME} --command="{sql}"'
                for sql in sql_list
            ],
        ],
        "params": [
            {
                "name": "log_directory",
                "long": "log_directory",
                "default": "log",
            },
            {
                "name": "log_file",
                "long": "log_file",
                "default": "postgresql-%Y-%m-%d_%H%M%S.log",
            },
        ],
        "verbosity": VERBOSITY_DEFAULT,
    }

def task_perform_vacuum():
    """
    Postgres: Performs vacuuming on the database system.
    """
    return {
        "actions": [
            *[
                f'PGPASSWORD={DB_PASSWORD} {PSQL} --host=localhost --dbname={DEFAULT_DB} --username={DB_USERNAME} --command="VACUUM;"'
            ],
        ],
        "params": [],
        "verbosity": VERBOSITY_DEFAULT,
    }

def task_update_config():
    def update_xml(benchmark, scalefactor=1, time=60, rate=10, terminals=1):
        kwargs = locals().copy()
        del kwargs["benchmark"]

        config = get_config_path(benchmark)
        logger.info(f"Updating arguments in config file {config} with values: {kwargs}")

        actions = []
        for param in kwargs:
            # We're assuming that all keys in kwargs are in BENCHBASE_CONFIG_TAGS
            key = BENCHBASE_CONFIG_TAGS[param]
            value = locals()[param]
            cmd = f"xmlstarlet edit --inplace --update '{key}' --value \"{value}\" {config}"
            actions.append(cmd)

        return "; \n".join(actions)

    return {
        "actions": [
            CmdAction(update_xml),
        ],
        "params": [
            {
                "name": "benchmark",
                "long": "benchmark",
                "help": "The benchmark to run.",
                "default": "epinions",
            },
            {
                "name": "scalefactor",
                "long": "scalefactor",
                "default": 1,
            },
            {
                "name": "time",
                "long": "time",
                "default": 60,  # 60s
            },
            {
                "name": "rate",
                "long": "rate",
                "default": 10,
            },
            {
                "name": "terminals",
                "long": "terminals",
                "default": 1,
            },
        ],
        "verbosity": VERBOSITY_DEFAULT,
    }

def task_benchbase_workload_create():
    """
    Benchbase: initializes the specified benchmark.
    """

    def invoke_benchbase(benchmark, config, directory):
        config = get_config_path(benchmark, config)
        return f"echo {config}; java -jar benchbase.jar -b {benchmark} -c {config} -d {directory} --create=true --load=true"

    return {
        "actions": [
            lambda: os.chdir(str(ARTIFACTS_PATH)),
            # Invoke BenchBase.
            CmdAction(invoke_benchbase),
            # Reset working directory.
            lambda: os.chdir(doit.get_initial_workdir()),
        ],
        "file_dep": [ARTIFACT_benchbase],
        "uptodate": [False],
        "verbosity": VERBOSITY_DEFAULT,
        "params": [
            {
                "name": "benchmark",
                "long": "benchmark",
                "help": "The benchmark to run.",
                "default": "epinions",
            },
            {
                "name": "config",
                "long": "config",
                "help": (
                    "The config file to use for BenchBase."
                    "Defaults to the config in the artifacts folder for the selected benchmark."
                ),
                "default": None,
            },
            {
                "name": "directory",
                "long": "directory",
                "default": f"{ARTIFACT_benchbase_results}",
            },
        ],
    }

def task_benchbase_run():
    """
    BenchBase: run a specific benchmark.
    """

    def invoke_benchbase(benchmark, config, directory, args):
        config = get_config_path(benchmark, config)
        return f"echo {config}; java -jar benchbase.jar -b {benchmark} -c {config} -d {directory} {args}"

    return {
        "actions": [
            lambda: os.chdir(str(ARTIFACTS_PATH)),
            # Invoke BenchBase.
            CmdAction(invoke_benchbase),
            # Reset working directory.
            lambda: os.chdir(doit.get_initial_workdir()),
        ],
        "file_dep": [ARTIFACT_benchbase],
        "uptodate": [False],
        "verbosity": VERBOSITY_DEFAULT,
        "params": [
            {
                "name": "benchmark",
                "long": "benchmark",
                "help": "The benchmark to run.",
                "default": "epinions",
            },
            {
                "name": "config",
                "long": "config",
                "help": (
                    "The config file to use for BenchBase."
                    "Defaults to the config in the artifacts folder for the selected benchmark."
                ),
                "default": None,
            },
            {
                "name": "directory",
                "long": "directory",
                "default": f"{ARTIFACT_benchbase_results}",
            },
            {
                "name": "args",
                "long": "args",
                "help": "Arguments to pass to BenchBase invocation.",
                "default": "--create=false --load=false --execute=false",
            },
        ],
    }


from airflow import DAG
from airflow.operators.bash_operator import BashOperator
import datetime as dt
from airflow.utils.dates import days_ago

default_args = {
    'owner': 'gregh',
    'start_date': days_ago(0),
    'email': ['myemail@gmail.com'],
    'email_on_failure': True,
    'email_on_retry': True,
    'retries': 2,
    'retry_delay': dt.timedelta(minutes=5)
}

dag = DAG(
    dag_id='process_web_log',
    schedule_interval=dt.timedelta(days=1),
    default_args=default_args,
    description='Airflow Web Log Daily Processor'
)

extract_data = BashOperator(
    task_id='extract',
    bash_command='cut -d "-" -f1 /home/project/airflow/dags/capstone/accesslogs.txt > /home/project/airflow/dags/capstone/extracted_data.txt',
    dag=dag
)

transform_data = BashOperator(
    task_id='transform',
    bash_command='sed "/198.46.149.143/d" /home/project/airflow/dags/capstone/extracted_data.txt > /home/project/airflow/dags/capstone/transformed_data.txt',
    dag=dag
)

load_data = BashOperator(
    task_id='load',
    bash_command='tar -cvf /home/project/airflow/dags/capstone/weblog.tar /home/project/airflow/dags/capstone/transformed_data.txt',
    dag=dag
)

extract_data >> transform_data >> load_data

Projects/Capstone Project/Task 5/Part Two - Apache Airflow ETL/process_web_log.py


from datetime import datetime as dt
from datetime import timedelta
import pickle
import time

import dask.dataframe as dd
from dask.distributed import as_completed, worker_client
import numpy as np
import pandas as pd
import requests
import s3fs

BUCKET = "insulator-citi-bikecaster"
INSULATOR_URLS = [
    "https://api-dev.insulator.ai/v1/time_series",
    "https://ybcbwoz3w6.execute-api.us-east-1.amazonaws.com/staging/v1/time_series"
]

s3 = s3fs.S3FileSystem()

def model_key(station_id):
    return f"models/station_{station_id}.pkl"

def load_model(station_id):
    with s3.open(f"{BUCKET}/{model_key(station_id)}", "rb") as f:
        return pickle.loads(f.read())

def load_local_model(station_id):
    with open(f"models/station_{station_id}.pkl", "rb") as f:
        return pickle.load(f)

def ts_to_unixtime(series):
    return series.astype(np.int64) // 10 ** 9

def post_outcome(df, station_id, usernames, api_keys):
    two_hours_ago = dt.now() - timedelta(hours=2)
    past_two_hours = df[df["last_reported"] >= two_hours_ago]
    past_two_hours = past_two_hours.sort_values("last_reported")

    series_timestamps = ts_to_unixtime(past_two_hours["last_reported"]).tolist()
    series_values = past_two_hours["num_bikes_available"].astype("int").tolist()
    post_event(station_id, series_timestamps, series_values, "outcome", usernames, api_keys)

def post_event(station_id, series_timestamps, series_values, event_type, usernames, api_keys):
    payload = {
        "service_name": "bikecaster",
        "model_name": "lin_reg",
        "model_version": "0.1.0",
        "timestamp": time.time(),
        "entities": {"station_id": station_id},
        "series_timestamps": series_timestamps,
        "series_values": series_values
    }
    assert event_type in ("prediction", "outcome")
    for username, api_key, insulator_url in zip(usernames, api_keys, INSULATOR_URLS):
        url = f"{insulator_url}/{event_type}"
        try:
            response = requests.post(url, auth=(username, api_key), json=payload)
            if not response:
                print(f"Error posting to insulator ingest API: {response.text}")
        except Exception as e:
            print(e)

def make_forecast(df, station_id, usernames, api_keys):
    station_df = df[df["station_id"] == station_id]

    post_outcome(station_df, station_id, usernames, api_keys)

    station_df = (
        station_df
        .set_index("last_reported")
        .sort_index()
        .resample("5T", label="right", closed="right")
        .last()
        .fillna(method="ffill")
    )
    y = station_df["num_bikes_available"].values.copy()
    X = y.reshape(-1, 1).copy()

    try:
        model = load_local_model(station_id)
    except:
        print(f"There's no model for station {station_id}")
        return False

    try:
        series_values = np.squeeze(model.predict(X, start_idx=len(X) - 1))
    except:
        print(f"Error predicting for station {station_id}")
        return False

    series_values = np.clip(series_values.astype(int), 0, None).astype("int").tolist()
    series_timestamps = pd.date_range(
        station_df.index[-1], periods=len(series_values) + 1, freq="5T"
    )
    # Remove the first value because it's the last value in the original data.
    series_timestamps = series_timestamps[1:]
    series_timestamps = ts_to_unixtime(series_timestamps).astype("int").tolist()
    post_event(station_id, series_timestamps, series_values, "prediction", usernames, api_keys)
    return True

def pipeline(s3_path, usernames, api_keys):
    df = dd.read_csv(s3_path).compute()
    df["last_reported"] = pd.to_datetime(df["last_reported"])
    MIN_DATE = "2016-01-01"
    df = df[df.last_reported >= MIN_DATE]
    with worker_client() as client:
        df_future = client.scatter(df)
        futures = []
        for station_id in sorted(df["station_id"].unique().tolist()):
            futures.append(client.submit(make_forecast, df_future, station_id, usernames, api_keys))
        total = len(futures)
        success = 0
        for result in as_completed(futures):
            if result.result():
                success += 1
                if success % 50 == 0:
                    print(f"{success} / {total} tasks successfully completed")
    print(f"Done. Final tally: {success} / {total} tasks successfully completed")
    return True


import json
import os

class FileUtils:
    @staticmethod
    def readJsonFile(filePath):
        with open(filePath, 'r', encoding='utf-8') as file:
            jsonData = json.load(file)
        return jsonData

    @staticmethod
    def writeJsonFile(filePath, jsonData):
        with open(filePath, 'w', encoding='utf-8') as file:
            file.write(json.dumps(jsonData, sort_keys=False, indent=4, separators=(',', ': ')))

    @staticmethod
    def readLinesFromFile(filePath) -> list:
        with open(filePath, 'r', encoding='utf-8') as f:
            return [line.replace('\n', '') for line in f.readlines()]


import numpy as np
from os.path import join
from psbody.mesh import Mesh
from fitting.landmarks import load_embedding, landmark_error_3d, mesh_points_by_barycentric_coordinates, load_picked_points
from fitting.util import load_binary_pickle, write_simple_obj, safe_mkdir, get_unit_factor
import open3d as o3d
import argparse, os
from tqdm import tqdm
import logging
logger = logging.getLogger(__name__)

def get_config():
    parser = argparse.ArgumentParser(description='modify mean and std and orientation')
    parser.add_argument("--scans", type=str, default= "mesh", help='path of the scan') # for a mesh path, replace 'mesh' to 'lmk' get its corresponding lmk path
    parser.add_argument("--lmks", type=str, default= "lmk", help='path of the output')
    parser.add_argument("--save", type=str, default= "lx_result", help='path of the output')
    args = parser.parse_args()
    return args

def x_rotate(v):
    return v*[1, -1, -1]

def transl(v, old_mean, new_mean):
    return v-old_mean+new_mean

def transl_scale(v, old_mean, old_std, new_mean, new_std):
    return (v-old_mean)/old_std*new_std+new_mean

def modify_face(face):
    return face

def get_vertice_mean_std(v):
    return np.mean(v, axis=0), np.std(v)

def get_mean_std(filename):
    mesh = Mesh(filename=filename)
    if hasattr(mesh, 'f'):
        mesh.f = modify_face(mesh.f) # TODO: 尚未确定是否需要扭转面片方向
    mean = np.mean(mesh.v, axis=0)
    std = np.std(mesh.v)
    return mean, std, mesh

def flamefit_test():
    eg = './data/scan.obj'
    lmk = './data/scan_lmks.npy'
    eg_mean, eg_std, eg_mesh = get_mean_std(eg) # mean x-y-z分开算, std整体算
    eg_lmk = np.load(lmk)
    print(f'my example scan mean: {eg_mean}, std: {eg_std}')

    my_scan = "/mnt/cephfs/home/liuxu/cvte/tools/flame-fitting/data/test/mesh/3_pointcloud.obj"
    my_lmk = "/mnt/cephfs/home/liuxu/cvte/tools/flame-fitting/data/test/lmk/3_pointcloud.npy"
    mean, std, mesh = get_mean_std(my_scan)
    lmk = np.load(my_lmk)
    v = mesh.v
    print(f'my origina scan mean: {mean}, std: {std}')

    v = x_rotate(v)
    lmk = x_rotate(lmk)
    write_simple_obj(v, mesh.f if hasattr(mesh, 'f') else None, my_scan.replace('.obj', '_x.obj'))
    np.save(my_lmk.replace('.npy', '_x.npy'), lmk)
    mean, std = get_vertice_mean_std(v)
    print(f'my rotated scan mean: {mean}, std: {std}')

    v_transl = transl(v, mean, eg_mean)
    lmk_transl = transl(lmk, mean, eg_mean)
    write_simple_obj(v_transl, mesh.f if hasattr(mesh, 'f') else None, my_scan.replace('.obj', '_x_transl.obj'))
    np.save(my_lmk.replace('.npy', '_x_transl.npy'), lmk_transl)
    mean_transl, std_transl = get_vertice_mean_std(v_transl)
    print(f'my transla scan mean: {mean_transl}, std: {std_transl}')

    v = transl_scale(v, mean, std, eg_mean, eg_std)
    lmk = transl_scale(lmk, mean, std, eg_mean, eg_std)
    write_simple_obj(v, mesh.f if hasattr(mesh, 'f') else None, my_scan.replace('.obj', '_x_transl_scale.obj'))
    np.save(my_lmk.replace('.npy', '_x_transl_scale.npy'), lmk)
    mean, std = get_vertice_mean_std(v)
    print(f'my tra_sca scan mean: {mean}, std: {std}')

    # scale to similar size based on lmk
    eg_lmk = eg_lmk - eg_mean
    lmk = lmk - mean # 关键点相对于原点的坐标
    times = np.mean(np.mean(eg_lmk/lmk, axis=1)) # 关键点的avg倍数
    v = (v - mean)*times
    lmk = lmk*times
    mean, std = get_vertice_mean_std(v)
    print(f'my fang_da scan mean: {mean}, std: {std}')
    v = transl_scale(v, mean, std, eg_mean, eg_std)
    lmk = transl_scale(lmk, mean, std, eg_mean, eg_std)
    write_simple_obj(v, mesh.f if hasattr(mesh, 'f') else None, my_scan.replace('.obj', '_x_transl_scale_fangda.obj'))
    np.save(my_lmk.replace('.npy', '_x_transl_scale_fangda.npy'), lmk)
    mean, std = get_vertice_mean_std(v)
    print(f'my finally scan mean: {mean}, std: {std}')

# 只需要旋转并平移一下就ok了，调这个函数
def liuxu_flamefit():
    eg = './data/scan.obj'
    lmk = './data/scan_lmks.npy'
    eg_mean, eg_std, eg_mesh = get_mean_std(eg) # mean x-y-z分开算, std整体算
    eg_lmk = np.load(lmk)
    print(f'my example scan mean: {eg_mean}, std: {eg_std}')

    my_scan = "/mnt/cephfs/home/liuxu/cvte/tools/flame-fitting/data/new_cap/mesh/0_face.obj"
    my_lmk = "/mnt/cephfs/home/liuxu/cvte/tools/flame-fitting/data/new_cap/lmk/0_face.npy"
    mean, std, mesh = get_mean_std(my_scan)
    lmk = np.load(my_lmk)[-51:]
    v = mesh.v
    print(f'my origina scan mean: {mean}, std: {std}')

    v = x_rotate(v)
    lmk = x_rotate(lmk)
    # write_simple_obj(v, mesh.f if hasattr(mesh, 'f') else None, my_scan.replace('.obj', '_x.obj'))
    # np.save(my_lmk.replace('.npy', '_x.npy'), lmk)
    mean, std = get_vertice_mean_std(v)
    # print(f'my rotated scan mean: {mean}, std: {std}')

    v_transl = transl(v, mean, eg_mean) # 到这一步得到的obj，fit效果最好
    lmk_transl = transl(lmk, mean, eg_mean)
    write_simple_obj(v_transl, mesh.f if hasattr(mesh, 'f') else None, my_scan.replace('.obj', '_x_transl.obj'))
    np.save(my_lmk.replace('.npy', '_x_transl.npy'), lmk_transl)
    mean_transl, std_transl = get_vertice_mean_std(v_transl)
    print(f'my transla scan mean: {mean_transl}, std: {std_transl}')

    # v = transl_scale(v, mean, std, eg_mean, eg_std)
    # lmk = transl_scale(lmk, mean, std, eg_mean, eg_std)
    # write_simple_obj(v, mesh.f if hasattr(mesh, 'f') else None, my_scan.replace('.obj', '_x_transl_scale.obj'))
    # np.save(my_lmk.replace('.npy', '_x_transl_scale.npy'), lmk)
    # mean, std = get_vertice_mean_std(v)
    # print(f'my tra_sca scan mean: {mean}, std: {std}')

def get_lmk_meanstd(lmk):
    mean = np.mean(lmk, axis=0)
    std = np.std(lmk)
    return mean, std

# 只需要旋转并平移一下就ok了，调这个函数
def liuxu_modify_basedon_lmk():
    eg = 'data/scan.obj'
    lmk = 'data/scan_lmks.npy'
    eg_lmk = np.load(lmk)
    eg_mean, eg_std = get_lmk_meanstd(eg_lmk) # mean x-y-z分开算, std整体算
    print(f'my example lmk mean: {eg_mean}, std: {eg_std}')

    my_scan = "data/lizhenliang2/lizhenliang2_down10.ply"
    my_lmk = "data/lizhenliang2/lizhenliang2_picked_points.pp"
    lmk = get_lmk(my_lmk)[-51:]
    mean, std = get_lmk_meanstd(lmk)
    mesh = Mesh(filename=my_scan)
    v = mesh.v
    print(f'my origina lmk mean: {mean}, std: {std}')

    v = x_rotate(v)
    lmk = x_rotate(lmk)
    mean, std = get_lmk_meanstd(lmk)

    v_transl = transl(v, mean, eg_mean) # 到这一步得到的obj，fit效果最好
    lmk_transl = transl(lmk, mean, eg_mean)
    write_simple_obj(v_transl, mesh.f if hasattr(mesh, 'f') else None, my_scan.replace('.ply', '_x_transl_by_lmk.obj'))
    np.save(my_lmk.replace('.pp', '_x_transl_by_lmk.npy'), lmk_transl)
    mean_transl, std_transl = get_lmk_meanstd(lmk_transl)
    print(f'my transla lmk mean: {mean_transl}, std: {std_transl}')

    # v = transl_scale(v, mean, std, eg_mean, eg_std)
    # lmk = transl_scale(lmk, mean, std, eg_mean, eg_std)
    # write_simple_obj(v, mesh.f if hasattr(mesh, 'f') else None, my_scan.replace('.obj', '_x_transl_scale_by_lmk.obj'))
    # np.save(my_lmk.replace('.npy', '_x_transl_scale_by_lmk.npy'), lmk)
    # mean, std = get_lmk_meanstd(lmk)
    # print(f'my tra_sca lmk mean: {mean}, std: {std}')

    # print(f'the 13th lmk of example: {eg_lmk[13]}, my: {lmk[13]}')

def get_lmk(lmk_path):
    if lmk_path.endswith('.npy'):
        lmk = np.load(lmk_path)
    elif lmk_path.endswith('.pp'):
        lmk = load_picked_points(lmk_path)
    return lmk

def stupid_test():
    eg = './data/scan.obj'
    eg_mean, eg_std, eg_mesh = get_mean_std(eg)
    args = get_config()
    save_root = join('data', args.save)
    os.makedirs(save_root, exist_ok=True)
    save_scan = join(save_root, args.scans)
    os.makedirs(save_scan, exist_ok=True)
    save_lmk = join(save_root, args.lmks)
    os.makedirs(save_lmk, exist_ok=True)
    scans = join('./data/test', args.scans)
    for r, ds, fs in os.walk(scans):
        for f in tqdm(fs):
            if f.endswith("obj"):
                scan_path = os.path.join(r,f)
                print(scan_path)
                output = join(save_scan, f)
                mean, std, mesh = get_mean_std(scan_path)
                moved_v = (mesh.v - mean) # 把自己的mesh移到原点并归一化
                avg_v = np.mean(moved_v, axis=0)
                eg_v = (eg_mesh.v - eg_mean) # 把参考mesh移到原点并归一化
                avg_eg_v = np.mean(eg_v, axis=0)
                print(f'my origin scan mean: {mean}, origin example mean: {eg_mean}')
                print(f'my scan mean: {np.mean(moved_v, axis=0)}, example mean: {np.mean(eg_v, axis=0)}')
                avg_scale = np.mean(avg_eg_v/avg_v) * 8.5
                print("scale times: ", avg_scale)
                scaled_v = moved_v * avg_scale #  这时的mesh应该和示例大小差不多
                v = moved_v + eg_mean # 没有放大，只是移动了位置
                print(f"my new mean: {np.mean(v, axis=0)}, eg_mean: {eg_mean}")
                write_simple_obj(v, mesh.f if hasattr(mesh, 'f') else None, output)

                # 对应修改关键点坐标
                lmk_path = scan_path.replace(args.scans, args.lmks).replace('obj', 'npy')
                ori_lmk = np.load(lmk_path)
                ori_lmk *= [1, -1, -1]
                lmk_output = join(save_lmk, f.replace('obj', 'npy'))
                moved_lmk = (ori_lmk - mean)
                scaled_lmk = moved_lmk * avg_scale
                modified_lmk = moved_lmk + eg_mean
                np.save(lmk_output, modified_lmk)

                # res_lmk = o3d.geometry.PointCloud()
                # res_lmk.points = o3d.utility.Vector3dVector(modified_lmk)
                # res_mesh = o3d.io.read_triangle_mesh(output)
                # o3d.visualization.draw_geometries([res_mesh, res_lmk, eg_mesh])

# 只需要旋转并平移一下就ok了，调这个函数
def modify(my_scan, my_lmk):
    eg = 'data/scan.obj'
    lmk = 'data/scan_lmks.npy'
    eg_lmk = np.load(lmk)
    eg_mean, eg_std = get_lmk_meanstd(eg_lmk) # mean x-y-z分开算, std整体算
    logger.info(f'my example lmk mean: {eg_mean}, std: {eg_std}')

    lmk = get_lmk(my_lmk)[-51:]
    mean, std = get_lmk_meanstd(lmk)
    mesh = Mesh(filename=my_scan)
    v = mesh.v
    logger.info(f'my origina lmk mean: {mean}, std: {std}')

    v = x_rotate(v)
    lmk = x_rotate(lmk)
    mean, std = get_lmk_meanstd(lmk)

    v_transl = transl(v, mean, eg_mean) # 到这一步得到的obj，fit效果最好
    lmk_transl = transl(lmk, mean, eg_mean)
    write_simple_obj(v_transl, mesh.f if hasattr(mesh, 'f') else None, my_scan.replace('.ply', '_x_transl_by_lmk.obj'))
    np.save(my_lmk.replace('.pp', '_x_transl_by_lmk.npy'), lmk_transl)
    mean_transl, std_transl = get_lmk_meanstd(lmk_transl)
    logger.info(f'my transla lmk mean: {mean_transl}, std: {std_transl}')

    trans = -mean + eg_mean
    logger.info(f"trans: {trans}")

    return my_scan.replace('.ply', '_x_transl_by_lmk.obj'), my_lmk.replace('.pp', '_x_transl_by_lmk.npy'), trans

if __name__ == '__main__':
    # flamefit_test()
    # liuxu_flamefit()
    liuxu_modify_basedon_lmk()


# pylint: disable=missing-docstring
# pylint: disable=invalid-name
import functools
import re

# import unicodedata
from string import punctuation as PUNCTUATIONS

import numpy as np

from doors.dates import get_timestamp

SPECIAL_PUNCTUATIONS = PUNCTUATIONS.replace("_", "")

def not_is_feat(col):
    return not is_feat(col)

def is_feat(col):
    return "feat:" in col

def clean_string(string):
    return string.lower().rstrip().replace(" ", "_").replace("'", "")

def to_lowercase(strings):
    strings = [string.lower() for string in strings]
    return strings

def get_pronounceable_name():
    consonants = ["b", "d", "f", "g", "h", "j", "k", "l", "m", "n", "p", "r", "s", "t"]
    vowels = ["a", "e", "i", "o", "u"]
    final_consonants = ["b", "f", "k", "l", "m", "n", "r", "s", "t"]
    return (
        np.random.choice(consonants)
        + np.random.choice(vowels)
        + np.random.choice(consonants)
        + np.random.choice(vowels)
        + np.random.choice(final_consonants)
    )

def get_unique_id():
    """Pronounceable hash to be pronounced more or less ecclesiastically.
    More details: https://www.ewtn.com/expert/answers/ecclesiastical_latin.htm
    """
    return get_pronounceable_name() + "_" + get_timestamp("%y%m%d_%H%M%S")

def add_as_strings(*args, **kwargs):
    result = args[0].astype(str)
    sep = kwargs.get("sep")
    if sep:
        seperator = np.repeat(sep, len(result))
    else:
        seperator = None

    for arr in args[1:]:
        if seperator is not None:
            result = _add_strings(result, seperator)
        result = _add_strings(result, arr.astype(str))
    return result

def _add_strings(v, w):
    return np.core.defchararray.add(v, w)

def camelcase_to_underscore(string):
    s1 = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", string)
    return re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1).lower()

def remove_punctuation(string):
    for punctuation in SPECIAL_PUNCTUATIONS:
        string = string.replace(punctuation, "")
    return string

# def utf_to_ascii(string):
#     uni_string = unicode(string, "utf")
#     ascii_string = unicodedata.normalize("NFKD", uni_string).encode("ascii", "ignore")
#     return ascii_string

def is_ascii(string):
    try:
        string.decode("ascii")
        return True
    except UnicodeDecodeError:
        return False

def as_string(obj):
    if hasattr(obj, "__name__"):
        representation = obj.__name__
    elif isinstance(obj, functools.partial):
        representation = _get_partial_representation(obj)
    elif hasattr(obj, "__dict__"):
        representation = get_class_representation(obj)
    elif hasattr(obj, "__name__"):
        representation = obj.__name__
    else:
        representation = str(obj)
    return representation

def _get_partial_representation(obj):
    func_rep = as_string(obj.func)
    input_rep = "func=" + func_rep
    if _args_provided(obj):
        arg_rep = _get_arg_representation(obj.args)
        input_rep += ", " + arg_rep
    if _kwargs_provided(obj):
        kwarg_rep = get_dict_string_representation(obj.keywords)
        input_rep += ", " + kwarg_rep
    partial_rep = "partial({})".format(input_rep)
    return partial_rep

def _kwargs_provided(obj):
    return len(obj.keywords) > 0

def _args_provided(obj):
    return len(obj.args) > 0

def _get_arg_representation(args):
    return ", ".join([str(arg) for arg in args])

def get_class_representation(obj):
    joint_str_rep = get_dict_string_representation(obj.__dict__)
    cls_name = obj.__class__.__name__
    return "{}({})".format(cls_name, joint_str_rep)

def get_dict_string_representation(dct):
    str_rep = []
    for key, value in dct.items():
        if key[0] != "_":
            value_representation = as_string(value)
            str_rep.append("{}={}".format(key, value_representation))
    joint_str_rep = ", ".join(str_rep)
    return joint_str_rep

def convert_camelcase(camelcase):
    """
    Credit to:
    http://stackoverflow.com/questions/1175208/elegant-python-function-to-convert-
    camelcase-to-snake-case
    """
    s1 = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", camelcase)
    return re.sub("([a-z0-9])([A-Z])", r"\1_\2", s1).lower()

def clean_white_space(array):
    array = np.array([_clean_white_space(i) for i in array])
    return array

def _clean_white_space(v):
    if isinstance(v, str):
        v = v.strip(" ")
    return v


import glob
import os
from os import path as op

import cv2
import numpy as np
from torch.utils.data import DataLoader
from pathlib import Path
from PIL import Image, ImageFilter
from detection.dummy_cnn.dataset import BaseBillOnBackGroundSet
from tqdm import tqdm
from sewar.full_ref import sam as sim_measure
from itertools import combinations, product
import time
from matplotlib import pyplot as plt
from multiprocessing import Pool
import pandas as pd

repo = Path(os.getcwd())

im_dir_gen = os.path.join(repo, "processed_data", "genbills")
im_dir_real = os.path.join(repo, "processed_data", "realbills")
im_dir_unseen = os.path.join(repo, "processed_data", "realbills", "unseen")

def resize(list_of_images, size):
    outp = []
    for im in tqdm(list_of_images):
        copy = im.copy()
        copy.thumbnail(size=(size, size), resample=Image.ANTIALIAS)
        if copy.width > copy.height:
            copy = copy.rotate(90, fillcolor=(0,), expand=True)
        outp.append(copy)
    return outp

def combs_self(list_of_images):
    return np.array(list(combinations(range(len(list_of_images)), r=2))).astype(int)

def combs_between(list_of_images1, list_of_images2):
    return np.array(list(product(range(len(list_of_images1)), range(len(list_of_images2))))).astype(int)

def simil(pair): # subfunction to put in parallel loop
    im_1, im_2 = pair
    m = ""
    if im_1.width != im_2.width or im_1.height != im_2.height:
        m = f"crop happened\n im1 dims = {im_1.width},{im_1.height},\n im2 dims = {im_2.width},{im_2.height}"
        min_w = min(im_1.width, im_2.width)
        min_h = min(im_1.height, im_2.height)
        im_1 = im_1.crop((1, 1, min_w-1, min_h-1))
        im_2 = im_2.crop((1, 1, min_w-1, min_h-1))
        m+= f"\n crop dims = 1to{min_w-1}, 1to{min_h-1}"
        m+= f"\n final dims = {im_1.width},{im_1.height}"

    try:
        score = sim_measure(np.array(im_1), np.array(im_2))
    except Exception as e:
        score = 0.5
        print(e)
        print(m)
    return score
def similarity(list_of_images1, list_of_images2, combs):
    similarity_score = 0
    list_of_images1 = [list_of_images1[idx] for idx in combs[:,0]]
    list_of_images2 = [list_of_images2[idx] for idx in combs[:,1]]
    with Pool(12) as pool:
        for score in tqdm(pool.imap(simil, zip(list_of_images1, list_of_images2)), total=len(list_of_images1)):
            similarity_score += score
        pool.close()
    similarity_score /= len(combs)
    return similarity_score

def edgin(image): #task function to put in Pool loop
    corners = cv2.goodFeaturesToTrack(np.array(image.convert("L")), int(1e+6), 1e-6, 1e-6)
    return len(corners)
def edginess(list_of_images):
    score = 0
    with Pool(12) as pool:
        for corners in tqdm(pool.imap(edgin, list_of_images), total=len(list_of_images)):
            score += corners
    score /= len(list_of_images)
    return score

# This script is meant do discover which size for training corner_cnn is the best
generated_images = BaseBillOnBackGroundSet(image_dir=im_dir_gen)
loader = DataLoader(dataset=generated_images,
                    batch_size=1,
                    num_workers=12,
                    shuffle=True)
temp = []
for im, _ in tqdm(loader, total=200):
    im = im[0].numpy()
    where_0 = np.sum(im, axis=2) > 0
    for row, element in enumerate(where_0):
        if np.all(element == 0): break

    for col, element in enumerate(where_0.T):
        if np.all(element == 0): break

    im = im[:row, :col, :]
    try:
        temp.append(Image.fromarray(im))
    except:
        print("Error occured")

    if len(temp) == 200: break

generated_images = temp

real_images = glob.glob(op.join(im_dir_real, "*.jpg"), recursive=False)
real_images = [Image.open(file) for file in real_images if not "mask" in file]#[:8]

test_images = glob.glob(op.join(im_dir_unseen, "*.jpg"), recursive=False)
test_images = [Image.open(file) for file in test_images if not "mask" in file]#[:8]

sizes = np.geomspace(1000, 10, 100).astype(int)
scores = {'sim_gen': [],
          'sim_real': [],
          'sim_test': [],
          'sim_gen_vs_real': [],
          'sim_gen_vs_test': [],
          'sim_test_vs_real': [],
          "edg_gen": [],
          "edg_real": [],
          "edg_test": []}
print("#" * 100)
print()
for size in sizes:
    images_of_size = {"gen": [], "real": [], "test": []}

    print(f"Resizing {size}")
    images_of_size['gen'] = resize(generated_images, size)
    images_of_size['real'] = resize(real_images, size)
    images_of_size['test'] = resize(test_images, size)
    time.sleep(2)

    print(f"\nCollect similarity inside every set {size}")
    for k in images_of_size.keys():
        sim = similarity(list_of_images1=images_of_size[k],
                         list_of_images2=images_of_size[k],
                         combs=combs_self(images_of_size[k]))
        scores[f'sim_{k}'].append(sim)
    time.sleep(2)

    print(f"\nCollect similarity inbetween sets {size}")
    for k_pair in [("gen", "real"), ("gen", "test"), ("test", "real")]:
        sim = similarity(list_of_images1=images_of_size[k_pair[0]],
                         list_of_images2=images_of_size[k_pair[1]],
                         combs=combs_between(list_of_images1=images_of_size[k_pair[0]],
                                             list_of_images2=images_of_size[k_pair[1]]))

        scores[f'sim_{k_pair[0]}_vs_{k_pair[1]}'].append(sim)
    time.sleep(2)

    print(f"\nCollect edginess of every set {size}")
    for k in images_of_size.keys():
        edg = edginess(list_of_images=images_of_size[k])
        scores[f'edg_{k}'].append(edg)
    time.sleep(2)

    # plotting current results
    num_el = len(scores["sim_gen"])
    f, ax = plt.subplots(nrows=3, ncols=1, figsize=(10, 15))

    ax[0].set_title("Dissimilarity of images within each set")
    ax[0].set_xlabel("Size of image")
    ax[0].plot(sizes[:num_el][::-1], scores["sim_gen"][::-1], label="generated images", c="red")
    ax[0].plot(sizes[:num_el][::-1], scores["sim_real"][::-1], label="real images", c="blue")
    ax[0].plot(sizes[:num_el][::-1], scores["sim_test"][::-1], label="test images", c="blue", ls=":")

    ax[1].set_title("Dissimilarity of images between sets")
    ax[1].set_xlabel("Size of image")
    ax[1].plot(sizes[:num_el][::-1], scores["sim_gen_vs_real"][::-1], label="generated vs real images", c="blue")
    ax[1].plot(sizes[:num_el][::-1], scores["sim_gen_vs_test"][::-1], label="generated vs test images", c="blue", ls=":")
    ax[1].plot(sizes[:num_el][::-1], scores["sim_test_vs_real"][::-1], label="real vs test images", c="green")

    ax[2].set_title("Number of corners detected of images within each set")
    ax[2].set_xlabel("Size of image")
    ax[2].plot(sizes[:num_el][::-1], scores["edg_gen"][::-1], label="generated images", c="red")
    ax[2].plot(sizes[:num_el][::-1], scores["edg_real"][::-1], label="real images", c="blue")
    ax[2].plot(sizes[:num_el][::-1], scores["edg_test"][::-1], label="test images", c="blue", ls=":")
    ax[2].set_yscale('log')

    for a in ax:
        a.legend()
        a.grid(axis="x", which="both")
        a.invert_xaxis()
        a.set_xscale('log')

    plt.tight_layout()
    plt.savefig("/home/sasha/Documents/BachelorsProject/Repo/real_bills_results/comp_sizes/0_stats.png", dpi=150)
    plt.close("all")

    # save examples of images
    images_of_size['gen'][0].save(f"/home/sasha/Documents/BachelorsProject/Repo/real_bills_results/comp_sizes/generated_{size}.png")
    images_of_size['real'][0].save(f"/home/sasha/Documents/BachelorsProject/Repo/real_bills_results/comp_sizes/real_{size}.png")
    images_of_size['test'][0].save(f"/home/sasha/Documents/BachelorsProject/Repo/real_bills_results/comp_sizes/test_{size}.png")

    #save scores
    frame = pd.DataFrame(scores)
    frame.set_index(sizes[:num_el], inplace=True)
    frame.to_csv(f"/home/sasha/Documents/BachelorsProject/Repo/real_bills_results/comp_sizes/0_scores.csv", sep=";")
    print("#" * 100)


from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from bs4 import BeautifulSoup
import time
import csv

class Scraper:
    def __init__(self, url):
        self.driver = webdriver.Chrome("./chromedriver", options=self.set_chrome_options())
        self.url = url
        self.open_url()
        self.content = self.get_content()

    def set_chrome_options(self):
        chrome_options = Options()
        chrome_options.add_argument("--headless")
        chrome_options.add_argument("--disable-gpu")
        return chrome_options

    def open_url(self):
        self.driver.get(self.url)

    def get_content(self):
        content = self.driver.page_source
        soup = BeautifulSoup(content, "html.parser")
        return soup

    # retrieves all elements with a chosen html tag
    def get_all_tags(self, tag="h1"):
        all_tags = []
        for element in self.content.select(tag):
            all_tags.append(element.text.strip())

        return all_tags

    def get_items(self, product_container='div.thumbnail'):
        top_items = []

        products = self.content.select(product_container)
        for elem in products:
            title = elem.select('h4 > a.title')[0].text
            review_label = elem.select('div.ratings')[0].text
            info = {
                "title": title.strip(),
                "review": review_label.strip()
            }
            top_items.append(info)

        print(top_items)
        # return(top_items)

    def get_all_products(self, content_container='div.thumbnail'):
        all_products = []

        products = self.content.select(content_container)
        for product in products:
            name = product.select('h4 > a')[0].text.strip()
            description = product.select('p.description')[0].text.strip()
            price = product.select('h4.price')[0].text.strip()
            reviews = product.select('div.ratings')[0].text.strip()
            image = product.select('img')[0].get('src')

            all_products.append({
                "name": name,
                "description": description,
                "price": price,
                "reviews": reviews,
                "image": image
            })

        # print(all_products)
        return all_products

    def quit(self):
        self.driver.quit()

    def save_product_csv(self, all_products):
        keys = all_products[0].keys()

        with open('products.csv', 'w', newline='') as output_file:
            dict_writer = csv.DictWriter(output_file, keys)
            dict_writer.writeheader()
            dict_writer.writerows(all_products)

if __name__ == "__main__":
    urls = [
        "https://webscraper.io/test-sites/e-commerce/allinone",
        "https://webscraper.io/test-sites/e-commerce/allinone/computers",
        "https://webscraper.io/test-sites/e-commerce/allinone/computers/laptops",
        "https://webscraper.io/test-sites/e-commerce/allinone/computers/tablets",
        "https://webscraper.io/test-sites/e-commerce/allinone/phones",
        "https://webscraper.io/test-sites/e-commerce/allinone/phones/touch"
    ]

    start_time = time.time()

    for url in urls:
        scraper = Scraper(url)
        print("products:", scraper.get_all_products())
        scraper.quit()

    total_time = time.time() - start_time

    print("time:", total_time)


import argparse
import cv2

# ArgParse é usado para captar argumentos passados na chamada do .py no CMD
ap = argparse.ArgumentParser()
# Aqui definimos a label do argumento esperado
ap.add_argument("-i", "--image", required=True,
                help= "Path to the image")
# Criamos um dicionário que receberá os valores dos argumentos
# As chaves do dicionário serão as labels criadas no na definição do argumento
args = vars(ap.parse_args())
# A função vars() retorna os valores correspondente ao atributo __dict__ do objeto

# Aqui lemos a imagem que é acessada através do caminho no disco passado como argumento.
# Acessamos o valor em args usando como chave do dicionário args o mesmo valor que a definição do argumento
image = cv2.imread(args["image"])
print("width: {} pixels".format(image.shape[1]))
print("height: {} pixels".format(image.shape[0]))
print("channels: {}".format(image.shape[2]))
print("Matrix shape: {}".format(image.shape))

cv2.imshow("Image", image)
cv2.waitKey(0)

cv2.imwrite("newimage.jpg", image)


from django.views.generic.base import TemplateView
from albums.forms import FileForm
from albums.models import Album, File
from core.decorators import view_decorator
from core.views import ResourceView

class AlbumPage(TemplateView):
    template_name = "albums/main.html"

def expose(view):
    view.expose = True
    return view

@view_decorator(expose)
class AlbumView(ResourceView):
    model = Album

@view_decorator(expose)
class FileView(ResourceView):
    create_form = FileForm
    model = File


from PIL import Image
import os
from tkinter import filedialog
import tkinter as tk

def convert_pdf():
    index = 0
    path_picture = filedialog.askdirectory()
    dire = 'Converted'
    path_pdf = os.path.join(path_picture , dire)
    os.mkdir(path_pdf)

    my_list = os.listdir(path_picture)
    for i in my_list:
        image = Image.open(r'' + path_picture+'//' + i)
        im = image.convert('RGB')
        im.save(r''+ path_pdf+'//' + i[:-4] +'.pdf', quality=15, optimze=True)
        index = index + 1

root = tk.Tk()
convert_pdf()
tk.mainloop()


import numpy as np
import argparse
import imutils
import cv2

ap = argparse.ArgumentParser()
ap.add_argument("-i","--image",required = True, help="Path of Image File")
args = vars(ap.parse_args())

#image = cv2.imread("image.png")
print("Path: ", args["image"])
image = cv2.imread(args["image"])

# find all the 'black' shapes in the image
upper = np.array([15,15,15])
lower = np.array([0,0,0])
shapeMask = cv2.inRange(image,lower,upper)

# find the contours in the mask
cnts = cv2.findContours(shapeMask.copy(), cv2.RETR_EXTERNAL,
	cv2.CHAIN_APPROX_SIMPLE)
cnts = imutils.grab_contours(cnts)
print("Found {} black shapes".format(len(cnts)))
cv2.imshow("Mask", shapeMask)
# loop over the contours
for c in cnts:
	# draw the contour and show it
	cv2.drawContours(image, [c], -1, (0, 255, 0), 2)
	cv2.imshow("Image", image)
	cv2.waitKey(0)


import matplotlib.pyplot as plt
import random
import numpy as np
from IPython.display import display, clear_output
import time

def head_home(x, y):
    """
    Head home down and to the left.

    Parameters
    ----------

    x : float
        Horizontal coordinate.

    y : float
        Vertical coordinate.

    Returns
    -------
    x : float
        Updated horizontal coordinate.

    y : float
        Updated vertical coordinate.

    """

    pick = np.zeros(x + y)
    pick[0:x] = 1
    if (np.random.choice(pick) == 1):
        x -= 1
    else:
        y -= 1
    if (x < 0):
        x = 0
    if (y < 0):
        y = 0
    return x, y

def search_for_food(x, y, smell):
    """
    Search for food by following the smell.

    Parameters
    ----------

    x : float
        Horizontal coordinate.

    y : float
        Vertical coordinate.

    smell : numpy.ndarray
        2D array of smells

    Returns
    -------
    x : float
        Updated horizontal coordinate.

    y : float
        Updated vertical coordinate.

    """
    directions = ['up', 'left', 'down', 'right']

    x_dim = smell.shape[0]
    y_dim = smell.shape[1]

    # First check to see if there is food up and to the right.
    g = []  # follow gradient
    m = []

    if (x + 1 < x_dim):
        if (smell[x + 1, y] > 0):
            m.append(smell[x + 1, y])
            g.append('right')
    if (y + 1 < y_dim):
        if (smell[x, y + 1] > 0):
            m.append(smell[x, y + 1])
            g.append('up')
    if (g != []):
        grad = g[m.index(max(m))]
        # print("Following smell", grad)
    else:
        # else just pick a random direction.
        grad = random.choice(directions)
        # print("Choosing ",grad)

    # move the ant
    if (grad == 'up'):
        y = y + 1
    elif (grad == 'right'):
        x = x + 1
    elif (grad == 'down'):
        y = y - 1
    elif (grad == 'left'):
        x = x - 1
    else:
        print(grad)
        print("ERROR!!!!!!!!!!!!")

    # make sure we don't go off the gird.
    if (x < 0):
        x = 0
    if (y < 0):
        y = 0
    if (x > x_dim - 1):
        x = x_dim - 1
    if (y > y_dim - 1):
        y = y_dim - 1
    return x, y

def run(num_ants=100, x_dim=70, y_dim=30):
    """
    Run the simulation

    Parameters
    ----------
    num_ants : int
        Initial number of ants to simulate. Dafualt =100

    x_dim : int
        Horizontal dimension of the board. Default = 70

    y_dim : int
        Vertical  dimension of the board. Default = 30

    """
    smell = np.zeros((x_dim, y_dim))
    food = np.zeros((x_dim, y_dim))

    # place food
    food[45:50, 25:30] = 10
    food[45:50, 25:30] = 10
    food[65:70, 0:5] = 10

    x_loc = np.random.randint(0, x_dim, size=(num_ants, 1))
    y_loc = np.random.randint(0, y_dim, size=(num_ants, 1))
    ant_loc = np.concatenate((x_loc, y_loc), axis=1)

    has_food = np.zeros((num_ants, 1))

    fig, ax = plt.subplots(figsize=(10, 5))

    # Main simulation loop
    for i in range(0, 100):

        # Loop over ants
        for a in range(0, num_ants):
            x = ant_loc[a, 0]
            y = ant_loc[a, 1]

            if (x == 0 and y == 0):
                has_food[a] = 0

            if has_food[a] > 0:
                x, y = head_home(x, y)
                smell[x, y] = smell[x, y] + 100

            else:
                x, y = search_for_food(x, y, smell)

            if food[x, y] > 0:
                food[x, y] -= 1
                has_food[a] = 1

            ant_loc[a, 0] = x
            ant_loc[a, 1] = y

        smell = smell - 1
        smell[smell < 0] = 0

        # plot world
        plt.imshow(food.T, origin='lower', aspect='equal', cmap="magma")
        for a in range(0, num_ants):
            color = 'r'
            if (has_food[a] > 0):
                color = 'g'
            plt.scatter(ant_loc[a, 0], ant_loc[a, 1], color=color)

        # Animaiton part (dosn't change)
        clear_output(wait=True)  # Clear output for dynamic display
        display(fig)  # Reset display
        fig.clear()  # Prevent overlapping and layered plots
        time.sleep(0.0001)  # Sleep for a fraction of a second to allow animation to catch up