from .models import *
from .forms import *
from app import filtersets

import cx_Oracle

from django.http.response import Http404
from django.shortcuts import render, redirect
from django.contrib.auth import  login

from django.contrib.auth.forms import  AuthenticationForm

from django.contrib.auth.models import  User

from django.contrib.auth.decorators import login_required
from django.contrib import messages
from django.core.paginator import Paginator
from django.http import Http404, HttpResponse, HttpResponseRedirect

from django.db import connection

from functools import wraps

from django.contrib.admin.views import decorators

def staff_member_required(view_func):
    def _checklogin(request, *args, **kwargs):
        if request.user.is_active and request.user.is_staff:
            # The user is valid. Continue to the admin page.
            return view_func(request, *args, **kwargs)
        else:
            return HttpResponseRedirect('/acceso-denegado')
    return wraps(view_func)(_checklogin)

decorators.staff_member_required = staff_member_required

def accesodenegado(request):
    return render(request,'acceso-denegado.html')

def groups_only(*groups):
    def inner(view_func):
        @wraps(view_func)
        def wrapper_func(request, *args, **kwargs):
            if request.user.groups.filter(name__in=groups).exists():
                return view_func(request, *args, **kwargs)
            else:
                return redirect(to='acceso-denegado')
        return wrapper_func
    return inner

@staff_member_required
def crear_grupo(request):
    data = {
        'form': GrupoForm
    }
    if request.method == 'POST':
        formulario = GrupoForm(data=request.POST)
        if formulario.is_valid():
            gru = formulario.save()
            messages.success(request, "Grupo "+gru.name+" creado correctamente!")
            return redirect (to='mantenedor-usr')
        else:
            data["form"] = formulario
    return render(request, 'registration/group.html',data )

@staff_member_required
def user_filter(request):
    # https://www.youtube.com/watch?v=dkJ3uqkdCcY
    #https://django-filter.readthedocs.io/en/stable/index.html
    """
    filtro = filtersets.UsertFilter(
        request.GET,
        queryset= User.objects.all()
    )

    PerfilF = filtersets.PerfilFilter(
        request.GET,
        queryset= Perfil.objects.all()
    )
    page = request.GET.get('page', 1)
    try:
        paginator = Paginator(PerfilF, 5)
        PerfilF = paginator.page(page)
    except:
        raise Http404
    context = {
        'filtro': filtro,
        'entity':PerfilF,
        'paginator': paginator
    }
    """

    filtro = filtersets.UsertFilter(
        request.GET,
        queryset= User.objects.all()
    )

    PerfilF = filtersets.PerfilFilter(
        request.GET,
        queryset= Perfil.objects.all()
    )
    context = {
        'filtro': filtro,
        'PerfilF':PerfilF,
    }
    return render(request, 'pruebas/ekisde.html', context)

@staff_member_required
def signup_view(request):
    context = {'form': CustomUserCreationForm(),
    'form_p':PerfilForm(),
    'adminform':AdminForm(),
    'proform': ProfesionalForm(),
    'cliform': ClienteForm(),
    }

    if request.method == 'POST':
        formulario = CustomUserCreationForm(data=request.POST)
        formPerfil = PerfilForm(data=request.POST)
        formAdm = AdminForm(data=request.POST)
        formProf = ProfesionalForm(data=request.POST)
        formCli = ClienteForm(data=request.POST)

        if formulario.is_valid() and formPerfil.is_valid():
            usuario = formulario.save()
            group = request.POST.get('groups')
            usuario.groups.add(group)

            perfil = formPerfil.save(commit=False)
            perfil.id_auth_user = usuario
            perfil.save()

            if perfil.tipo_perf=='1':
                admin = formAdm.save(commit=False)
                admin.id_perfil = perfil
                admin.save()
            elif perfil.tipo_perf=='2':
                prof = formProf.save(commit=False)
                prof.id_perfil = perfil
                prof.save()
            elif perfil.tipo_perf=='3':
                cli = formCli.save(commit=False)
                cli.id_perfil =perfil
                cli.save()

            messages.success(request, 'Usuario '+usuario.username+' creado correctamente')
            return redirect(to="mantenedor-usr")
        context = {'form': CustomUserCreationForm(),
        'form_p':PerfilForm(),
        'adminform':AdminForm(),
        'proform':ProfesionalForm(),
        'cliform': ClienteForm(),
        }

    return render(request, 'registration/signup.html', context)

def home(request):
    return render(request, 'home.html')
@staff_member_required
def home_admin(request):
    usuario = User.objects.all().order_by('id')
    context = {'usuario': usuario }
    return render(request,'administrador/home-adm.html', context)
@staff_member_required
def maintainer(request):
    return render(request, 'administrador/mantenedor.html')
@staff_member_required
def maintainer_user(request):
    return render(request, 'administrador/mantenedor-usuario.html')
@staff_member_required
def maintainer_plan(request):
    return render(request, 'administrador/mantenedor-plan.html')
@staff_member_required
def maintainer_service(request):
    return render(request, 'administrador/mantenedor-servicio.html')

def login_view(request):
    if request.method == 'POST':
        form = AuthenticationForm(data=request.POST)
        if form.is_valid():
            user = form.get_user()
            login(request, user)
            return redirect(to='home')
    else:
            form = AuthenticationForm()
    return render(request,'registration/login.html',{'form':form})

def login_filter(request):
    if request.user.groups.filter(name="Administrador") or request.user.is_staff:
        return redirect(to='home-adm')
    elif request.user.groups.filter(name="Profesional"):
        return redirect(to='home-prof')
    else:
        return redirect(to='home-cliente')
@staff_member_required
#mantenedor
def UserLista(request):
    usuario = User.objects.all().order_by('id')
    page = request.GET.get('page', 1)
    try:
        paginator = Paginator(usuario, 5)
        usuario = paginator.page(page)
    except:
        raise Http404

    context = {'entity': usuario,
                'paginator': paginator}
    return render(request, 'administrador/lista.html', context)
@staff_member_required
def UserEdit(request,id):
    usuario = User.objects.get(id=id)

    if request.method == 'GET':
        form = UserUpdateForm(instance=usuario)
    else:
        form = UserUpdateForm(request.POST, instance=usuario)
        if form.is_valid():
            form.save()
            messages.success(request, "Usuario "+usuario.username+" modificado correctamente")
        return redirect(to="listar")
    context = {
            'form':form,
        }
    return render(request,'administrador/editar.html', context)
@staff_member_required
def UserDelete(request,id):
    usuario = User.objects.get(id=id)
    usuario.is_active = 0
    if request.method == 'POST':
        form = UserActive(instance=usuario)
    else:
        form = UserActive(request.POST, instance=usuario)
        if form.is_valid():
            form.save()
            messages.success(request, "Usuario desactivado correctamente")
    return redirect(to="listar")
@staff_member_required
def UserActivate(request,id):
    usuario = User.objects.get(id=id)
    if request.method == 'POST':
        form = UserActive(instance=usuario)
    else:
        form = UserActive(request.POST, instance=usuario)
        if form.is_valid():
            user = form.save()
            user.is_active = True
            user.save()
            messages.success(request, "Usuario activado correctamente")
    return redirect(to="listar")
@staff_member_required
##PLAN
def PlanCreate(request):
    data = {
        'form': PlanForm
    }

    if request.method == 'POST':
        formulario = PlanForm(data=request.POST)
        if formulario.is_valid():
            formulario.save()
            messages.success(request, "Plan creado correctamente!")
            return redirect (to='mantenedor')
        else:
            data["form"] = formulario
    return render(request, 'administrador/planes/agregar-plan.html', data)
@staff_member_required
def plan_lista(request):
    plan = Plan.objects.all().order_by('id_plan')
    page = request.GET.get('page', 1)
    try:
        paginator = Paginator(plan, 5)
        plan = paginator.page(page)
    except:
        raise Http404

    context = {'entity': plan,
                'paginator': paginator}
    return render(request, 'administrador/planes/lista-plan.html', context)
@staff_member_required
def PlanEdit(request,id_plan):
    plan = Plan.objects.get(id_plan=id_plan)
    if request.method == 'GET':
        form = PlanUpdateForm(instance=plan)
    else:
        form = PlanUpdateForm(request.POST, instance=plan)
        if form.is_valid():
            form.save()
            messages.success(request, "Plan modificado correctamente")
        return redirect(to='lista-plan')
    return render(request,'administrador/planes/editar-plan.html',{'form':form})

@staff_member_required
def PlanDelete(request,id):
    plan = Plan.objects.get(id_plan=id)
    plan.estado = 0
    if request.method == 'POST':
        form = PlanActive(instance=plan)
    else:
        form = PlanActive(request.POST, instance=plan)
        if form.is_valid():
            form.save()
            messages.success(request, "Plan desactivado correctamente")
    return redirect(to="lista-plan")
@staff_member_required
def PlanActivate(request,id):
    plan = Plan.objects.get(id_plan=id)
    if request.method == 'POST':
        form = PlanActive(instance=plan)
    else:
        form = PlanActive(request.POST, instance=plan)
        if form.is_valid():
            plan = form.save()
            plan.estado = 1
            plan.save()
            messages.success(request, "Plan activado correctamente")
    return redirect(to="lista-plan")

##SERVICIOS
@staff_member_required
def ServicioCreate(request):
    data = {
        'form': ServicioForm
    }

    if request.method == 'POST':
        formulario = ServicioForm(data=request.POST)
        if formulario.is_valid():
            formulario.save()
            messages.success(request, "Servicio creado correctamente!")
            return redirect (to='mantenedor')
        else:
            data["form"] = formulario
    return render(request, 'administrador/servicios/agregar-servicio.html', data)
@staff_member_required
def Servicio_lista(request):
    servicio = Servicio.objects.all().order_by('id_servicio')
    page = request.GET.get('page', 1)
    try:
        paginator = Paginator(servicio, 5)
        servicio = paginator.page(page)
    except:
        raise Http404

    context = {'entity': servicio,
                'paginator': paginator}
    return render(request, 'administrador/servicios/lista-servicio.html', context)
@staff_member_required
def ServicioEdit(request,id_servicio):
    servicio = Servicio.objects.get(id_servicio=id_servicio)
    if request.method == 'GET':
        form = ServicioUpdateForm(instance=servicio)
    else:
        form = ServicioUpdateForm(request.POST, instance=servicio)
        if form.is_valid():
            form.save()
            messages.success(request, "Servicio modificado correctamente")
        return redirect(to='lista-servicios')
    return render(request,'administrador/servicios/editar-servicio.html',{'form':form})

@staff_member_required
def ServicioDelete(request,id):
    serv = Servicio.objects.get(id_servicio=id)
    serv.estado = 0
    if request.method == 'POST':
        form = ServicioActive(instance=serv)
    else:
        form = ServicioActive(request.POST, instance=serv)
        if form.is_valid():
            form.save()
            messages.success(request, "Servicio desactivado correctamente")
    return redirect(to="lista-servicios")

@staff_member_required
def ServicioActivate(request,id):
    serv = Servicio.objects.get(id_servicio=id)
    if request.method == 'POST':
        form = ServicioActive(instance=serv)
    else:
        form = ServicioActive(request.POST, instance=serv)
        if form.is_valid():
            serv = form.save()
            serv.estado = 1
            serv.save()
            messages.success(request, "Servicio activado correctamente")
    return redirect(to="lista-servicios")

#informacion de clientes ClienteForm
def cliente_datos():
    django_cursor = connection.cursor()
    cursor = django_cursor.connection.cursor() #Este llama
    out_cur = django_cursor.connection.cursor() # este recive
    cursor.callproc("sp_listar_datos_cliente",[out_cur])
    lista =[]
    for fila in out_cur:
        lista.append(fila)
    return lista

@staff_member_required
def infoCliente(request):

    cliente = cliente_datos()
    page = request.GET.get('page', 1)

    try:
        paginator = Paginator(cliente, 5)
        cliente = paginator.page(page)
    except:
        raise Http404
    context = {'entity': cliente,
                'paginator': paginator,
                }
    return render(request, 'administrador/info_cliente/info-cliente.html',context)

#informacion de clientes ProfesionalForm
@staff_member_required
def infoProfesional(request):
    pro = Profesional.objects.all().order_by('id_prof')
    page = request.GET.get('page', 1)
    try:
        paginator = Paginator(pro, 5)
        pro = paginator.page(page)
    except:
        raise Http404
    context = {'entity': pro,
                'paginator': paginator}
    return render(request, 'administrador/info_profesional/info-profesional.html',context)

#informacion de perfiles

@staff_member_required
def infoPerfil(request):

    PerfilF = filtersets.PerfilFilter(request.GET,queryset= Perfil.objects.all())
    page = request.GET.get('page', 1)
    try:
        paginator = Paginator(PerfilF, 5)
        PerfilF = paginator.page(page)
    except:
        """raise Http404"""
    context = {
        'entity':PerfilF,
        'paginator': paginator
    }
    return render(request, 'administrador/info_perfil/info-perfil.html', context)

@staff_member_required
def modificar_perfil(request,id_perfil):
    perfil = Perfil.objects.get(id_perfil=id_perfil)

    if request.method == 'GET':
        form = PerfilModificar(instance=perfil)
    else:
        form = PerfilModificar(request.POST, instance=perfil)
        if form.is_valid():
            form.save()
            messages.success(request,"Perfil de "+perfil.id_auth_user.first_name+" "+perfil.id_auth_user.last_name+" modificado correctamente!")
        else:
            messages.error(request, "Ha ingresado un rut ya registrado, no se han guardado cambios.")
        return redirect(to='infoPerfil')
    context = {
        'form':form
    }
    return render(request, 'administrador/info_perfil/modificar-perfil.html',context)

"""
Utilizando procedures
"""

def lista_actividades():
    django_cursor = connection.cursor()
    cursor = django_cursor.connection.cursor() #Este llama
    out_cur = django_cursor.connection.cursor() # este recive
    cursor.callproc("sp_listar_actividades",[out_cur])
    lista =[]
    for fila in out_cur:
        lista.append(fila)
    return lista

def lista_capacitacion():
    django_cursor = connection.cursor()
    cursor = django_cursor.connection.cursor() #Este llama
    out_cur = django_cursor.connection.cursor() # este recive
    cursor.callproc("sp_listar_capacitacion",[out_cur])
    lista =[]
    for fila in out_cur:
        lista.append(fila)
    return lista

def lista_asesoria():
    django_cursor = connection.cursor()
    cursor = django_cursor.connection.cursor() #Este llama
    out_cur = django_cursor.connection.cursor() # este recive
    cursor.callproc("sp_listar_asesoria",[out_cur])
    lista =[]
    for fila in out_cur:
        lista.append(fila)
    return lista

def lista_visita():
    django_cursor = connection.cursor()
    cursor = django_cursor.connection.cursor() #Este llama
    out_cur = django_cursor.connection.cursor() # este recive
    cursor.callproc("sp_listar_visita",[out_cur])
    lista =[]
    for fila in out_cur:
        lista.append(fila)
    return lista

def lista_cliente():
    django_cursor = connection.cursor()
    cursor = django_cursor.connection.cursor() #Este llama
    out_cur = django_cursor.connection.cursor() # este recive
    cursor.callproc("sp_listar_cliente",[out_cur])
    lista =[]
    for fila in out_cur:
        lista.append(fila)
    return lista
def lista_profesional():
    django_cursor = connection.cursor()
    cursor = django_cursor.connection.cursor() #Este llama
    out_cur = django_cursor.connection.cursor() # este recive
    cursor.callproc("sp_listar_profesional",[out_cur])
    lista =[]
    for fila in out_cur:
        lista.append(fila)
    return lista

def guardar_actividad(nombre,descripcion,tipo_act,fec_estimada,estado,id_cli,id_prof):
    django_cursor = connection.cursor()
    cursor = django_cursor.connection.cursor() #Este llama
    salida = cursor.var(cx_Oracle.NUMBER)
    cursor.callproc('sp_agregar_actividad_corta',[nombre,descripcion,tipo_act,fec_estimada,estado,id_cli,id_prof, salida])
    return salida.getvalue()

# actividades
@staff_member_required
def actividades(request):

    actividad = lista_actividades()
    page = request.GET.get('page', 1)

    try:
        paginator = Paginator(actividad, 5)
        actividad = paginator.page(page)
    except:
        raise Http404
    context = {'entity': actividad,
                'paginator': paginator,
                }
    return render(request, 'administrador/actividades/actividades_lista.html',context)

@staff_member_required
def crear_actividad(request):
    capacitacion = lista_capacitacion()
    asesoria = lista_asesoria()
    visita = lista_visita()
    cliente = Cliente.objects.all()
    profesional = Profesional.objects.all()
    data = {
                'capacitacion':capacitacion,
                'asesoria':asesoria,
                'visita':visita,
                'cliente':cliente,
                'profesional':profesional
                }

    if request.method == 'POST':
        nombre = request.POST.get('nombre')
        descripcion =request.POST.get('descripcion')
        tipo_act = request.POST.get('tipo_act')
        fec_estimada = request.POST.get('fec_estimada')
        estado = request.POST.get('estado')
        id_cli = request.POST.get('id_cli')
        id_prof = request.POST.get('id_prof')
        salida = guardar_actividad(nombre,descripcion,tipo_act,fec_estimada,estado,id_cli,id_prof)
        if salida == 1:
            data['mensaje'] = 'Agregado Correctamente'
            return redirect(to='actividades')
        else:
            data['mensaje'] = 'No se a podido guardar'
    return render(request, 'administrador/actividades/crear.html',data)

@staff_member_required
def actualizar_actividad(request,id_actividad):
    act = Actividad.objects.get(id_actividad=id_actividad)

    if request.method == 'GET':
        form = ActualizarActividad(instance=act)
    else:
        form = ActualizarActividad(request.POST, instance=act)
        if form.is_valid():
            form.save()
            messages.success(request, "Actividad modificada correctamente")
        return redirect(to='actividades')

    context = {'form':form}
    return render(request, 'administrador/actividades/actualizar.html',context)

@staff_member_required
def checklist(request):
    data = {
        'form': listaForm
    }

    if request.method == 'POST':
        formulario = listaForm(data=request.POST)
        if formulario.is_valid():
            formulario.save()
            messages.success(request, "Creado correctamente!")
            return redirect(to='listaCheck')
        else:
            data["form"] = formulario

    return render(request, 'administrador/checklist/checklist.html', data)

@staff_member_required
def listaCheck(request):
    lista = CliCheckPro.objects.all().order_by('id_clicheck')
    page = request.GET.get('page', 1)
    try:
        paginator = Paginator(lista, 5)
        lista = paginator.page(page)
    except:
        raise Http404

    context = {'entity': lista,
                'paginator': paginator}
    return render(request, 'administrador/checklist/listado.html', context)
@staff_member_required
def modificaCheck(request,id_clicheck):
    lista = CliCheckPro.objects.get(id_clicheck=id_clicheck)
    if request.method == 'GET':
        form = listaForm(instance=lista)
    else:
        form = listaForm(request.POST, instance=lista)
        if form.is_valid():
            form.save()
            messages.success(request, "Modificado correctamente")
        return redirect(to='listaCheck')
    return render(request,'administrador/checklist/modificar.html',{'form':form})


"""Added instructions html to make instructions dynamic

Revision ID: 6a1ef6fabfaf
Revises: 1c8b21137307
Create Date: 2017-08-12 01:36:17.185403

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql

# revision identifiers, used by Alembic.
revision = '6a1ef6fabfaf'
down_revision = '1c8b21137307'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('test', sa.Column('instruction_html', mysql.LONGTEXT(), nullable=True))
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('test', 'instruction_html')
    # ### end Alembic commands ###


# Aditya Halder // @AdityaHalder

import os
import aiofiles
import aiohttp
import ffmpeg
import requests
from os import path
from asyncio.queues import QueueEmpty
from typing import Callable
from pyrogram import Client, filters
from pyrogram.types import Message, Voice, InlineKeyboardButton, InlineKeyboardMarkup
from pyrogram.errors import UserAlreadyParticipant
from modules.cache.admins import set
from modules.clientbot import clientbot, queues
from modules.clientbot.clientbot import client as USER
from modules.helpers.admins import get_administrators
from youtube_search import YoutubeSearch
from modules import converter
from modules.downloaders import youtube
from modules.config import que, SUDO_USERS
from modules.cache.admins import admins as a
from modules.helpers.command import commandpro
from modules.helpers.filters import command, other_filters
from modules.helpers.decorators import errors, sudo_users_only
from modules.helpers.errors import DurationLimitError
from modules.helpers.gets import get_url, get_file_name
from pytgcalls import StreamType
from pytgcalls.types.input_stream import InputStream
from pytgcalls.types.input_stream import InputAudioStream

# plus
chat_id = None
useer = "NaN"

def transcode(filename):
    ffmpeg.input(filename).output(
        "input.raw", format="s16le", acodec="pcm_s16le", ac=2, ar="48k"
    ).overwrite_output().run()
    os.remove(filename)

# Convert seconds to mm:ss
def convert_seconds(seconds):
    seconds = seconds % (24 * 3600)
    seconds %= 3600
    minutes = seconds // 60
    seconds %= 60
    return "%02d:%02d" % (minutes, seconds)

# Convert hh:mm:ss to seconds
def time_to_seconds(time):
    stringt = str(time)
    return sum(int(x) * 60 ** i for i, x in enumerate(reversed(stringt.split(":"))))

@Client.on_message(
    commandpro(["ply"])
    & filters.group
    & ~filters.edited
    & ~filters.forwarded
    & ~filters.via_bot
)
@errors
@sudo_users_only
async def play(_, message: Message):
    global que
    global useer
    await message.delete()
    lel = await message.reply("**ðŸ”„ Æ¤É¤Ã¸ÆˆÉ˜ssÉ©É³Ê› ...**")

    administrators = await get_administrators(message.chat)
    chid = message.chat.id

    audio = (
        (message.reply_to_message.audio or message.reply_to_message.voice)
        if message.reply_to_message
        else None
    )
    url = get_url(message)

    if audio:

        file_name = get_file_name(audio)
        title = file_name
        thumb_name = "https://te.legra.ph/file/ed6920a2f0ab5af3fd55d.png"
        thumbnail = thumb_name
        duration = round(audio.duration / 60)
        views = "Locally added"

        requested_by = message.from_user.first_name
        file_path = await converter.convert(
            (await message.reply_to_message.download(file_name))
            if not path.isfile(path.join("downloads", file_name))
            else file_name
        )

    elif url:
        try:
            results = YoutubeSearch(url, max_results=1).to_dict()
            # print results
            title = results[0]["title"]
            duration = results[0]["duration"]
            url_suffix = results[0]["url_suffix"]
            views = results[0]["views"]
            durl = url
            durl = durl.replace("youtube", "youtubepp")

            secmul, dur, dur_arr = 1, 0, duration.split(":")
            for i in range(len(dur_arr) - 1, -1, -1):
                dur += int(dur_arr[i]) * secmul
                secmul *= 60

        except Exception as e:
            title = "NaN"
            thumb_name = "https://te.legra.ph/file/ed6920a2f0ab5af3fd55d.png"
            duration = "NaN"
            views = "NaN"

        requested_by = message.from_user.first_name
        file_path = await converter.convert(youtube.download(url))
    else:
        if len(message.command) < 2:
           return await lel.edit(
                "**ðŸ¤– WÉ¦É‘Êˆ ðŸ™ƒ YÃ¸Ê‹ ðŸ’¿ WÉ‘Å‹Êˆ ðŸ˜�\nðŸ’ž Æ¬Ã¸ ðŸ”Š Æ¤ÉÉ‘yâ�“ ...**"
            )

        await lel.edit("**ðŸ”Ž SÉ˜É‘É¤ÆˆÉ¦É©É³Ê› ...**")
        query = message.text.split(None, 1)[1]
        # print(query)
        await lel.edit("**ðŸ”„ Æ¤É¤Ã¸ÆˆÉ˜ssÉ©É³Ê› ...**")
        try:
            results = YoutubeSearch(query, max_results=1).to_dict()
            url = f"https://youtube.com{results[0]['url_suffix']}"
            # print results
            title = results[0]["title"]
            duration = results[0]["duration"]
            url_suffix = results[0]["url_suffix"]
            views = results[0]["views"]
            durl = url
            durl = durl.replace("youtube", "youtubepp")

            secmul, dur, dur_arr = 1, 0, duration.split(":")
            for i in range(len(dur_arr) - 1, -1, -1):
                dur += int(dur_arr[i]) * secmul
                secmul *= 60

        except Exception as e:
            await lel.edit(
                "**ðŸ”Š â±®Ê‹sÉ©Æˆ ðŸ˜• Æ�Ã¸Êˆ ðŸ“µ FÃ¸Ê‹É³É—â�—ï¸�\nðŸ’ž Æ¬É¤y â™¨ï¸� É…É³Ã¸ÊˆÉ¦É˜É¤ ðŸŒ·...**"
            )
            print(str(e))
            return

        requested_by = message.from_user.first_name
        file_path = await converter.convert(youtube.download(url))
    ACTV_CALLS = []
    chat_id = message.chat.id
    for x in clientbot.pytgcalls.active_calls:
        ACTV_CALLS.append(int(x.chat_id))
    if int(chat_id) in ACTV_CALLS:
        position = await queues.put(chat_id, file=file_path)
        await lel.edit("**ðŸ’¥ Æ˜É‘É‘ÉðŸ¤žÉ…É—É—É˜É— ðŸ’¿ SÃ¸É³Ê›â�—ï¸�\nðŸ”Š É…Êˆ ðŸ’ž Æ¤Ã¸sÉ©ÊˆÉ©Ã¸É³ Â» `{}` ðŸŒ· ...**".format(position),
    )
    else:
        await clientbot.pytgcalls.join_group_call(
                chat_id,
                InputStream(
                    InputAudioStream(
                        file_path,
                    ),
                ),
                stream_type=StreamType().local_stream,
            )

        await lel.edit("**ðŸ’¥ Æ˜É‘É‘ÉðŸ¤žMÊ‹sÉ©Æˆ ðŸŽ¸ NÃ¸Ï‰ ðŸ’ž\nðŸ”Š Æ¤ÉÉ‘yÉ©É³Ê› ðŸ˜� Ã˜Æ¤ ðŸ¥€ ...**".format(),
        )

    return await lel.delete()

@Client.on_message(commandpro(["pse"]) & other_filters)
@errors
@sudo_users_only
async def pause(_, message: Message):
    await message.delete()
    await clientbot.pytgcalls.pause_stream(message.chat.id)
    pase = await message.reply_text("**â–¶ï¸� Æ¤É‘Ê‹sÉ˜É— ðŸŒ· ...**")
    await pase.delete()

@Client.on_message(commandpro(["rsm"]) & other_filters)
@errors
@sudo_users_only
async def resume(_, message: Message):
    await message.delete()
    await clientbot.pytgcalls.resume_stream(message.chat.id)
    rsum = await message.reply_text("**â�¸ Æ¦É˜sÊ‹É±É˜É— ðŸŒ· ...**")
    await rsum.delete()

@Client.on_message(commandpro(["skp", "nxt"]) & other_filters)
@errors
@sudo_users_only
async def skip(_, message: Message):
    global que
    await message.delete()
    ACTV_CALLS = []
    chat_id = message.chat.id
    for x in clientbot.pytgcalls.active_calls:
        ACTV_CALLS.append(int(x.chat_id))
    if int(chat_id) not in ACTV_CALLS:
       novc = await message.reply_text("**ðŸ’¥ Æ�Ã¸ÊˆÉ¦É©É³Ê› ðŸ”‡ Æ¤ÉÉ‘yÉ©É³Ê› ðŸŒ· ...**")
       await novc.delete()
    else:
        queues.task_done(chat_id)

        if queues.is_empty(chat_id):
            empt = await message.reply_text("**ðŸ¥€ Empty Queue, Leaving VC âœ¨ ...**")
            await empt.delete()
            await clientbot.pytgcalls.leave_group_call(chat_id)
        else:
            next = await message.reply_text("**â�© SÆ™É©Æ¥Æ¥É˜É— ðŸŒ· ...**")
            await next.delete()
            await clientbot.pytgcalls.change_stream(
                chat_id,
                InputStream(
                    InputAudioStream(
                        clientbot.queues.get(chat_id)["file"],
                    ),
                ),
            )

@Client.on_message(commandpro(["end", "stp"]) & other_filters)
@errors
@sudo_users_only
async def stop(_, message: Message):
    await message.delete()
    try:
        clientbot.queues.clear(message.chat.id)
    except QueueEmpty:
        pass

    await clientbot.pytgcalls.leave_group_call(message.chat.id)
    leav = await message.reply_text("**â�Œ SÊˆÃ¸Æ¥Æ¥É˜É— ðŸŒ· ...**")
    await leav.delete()


from modules import s3upload, s3transcribe, parse
import argparse

if __name__ == "__main__":
    # Create Argument Parser
    parser = argparse.ArgumentParser(description='Process video, create transcripts, proofread with OpenAI GPT.')
    parser.add_argument('input_folder', type=str, help='Input folder with .mp4 interview video(s)')
    parser.add_argument('s3_folder', type=str, help='Output folder name to save files to in S3 bucket')
    args = parser.parse_args()

    # Step 1: Upload videos to S3
    print("Step 1: Uploading videos to S3...")
    s3upload.upload_to_s3(args.input_folder, args.s3_folder)

    # Step 2: Transcribe videos from S3 and download the transcriptions
    print("Step 2: Transcribing videos from S3 and downloading the transcriptions...")
    transcribe_folder = s3transcribe.transcribe_from_s3(args.s3_folder)
    s3transcribe.download_transcripts(transcribe_folder)

    # Step 3: Parse transcriptions
    print("Step 3: Parsing and proofreading transcriptions...")
    parse.proofread_transcripts(transcribe_folder)

    print("Finished processing videos! View the resulting transcript and .docx file in the timestamped folder.")


import json
import re
import os
from bs4 import BeautifulSoup
import io
import html2text
#import transformationScript
import datetime

#from pprint import pprint

class Word:
    content = ""
    tag = ""

    # The class "constructor" - It's actually an initializer
    def __init__(self, content, tag):
        self.content = content
        self.tag = tag

class Component:
    distance = 20
    startPosition = -1
    position = -1

    def setDistance(self, distance):
        self.distance = distance
        self.startPosition = -1
        self.position = -1

class Premise(Component):
    words = []

    # The class "constructor" - It's actually an initializer
    def __init__(self, words):
        self.words = words

    def getText(self):
        words = []
        for word in self.words:
            words.append(word.content)
        return ' '.join(words)

    def getTags(self):
        words = []
        for word in self.words:
            words.append(word.tag)
        return ' '.join(words)

class Claim(Component):
    words = []

    # The class "constructor" - It's actually an initializer
    def __init__(self, words):
        self.words = words

    def getText(self):
        words = []
        for word in self.words:
            words.append(word.content)
        return ' '.join(words)

    def getTags(self):
        words = []
        for word in self.words:
            words.append(word.tag)
        return ' '.join(words)

class Translator:

    contents = {}

    def __init__(self):
        self.contents = {}

    def addPair(self, htmlFile, jsonfile):
        self.contents[jsonfile] = htmlFile

    def createAssociation(self, nodeSet):
        fileName = "corpusInput/json/" + nodeSet
        file = open(fileName, "r")
        contents = file.read()
        elements = json.loads(contents)

        for node in elements['nodes']:
            if 'http' in node['text']:
                link = re.search("(?P<url>https?://[^\s]+)", node['text']).group("url")
                link = re.sub('http://web.fe.up.pt/~ei11124/argmine_news/', '', link)
                link = link[:-1]
                self.addPair(link,nodeSet)
                break

    def createAssociations(self):
        fileList = os.listdir("corpusInput/json")
        for file in fileList:
            self.createAssociation(file)

class TextDumper:
    _nArgTag = "(O,|)"

    words = []
    file = ""

    def __init__(self, htmlFile):
        self.file = "corpusInput/html/" + htmlFile + '.html'
        self.words = []

    def getText(self):
        words = []
        for word in self.words:
            words.append(word.content)
        return ' '.join(words)

    def stripHtml(self):
        with io.open(self.file, 'r', encoding='utf8') as f:
            contents = f.read()
        plainText = html2text.html2text(contents)
        sentences = plainText.split('\n')

        maxSize = sentenceNumber = chosen = 0
        for sentence in sentences:
            size = len(sentence)
            if(size > maxSize):
                chosen = sentenceNumber
            sentenceNumber += 1

        sentences[chosen] = re.sub(r'[.]+(?![0-9])', r' .', sentences[chosen])
        sentences[chosen] = re.sub(r'[:]+(?![0-9])', r' :', sentences[chosen])
        sentences[chosen] = re.sub(r'[,]+(?![0-9])', r' ,', sentences[chosen])
        sentences[chosen] = re.sub(r'[;]+(?![0-9])', r' ;', sentences[chosen])
        sentences[chosen] = re.sub(r'[?]+(?![0-9])', r' ?', sentences[chosen])
        sentences[chosen] = re.sub(r'[!]+(?![0-9])', r' !', sentences[chosen])
        sentences[chosen] = re.sub(r'[…]+(?![0-9])', r' …', sentences[chosen])
        sentences[chosen] = re.sub(r'[“]+(?![0-9])', r'', sentences[chosen])
        sentences[chosen] = re.sub(r'[”]+(?![0-9])', r'', sentences[chosen])
        sentences[chosen] = re.sub(r'["]+(?![0-9])', r'', sentences[chosen])
        sentences[chosen] = re.sub(r'[‘]+(?![0-9])', r'', sentences[chosen])
        sentences[chosen] = re.sub(r'[’]+(?![0-9])', r'', sentences[chosen])
        sentences[chosen] = re.sub(r'[(]+(?![0-9])', r'', sentences[chosen])
        sentences[chosen] = re.sub(r'[)]+(?![0-9])', r'', sentences[chosen])
        sentences[chosen] = re.sub(r'[\']+(?![0-9])', r'', sentences[chosen])
        sentences[chosen] = re.sub(r'[`]+(?![0-9])', r'', sentences[chosen])
        sentences[chosen] = re.sub(r'[`]+(?![0-9])', r'', sentences[chosen])
        sentences[chosen] = re.sub(r'[[]+(?![0-9])', r'', sentences[chosen])
        sentences[chosen] = re.sub(r'[]]+(?![0-9])', r'', sentences[chosen])
        sentences[chosen] = re.sub(r'[«]+(?![0-9])', r'', sentences[chosen])
        sentences[chosen] = re.sub(r'[»]+(?![0-9])', r'', sentences[chosen])
        sentences[chosen] = re.sub(r'[**]+(?![0-9])', r'', sentences[chosen])

        print(sentences[chosen])
        return sentences[chosen]

    def wordifyText(self):
        text = self.stripHtml()
        originalWords = text.split(' ')
        for word in originalWords:
            '''if(word == '.'):
                taggedWord = Word(word, '.')
                self.words.append(taggedWord)
            elif(word != ''):
                taggedWord = Word(word, self._nArgTag)
                self.words.append(taggedWord)'''
            if (word != ''):
                taggedWord = Word(word, self._nArgTag)
                self.words.append(taggedWord)

class claimsAndPremises:
    claims = []
    premises = []
    premisesToClaims = {}
    file = ""

    def __init__(self, jsonFile):
        self.file = "corpusInput/json/" + jsonFile + '.json'
        self.claims = []
        self.premises = []
        self.premisesToClaims = {}

    def removeHttp(self, elements):
        for node in elements['nodes']:
            if 'http' in node['text']:
                elements['nodes'].remove(node)

        return elements

    def removeInferences(self, elements):
        for node in elements['nodes']:
            if 'Default Inference' in node['text']:
                elements['nodes'].remove(node)

        return elements

    def collapseEdges(self, edges, nodes):
        collapsedEdges = []
        for originEdge in edges:
            for destinationEdge in edges:
                if ((destinationEdge['fromID'] == originEdge['toID']) & (self.getNodeText(nodes,originEdge['fromID']) != self.getNodeText(nodes,destinationEdge['toID']))):
                    edge = {originEdge['fromID']:destinationEdge['toID']}
                    collapsedEdges.append(edge)
                    #collapsedEdges[originEdge['fromID']] = destinationEdge['toID']

        #print(collapsedEdges)
        return collapsedEdges

    def getNodeText(self, nodes, nodeId):
        nodeText = ''
        for node in nodes:
            if (node['nodeID'] == nodeId):
                nodeText = node['text']
                nodeText = re.sub(r'[.]+(?![0-9])', r' .', nodeText)
                nodeText = re.sub(r'[:]+(?![0-9])', r' :', nodeText)
                nodeText = re.sub(r'[,]+(?![0-9])', r' ,', nodeText)
                nodeText = re.sub(r'[;]+(?![0-9])', r' ;', nodeText)
                nodeText = re.sub(r'[?]+(?![0-9])', r' ?', nodeText)
                nodeText = re.sub(r'[!]+(?![0-9])', r' !', nodeText)
                nodeText = re.sub(r'[…]+(?![0-9])', r' …', nodeText)
                nodeText = re.sub(r'[“]+(?![0-9])', r'', nodeText)
                nodeText = re.sub(r'[”]+(?![0-9])', r'', nodeText)
                nodeText = re.sub(r'["]+(?![0-9])', r'', nodeText)
                nodeText = re.sub(r'[‘]+(?![0-9])', r'', nodeText)
                nodeText = re.sub(r'[’]+(?![0-9])', r'', nodeText)
                nodeText = re.sub(r'[(]+(?![0-9])', r'', nodeText)
                nodeText = re.sub(r'[)]+(?![0-9])', r'', nodeText)
                nodeText = re.sub(r'[\']+(?![0-9])', r'', nodeText)
                nodeText = re.sub(r'[`]+(?![0-9])', r'', nodeText)
                nodeText = re.sub(r'[`]+(?![0-9])', r'', nodeText)
                nodeText = re.sub(r'[[]+(?![0-9])', r'', nodeText)
                nodeText = re.sub(r'[]]+(?![0-9])', r'', nodeText)
                nodeText = re.sub(r'[«]+(?![0-9])', r'', nodeText)
                nodeText = re.sub(r'[»]+(?![0-9])', r'', nodeText)
                nodeText = re.sub(r'[**]+(?![0-9])', r'', nodeText)

        return nodeText

    def tagClaimOrPremise(self, words, type):
        if (type == 'premise'):
            distance = '1'
        else:
            distance = '|'

        taggedSentence = []
        for wordIndex in range(0, len(words)):
            word = words[wordIndex]
            if (wordIndex == 0):
                #tag = '(B,' + type + ',' + distance + ')'
                #tag = '(B,' + type + ')'
                tag = '(I,' + type + ')'
            elif ((word == '.') or (word == ':') or (word == ';') or (word == '?') or (word == '!')):
                #tag = '(O,|,|)'
                tag = '(O,|)'
                #tag = '.'
            else:
                #tag = '(I,' + type + ',' + distance + ')'
                tag = '(I,' + type + ')'
            taggedWord = Word(word,tag)
            taggedSentence.append(taggedWord)
        return taggedSentence

    def isIntermediatePremise(self, claim, connections):
        isIntermediate = False
        for connection in connections:
            if next(iter(connection)) == claim:
                isIntermediate = True

        return isIntermediate

    def getPremisesAndClaims(self):
        file = open(self.file, "r")
        contents = file.read()
        elements = self.removeHttp(json.loads(contents))
        #elements = self.removeInferences(elements)

        connections = self.collapseEdges(elements['edges'], elements['nodes'])
        #print(self.file)
        #print(connections)
        nodes = elements['nodes']

        for connection in connections:
            claim = self.getNodeText(nodes, connection[next(iter(connection))])
            claimWords = claim.split()
            taggedClaim = Claim(self.tagClaimOrPremise(claimWords, 'claim'))
            self.claims.append(taggedClaim)

            premise = self.getNodeText(nodes, next(iter(connection)))
            premiseWords = premise.split()
            taggedPremise = Premise(self.tagClaimOrPremise(premiseWords, 'premise'))
            self.premises.append(taggedPremise)
            #print(taggedPremise.getText())

            self.premisesToClaims[premise] = claim

class claimsReplacer:
    processedText = []
    originalText = []
    existingClaimsAndPremises = []

    def __init__(self, originalText, existingClaimsAndPremises):
        self.originalText = originalText
        self.existingClaimsAndPremises = existingClaimsAndPremises
        self.processedText = originalText

    def getOriginalText(self):
        words = []
        for word in self.originalText:
            words.append(word.content)
        return ' '.join(words)

    def getProcessedText(self):
        words = []
        for word in self.processedText:
            words.append(word.content)
        return ' '.join(words)

    def getTags(self):
        tags = []
        for word in self.processedText:
            tags.append(word.tag)
        return ' '.join(tags)

    def matchText(self, wordPosition, component, textSize):
        #print(textSize)
        isMatch = True
        for word in component.words:
            #print(word.content)
            if ((wordPosition >= textSize) or
                    (word.content.lower() != self.originalText[wordPosition].content.lower())):
                isMatch = False
                break
            wordPosition += 1

        return isMatch

    def replaceText(self, wordPosition, component):
        for word in component.words:
            self.processedText[wordPosition] = word
            wordPosition += 1

    def processText(self):
        #print(self.getOriginalText())
        for claim in self.existingClaimsAndPremises.claims:
            wordPosition = 0
            for word in self.originalText:
                if (claim.words[0].content.lower() == word.content.lower()):
                    if(self.matchText(wordPosition, claim, len(self.originalText))):
                        self.replaceText(wordPosition, claim)
                        claim.startPosition = wordPosition
                wordPosition += 1

        for premise in self.existingClaimsAndPremises.premises:
            wordPosition = 0
            for word in self.originalText:
                if (premise.words[0].content.lower() == word.content.lower()):
                    if(self.matchText(wordPosition, premise, len(self.originalText))):
                        self.replaceText(wordPosition, premise)
                        premise.startPosition = wordPosition
                wordPosition += 1

class DistanceCalculator:
    processedText = []
    claimsAndPremises = []

    def __init__(self, processedText, existingClaimsAndPremises):
        self.processedText = processedText
        self.claimsAndPremises = existingClaimsAndPremises

    def getKey(self, component):
        return component.startPosition

    def returnUniqueComponents(self, components):
        index = 0
        uniqueComponents = []
        nonUniqueComponents = []
        unique = True
        for component in components:
            for secondComponent in uniqueComponents:
                if component.getText() == secondComponent.getText():
                    unique = False
                    nonUniqueComponents.append(component)
            if (unique):
                #print(component.getText())
                uniqueComponents.append(component)
            index += 1

        return uniqueComponents

    def arrangeComponents(self):
        #claims = self.returnUniqueComponents(self.claimsAndPremises.claims)
        #premises = self.returnUniqueComponents(self.claimsAndPremises.premises)

        '''for claim in claims:
            print("claim - " + claim.getText())
        for premise in premises:
            print("premise - " + premise.getText())'''
        components = self.claimsAndPremises.claims + self.claimsAndPremises.premises
        components = self.returnUniqueComponents(components)
        components = sorted(components, key=self.getKey)

        position = 1
        for component in components:
            #print("component - " + component.getText())
            #print (component.startPosition)
            for claim in self.claimsAndPremises.claims:
                if claim.startPosition == component.startPosition:
                    claim.position = position
                    #print("premise " + premise.getText())

            for premise in self.claimsAndPremises.premises:
                if premise.startPosition == component.startPosition:
                    premise.position = position
                    #print(premise.position)
                    #print("premise " + premise.getText())

            position += 1

    def calculateDistances(self):
        index = 0
        for premise in self.claimsAndPremises.premises:
            distance = self.claimsAndPremises.claims[index].position - self.claimsAndPremises.premises[index].position
            premise.distance = distance
            #print(distance)
            index += 1

    def updatePremises(self):
        for premise in self.claimsAndPremises.premises:
            for word in premise.words:
                tag = list(word.tag)
                if tag[1] != 'O':
                    tag[len(tag)-2] = str(premise.distance)
                    tag = "".join(tag)
                    word.tag = tag

class OutputWriter:
    processedText = []
    textFile = ""
    tagFile = ""
    file = ""

    def __init__(self, processedText, file):
        self.processedText = processedText
        self.file = open("corpusOutputPunctuation/txt/" + file + '.txt', "w", encoding='utf-8')
        #self.textFile = open("corpusOutput/txt/textsWithSentences/" + file + '.txt', "w", encoding='utf-8')
        self.textFile = open("corpusOutputPunctuation/txt/texts/" + file + '.txt', "w", encoding='utf-8')
        self.tagFile = open("corpusOutputPunctuation/txt/tags/" + file + '.txt', "w", encoding='utf-8')

    def writeToTextFile(self):
        for word in self.processedText:
            content = word.content
            tag = word.tag
            self.textFile.write(u'' + content + '\n')
            self.tagFile.write(u'' + tag + '\n')
            self.file.write(u'' + content + '' + tag + '\n')

class Pipeline:

    def translate(self):
        translator = Translator()
        translator.createAssociations()
        files = translator.contents

        startTime = datetime.datetime.now().replace(microsecond=0)
        for (jsonFile, htmlFile) in files.items():
            htmlFile = re.sub('.html', '', htmlFile)
            jsonFile = re.sub('.json', '', jsonFile)

            dumper = TextDumper(htmlFile)
            dumper.wordifyText()

            claims = claimsAndPremises(jsonFile)
            claims.getPremisesAndClaims()

            replacer = claimsReplacer(dumper.words, claims)
            replacer.processText()

            #distanceCalculator = DistanceCalculator(replacer.processedText, replacer.existingClaimsAndPremises)
            #distanceCalculator.arrangeComponents()
            #distanceCalculator.calculateDistances()
            #distanceCalculator.updatePremises()

            #replacer = claimsReplacer(dumper.words, distanceCalculator.claimsAndPremises)
            #replacer.processText()

            output = OutputWriter(replacer.processedText, jsonFile)
            output.writeToTextFile()

        endTime = datetime.datetime.now().replace(microsecond=0)
        timeTaken = endTime - startTime

        print("Isto demorou ")
        print(timeTaken)

pipeline = Pipeline()
pipeline.translate()


#Pasos a seguir:
#1) Installar el repostorio de proyecto EELabs, con .env  y credentials.json en \resources_folder\google, y el entorno conda
#2) Pegar este script en dentro del repositorio \
#3) Ejecutar desde su ubicacion
#IMPORTANTE no permite realizar actualizaciones de fecha, está pensado para una descarga única no para realizar descargas actualizadas
import select
from utils.devices_api.eelabs_devices_api import EELabsDevicesApi
from utils.devices.tess import TESS
from utils.devices.skyglow import SkyGlow
from utils.devices.sqm import SQM
from utils.devices.astmon import ASTMON
import pandas as pd
import numpy as np
from datetime import datetime
from utils.my_utils import Utils

import config as conf #Configuration variables
from utils.filter import Filter
from datetime import date

import argparse
import os

#Input of the script
parser = argparse.ArgumentParser()
parser.add_argument('--f', '--from',  type=int, help='Year from')
parser.add_argument('--to',  type=int, help='Year to')
parser.add_argument('--out','--output', required=True, type=str, help='Output filename')
parser.add_argument('--filter', type=str, help='Data filtering: sun, moon, clouds, galaxy, zodiacal, sigma. Format example: [sun,galaxy,zodaical] Write all without brackets for all filters. Format example: all')
parser.add_argument('--device',type=str,help='Format exalmple: [LPL1_001,LPL2_033,stars1] rite all without brackets for ones device. Format example: stars1')
parser.add_argument('--ephemeris',type=bool,help='True for ephemeris included')
args = parser.parse_args()

initial_year = args.f
final_year = args.to
output = args.out
filter = args.filter
ephemerides = args.ephemeris
if filter:
    if filter[0]=='[':
        filter=filter[1:-1].split(',')
    else:
        filter=filter
else:
    filter=[]
select_devices=args.device
if select_devices:
    if select_devices[0]=='[':
        select_devices=set(select_devices[1:-1].split(','))
    else:
        select_devices=set([select_devices])

#Create the save folder
output=output+'\Photometer_data'
if not os.path.exists(output):
    os.mkdir(output)

#Photometers dataset
devices=EELabsDevicesApi().get_all_devices_info()
devices=pd.DataFrame(devices)
devices=devices.drop(['sg_type','lpl','zero_point','filters','mov_sta_position','local_timezone','location','info_img','info_tess','place','tester','info_org','__v','latitude','longitude','country','city'],axis=1)
localizacion=pd.DataFrame(list(devices['info_location'])).drop(['latitude_hide','longitude_hide'],axis=1)
devices['place']=localizacion['place']
devices['town']=localizacion['town']
devices['sub_region']=localizacion['sub_region']
devices['region']=localizacion['region']
devices['country']=localizacion['country']
devices['latitude']=localizacion['latitude']
devices['longitude']=localizacion['longitude']
devices['elevation']=localizacion['elevation']
devices=devices.drop(['info_location'],axis=1)
devices.to_csv(output+'\All_devices.csv', index = False)

#Folder to save records
if not os.path.exists(output+'\Records'):
    os.mkdir(output+'\Records')

#Obtain the device class regardless of its type
def Device(device_name):
    devices=pd.read_csv(output+'\All_devices.csv')
    type=devices[devices['name']==device_name]['TYPE'].values[0]
    if type==TESS.TYPE:
        device_obj=TESS(name=device_name)
    elif type==SkyGlow.TYPE:
        device_obj=SkyGlow(name=device_name)
    elif type==SQM.TYPE:
        device_obj=SQM(name=device_name)
    elif type==ASTMON.TYPE:
        device_obj=ASTMON(name=device_name)
    return device_obj
#Obtain filtered data for a device and year
def Data(device_name,year,filter): #filter: data vector such as ['sun', 'moon'] for example.
    device_obj=Device(device_name)
    FIRST_DATE=pd.Timestamp(datetime(year, 1, 1, 0, 0), tz='UTC')
    LAST_DATE=pd.Timestamp(datetime(year+1, 1, 1, 0, 0), tz='UTC')
    df_all=None
    try:
        df_all = device_obj.get_all_data(date_from=FIRST_DATE, date_to=LAST_DATE,force=False)
        No_data=False
    except:
        df_all=None
        No_data=True
    if No_data:
        print('The device '+device_name+' not responded due to an error')
    df_all=df_all[(df_all['mag']>conf.MAG_MIN) & (df_all['mag']<conf.MAG_MAX)] #Filter for extreme magnitudes
    if __name__ == '__main__':
        df_all = Utils().add_ephems(df_all, device_obj.getObserver(), parallelize=False) # The parallelize option is causing issues
    V=[]
    if 'sun' in filter or filter=='all':
        df_all = Filter().filter_sun(df_all, max_sun_alt=conf.SUN_ALT_MAX)
    else:
        df_filter=Filter().filter_sun(df_all, max_sun_alt=conf.SUN_ALT_MAX)
        F=np.array([True]*(df_all.index[-1]+1)) #Vector with all True for all indices
        F[df_filter.index]=False #Replace remaining indices to False after filtering
        df_all['sun']=F[df_all.index] #Retrieve data according to the original index
        V=V+['sun']
    if 'moon' in filter or filter=='all':
        df_all = Filter().filter_moon(df_all, max_moon_alt=conf.MOON_ALT_MAX)
    else:
        df_filter=Filter().filter_moon(df_all, max_moon_alt=conf.MOON_ALT_MAX)
        F=np.array([True]*(df_all.index[-1]+1))
        F[df_filter.index]=False
        df_all['moon']=F[df_all.index]
        V=V+['moon']
    if 'clouds' in filter or filter=='all':
        clouds_threshold=conf.CLOUD_STD_FREQ
        df_all = Filter().filter_column(df_all, device_obj.getMagSTDColname(), max=clouds_threshold)
    else:
        clouds_threshold=conf.CLOUD_STD_FREQ
        df_filter=Filter().filter_column(df_all, device_obj.getMagSTDColname(), max=clouds_threshold)
        F=np.array([True]*(df_all.index[-1]+1))
        F[df_filter.index]=False
        df_all['clouds']=F[df_all.index]
        V=V+['clouds']
    if 'galaxy' in filter or filter=='all':
        df_all = Filter().filter_galactic_abs_lat(df_all, min_lat=conf.GALACTIC_LAT_MIN, max_lat=180)
    else:
        df_filter=Filter().filter_galactic_abs_lat(df_all, min_lat=conf.GALACTIC_LAT_MIN, max_lat=180)
        F=np.array([True]*(df_all.index[-1]+1))
        F[df_filter.index]=False
        df_all['galaxy']=F[df_all.index]
        V=V+['galaxy']
    if 'zodiacal' in filter or filter=='all':
        df_all = Filter().filter_column(df_all, col_name='ecliptic_f', max=conf.ECLIPTIC_F_MAX)
    else:
        df_filter=Filter().filter_column(df_all, col_name='ecliptic_f', max=conf.ECLIPTIC_F_MAX)
        F=np.array([True]*(df_all.index[-1]+1))
        F[df_filter.index]=False
        df_all['zodiacal']=F[df_all.index]
        V=V+['zodiacal']
    if 'sigma' in filter or filter=='all':
        sigma=conf.NSIGMA
        df_all = Filter().filter_nsigma(df_all, col_name='mag', sigma=sigma)
    else:
        sigma=conf.NSIGMA
        df_filter=Filter().filter_nsigma(df_all, col_name='mag', sigma=sigma)
        F=np.array([True]*(df_all.index[-1]+1))
        F[df_filter.index]=False
        df_all['sigma']=F[df_all.index]
        V=V+['sigma']
    if ephemerides:
        df=pd.DataFrame({'time':df_all['time'],'mag':df_all['mag'],'name':device_name,'moon_phase':df_all['moon_phase'],'moon_alt':df_all['moon_alt'],'galactic_lat':df_all['galactic_lat'],'galactic_lon':df_all['galactic_lon'],'helioecliptic_lon_abs':df_all['helioecliptic_lon_abs'],'ecliptic_lat_abs':df_all['ecliptic_lat_abs']})
    else:
        df=pd.DataFrame({'time':df_all['time'],'mag':df_all['mag'],'name':device_name})
    for ii in V:
        df[ii]=df_all[ii]
    return df
#Obtain all data between two years
def Data_download(V,initial_year=None,final_year=None,iterate=True): #Iterate to prompt for enter key per iteration
    #Downloaded devices
    Downloaded_devices=set()
    for j in range(0,1000):
        try:
            df_records=pd.read_csv(output+'\Records\Records_'+str(j)+'.csv')
            Downloaded_devices=Downloaded_devices|set(df_records['name'])
        except:
            Downloaded_devices=Downloaded_devices

    #Log devices
    try:
        df_log=pd.read_csv(output+'\Log.csv')
        Log_devices=set(df_log['Devices'])
        Log_exists=True
    except:
        Log_devices=set()
        Log_exists=False
    diff=Downloaded_devices-Log_devices
    Downloaded_devices=Downloaded_devices|Log_devices
    print(Downloaded_devices)
    df_all_devices=pd.read_csv(output+'\All_devices.csv')
    All_devices=set(df_all_devices['name'])
    if select_devices:
        Missing_devices=select_devices-Downloaded_devices
    else:
        Missing_devices=All_devices-Downloaded_devices #To know which devices need to be download
    n_missing_devices=len(Missing_devices)
    if initial_year:
        i_year=initial_year
    else:
        i_year=2010
    if final_year:
        f_year=final_year
    else:
        f_year=date.today().year

    Downloaded_missing_devices=[]
    v_empty=[]
    v_time=[]
    #Loop where it goes device by device and then year by year
    for i in Missing_devices:
        df=pd.DataFrame()
        empty=True
        for ii in range(i_year,f_year+1):
            try:
                dat=Data(i,ii,V)
                df=pd.concat([df,dat])
                if list(dat.values)!=[]:
                    empty=False
            except:
                df=df
            print('Year: '+str(ii))
        #Save
        #Saving with files limited to 1 GB
        try:
            df_records=pd.read_csv(output+'\Records\Records_1.csv')
            Records_exist=True
        except:
            df_final=df
            df_final.to_csv(output+'\Records\Records_1.csv', index = False)
            Records_exist=False
        if Records_exist==True:
            counter=0
            for j in range(1,1000):
                try:
                    df_records=pd.read_csv(output+'\Records\Records_'+str(j)+'.csv')
                    if os.stat(output+'\Records\Records_'+str(j)+'.csv').st_size<1000000000:
                        df_final=pd.concat([df_records,df])
                        df_final.to_csv(output+'\Records\Records_'+str(j)+'.csv', index = False)
                        counter=1
                except:
                    if counter==0:
                        df_final=df
                        df_final.to_csv(output+'\Records\Records_'+str(j)+'.csv', index = False)
                        counter=1

        time=datetime.now()
        v_empty=v_empty+[empty]
        v_time=v_time+[time]
        Downloaded_missing_devices=Downloaded_missing_devices+[i]

        Log_downloaded_devices=pd.DataFrame({'Devices':Downloaded_missing_devices,'Time':v_time,'Empty':v_empty})
        Log_downloaded_devices_2=pd.DataFrame({'Devices':list(diff),'Time':None,'Empty':False})
        Log=pd.concat([Log_downloaded_devices_2,Log_downloaded_devices])
        #Save log
        if Log_exists:
            Log_2=pd.concat([df_log,Log])
        else:
            Log_2=Log
        Log_2.to_csv(output+'\Log.csv', index = False)
        n_no_downloaded_missing_devices=n_missing_devices-len(Downloaded_missing_devices)
        print(str(n_no_downloaded_missing_devices)+' are still pending for download')
        if iterate:
            if input('Downloaded device:'+i+'\n')=='exit':
                break
        else:
            print('Downloaded device:'+i+'\n')
#Run
Data_download(filter,initial_year,final_year,iterate=False)


#!/usr/bin/python3

import os
import argparse
from subprocess import call

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('basedir', type=str, help='The base directory to walk from')
    args = parser.parse_args()
    print('The base dir is: {}'.format(args.basedir))
    for dirname, subdirs, filenames in os.walk(args.basedir):
        for filename in filenames:
            full_path = os.path.join(dirname, filename)
            if filename.endswith('.bz2'):
                print("Decompressing {}".format(full_path))
                call(["bzip2", "-d", full_path])
            else:
                print("Ignoring {}".format(full_path))


import unittest
import datetime
import json

from app.textsapi.models.submission import Submission
from app.textsapi.models.text import Text
from app.tests.base import BaseTestCase

def register_ok_submission(self, token):
    return self.client.post(
        '/submission/',
        headers=dict(
            Authorization="Token {}".format(token)
        ),
        data=json.dumps(dict(
            submitted_texts=['text1', 'text2']
        )),
        content_type='application/json'
    )

def register_illegal_submission(self, token):
    return self.client.post(
        '/submission/',
        headers=dict(
            Authorization="Token {}".format(token)
        ),
        data=json.dumps(dict(
            submitted_texts=[1, 'text2']
    )),
        content_type='application/json'
    )

def get_submissions(self, token):
    return self.client.get(
        '/submission/',
        headers=dict(
            Authorization="Token {}".format(token)
        )
    )

class TestSubmission(BaseTestCase):

    def test_create_valid_submission(self):
        """ Test for creating a valid submission """
        with self.client:
            # valid submission registration
            sub_response = register_ok_submission(self, self.token)
            response_data = json.loads(sub_response.data.decode())
            self.assertTrue(response_data['status']=='success')

    def test_create_invalid_submission(self):
        """ Test for creating an invalid submission """
        with self.client:
            # invalid submission registration
            sub_response = register_illegal_submission(self, self.token)
            response_data = json.loads(sub_response.data.decode())
            self.assertTrue(response_data['errors']!=None)

    def test_update_submission(self):
        """ Test for updating a submission """
        sub_response_register = register_ok_submission(self, self.token)
        response_data = json.loads(sub_response_register.data.decode())
        self.assertTrue(response_data['status']=='success')

        sub = [sub for sub in Submission.query(hash_key=self.new_user.username, range_key_condition=Submission.sort.startswith('SUBMISSION_'))][0]
        sub_response_update = self.client.put(
            '/submission/{}'.format(str(sub.public_id)),
            headers=dict(
            Authorization="Token {}".format(self.token)
            ),
            data=json.dumps(dict(
            submitted_texts=['updated_text1']
            )),
            content_type='application/json'
        )
        update_data = json.loads(sub_response_update.data.decode())
        upd_sub = Submission.get(hash_key=sub.username, range_key=sub.sort)
        self.assertTrue(update_data['status']=='success')
        self.assertTrue(upd_sub.text_count == 1)

    def test_get_submission(self):
        """ Test getting the submissions from database """
        # creating a submission
        sub_register = register_ok_submission(self, self.token)
        response_data = json.loads(sub_register.data.decode())
        self.assertTrue(response_data['status']=='success')

        # getting it from the service
        get_response = get_submissions(self, self.token)
        response_data = json.loads(get_response.data.decode())
        self.assertTrue(response_data['data'][0]['text_count']==2)
        self.assertTrue(isinstance(response_data['data'][0]['texts'], list))

if __name__ == '__main__':
    unittest.main()


"""
Table of Contents

1. drop_null_columns: Drop columns that exceed a threshold of null values.
"""

from pyspark.sql import functions as F, DataFrame

from ..parsing.melting import melt

def drop_null_columns(df: DataFrame, threshold: float = 1.0, subset: list = None) -> DataFrame:
    """
    Drop columns that exceed a threshold of null values.

    Inputs
        df: DataFrame.
        threshold: Threshold value. If a column has at least this fraction of nulls, it will be removed.
        subset: List of columns to check. All others will be kept by default regardless of the null count.

    Output
        Updated DataFrame.

    Example
        df = df.drop_null_columns(df, 0.5, ["col1", "col2"])
        Will remove col1 or col2 if either is at least 50% null.
    """
    # If looking only at a subset of columns, downselect, otherwise, evaluate all columns.
    if subset is None:
        temp_df = df
    else:
        temp_df = df.select(subset)

    # List of columns being evaluated.
    columns_evaluated = temp_df.columns

    # Replace each column with a 1 if null, 0 otherwise.
    for col in columns_evaluated:
        temp_df = temp_df.withColumn(col, F.when(F.col(col).isNull(), 1).otherwise(0))

    # Sum the number of nulls, represented with a 1, for each column.
    null_count = temp_df.agg(*[F.sum(c).alias(c) for c in columns_evaluated])

    # Total row count.
    count = temp_df.count()

    # Pivot this null_count DataFrame to do a few operations to find when there are too many nulls.
    # The names of the columns will now be listed in a column called "categories".
    null_count = melt(null_count, value_vars=columns_evaluated)
    null_count = (
        null_count
        # Compute the fraction of rows that are null.
        .withColumn("fraction", F.col("values") / F.lit(count))
        # Keep those rows that meet or exceed the threshold.
        .where(F.col("fraction") >= F.lit(threshold))
        # Now, pivot again so that the column names are restored.
        .groupBy().pivot("categories").sum("fraction")
    )

    # Get the list of the columns that need to be dropped; drop and return.
    columns_to_drop = null_count.columns
    df = df.drop(*columns_to_drop)

    return df


from django.urls import path, include
from rest_framework.routers import DefaultRouter
from blog import apiviews

router = DefaultRouter()
router.register('posts', apiviews.PostViewSet)
router.register('comments', apiviews.CommentViewSet)
router.register('replies', apiviews.ReplyViewSet)
router.register('users', apiviews.UserViewSet )

urlpatterns = [
    path('', include(router.urls)),
    path('rest-auth/', include('rest_auth.urls')),
    path('rest-auth/registration/', include('rest_auth.registration.urls')),
    path('categories', apiviews.CategoryListView.as_view(), name='categories-list'),
    path('category-filter/<int:pk>/', apiviews.PostsInCategoryView.as_view(), name='category-detail')
]

# urlpatterns += router.urls


import pytest
import stk

from .case_data import CaseData

@pytest.fixture(
    scope="session",
    params=(
        lambda name: CaseData(
            molecule=stk.BuildingBlock("C1=CC=CC=C1"),
            sub_group_data={
                "c6_planarity": [2.7518147481201438e-06],
                "c5n1_planarity": [],
                "x5_planarity": [],
                "c#c_angle": [],
            },
            name=name,
        ),
        lambda name: CaseData(
            molecule=stk.BuildingBlock("C1N=CC(CCC2CCOC2)N=1"),
            sub_group_data={
                "c6_planarity": [],
                "c5n1_planarity": [],
                "x5_planarity": [1.3688005804646254e-06, 0.932064037529801],
                "c#c_angle": [],
            },
            name=name,
        ),
        lambda name: CaseData(
            molecule=stk.BuildingBlock("C1=CC=C(C=C1)C#CC2=CN=CC=C2"),
            sub_group_data={
                "c6_planarity": [8.41286151020968e-08],
                "c5n1_planarity": [5.678704369238556e-08],
                "x5_planarity": [],
                "c#c_angle": [179.00063441359868],
            },
            name=name,
        ),
    ),
)
def case_data(request) -> CaseData:
    return request.param(
        f"{request.fixturename}{request.param_index}",
    )


from __future__ import annotations

import contextlib
import inspect
import os
import time
import warnings
from collections import OrderedDict
from importlib import import_module
from typing import Any, Callable, Dict, Optional, Union

import numpy as np
import torch
import torch.cuda.amp as amp
from tqdm import tqdm

from .tensorboard_logger import TensorboardLogger
from .wandb_logger import WandbLogger
from ..util import auto_compile, get_constructor_arguments, is_compiled

class DefaultTrainer:
    """Trainer class for 2d/3d training on a single GPU."""

    def __init__(
        self,
        name: Optional[str],
        train_loader: torch.utils.data.DataLoader,
        val_loader: torch.utils.data.DataLoader,
        model: torch.nn.Module,
        loss,
        optimizer,
        metric,
        device: Union[str, torch.device],
        lr_scheduler=None,
        log_image_interval=100,
        mixed_precision=True,
        early_stopping=None,
        logger=TensorboardLogger,
        logger_kwargs: Optional[Dict[str, Any]] = None,
        id_: Optional[str] = None,
        save_root: Optional[str] = None,
        compile_model: Optional[Union[bool, str]] = None,
    ):
        if name is None and not issubclass(logger, WandbLogger):
            raise TypeError("Name cannot be None if not using the WandbLogger")

        if not all(hasattr(loader, "shuffle") for loader in [train_loader, val_loader]):
            raise ValueError(f"{self.__class__} requires each dataloader to have 'shuffle' attribute.")

        self._generate_name = name is None
        self.name = name
        self.id_ = id_ or name
        self.train_loader = train_loader
        self.val_loader = val_loader
        self.model = model
        self.loss = loss
        self.optimizer = optimizer
        self.metric = metric
        self.device = device
        self.lr_scheduler = lr_scheduler
        self.log_image_interval = log_image_interval
        self.save_root = save_root
        self.compile_model = compile_model

        self._iteration = 0
        self._epoch = 0
        self._best_epoch = 0

        self.mixed_precision = mixed_precision
        self.early_stopping = early_stopping

        self.scaler = amp.GradScaler() if mixed_precision else None

        self.logger_class = logger
        self.logger_kwargs = logger_kwargs
        self.log_image_interval = log_image_interval

    @property  # because the logger may generate and set trainer.id on logger.__init__
    def checkpoint_folder(self):
        assert self.id_ is not None
        # save_root enables saving the checkpoints somewhere else than in the local
        # folder. This is handy for filesystems with limited space, where saving the checkpoints
        # and log files can easily lead to running out of space.
        save_root = getattr(self, "save_root", None)
        return os.path.join("./checkpoints", self.id_) if save_root is None else\
            os.path.join(save_root, "./checkpoints", self.id_)

    @property
    def iteration(self):
        return self._iteration

    @property
    def epoch(self):
        return self._epoch

    class Deserializer:
        """Determines how to deserialize the trainer kwargs from serialized 'init_data'

        Examples:
            To extend the initialization process you can inherite from this Deserializer in an inherited Trainer class.
            Note that `DefaultTrainer.Deserializer.load_generic()` covers most cases already.

            This example adds `the_answer` kwarg, which requires 'calculations' upon initialization:
            >>> class MyTrainer(DefaultTrainer):
            >>>     def __init__(self, *args, the_answer: int, **kwargs):
            >>>         super().__init__(*args, **kwargs)
            >>>         self.the_answer = the_answer  # this allows the default Serializer to save the new kwarg,
            >>>                                       # see DefaultTrainer.Serializer
            >>>
            >>>     class Deserializer(DefaultTrainer.Deserializer):
            >>>         def load_the_answer(self):
            >>>             generic_answer = self.init_data["the_answer"]
            >>>             # (device dependent) special deserialization
            >>>             if self.trainer_kwargs["device"].type == "cpu":  # accessing previously deserialized kwarg
            >>>                 self.trainer_kwargs["the_answer"] = generic_answer + 1
            >>>             else:
            >>>                 self.trainer_kwargs["the_answer"] = generic_answer * 2
        """

        def __init__(self, init_data: dict, save_path: str, device: Union[str, torch.device]):
            self.init_data = init_data
            self.save_path = save_path
            # populate with deserialized trainer kwargs during deserialization; possibly overwrite 'device'
            self.trainer_kwargs: Dict[str, Any] = dict(
                device=torch.device(self.init_data["device"]) if device is None else torch.device(device)
            )

        def load(self, kwarg_name: str, optional):
            """`optional` is True if self.trainer.__class__.__init__ specifies a default value for 'kwarg_name'"""

            if kwarg_name == "device":
                pass  # deserialized in __init__
            elif kwarg_name.endswith("_loader"):
                self.load_data_loader(kwarg_name, optional)
            else:
                load = getattr(self, f"load_{kwarg_name}", self.load_generic)
                load(kwarg_name, optional=optional)

        def load_data_loader(self, loader_name, optional) -> None:
            ds = self.init_data.get(loader_name.replace("_loader", "_dataset"))
            if ds is None and optional:
                return

            loader_kwargs = self.init_data[f"{loader_name}_kwargs"]
            loader = torch.utils.data.DataLoader(ds, **loader_kwargs)
            # monkey patch shuffle loader_name to the loader
            loader.shuffle = loader_kwargs.get("shuffle", False)
            self.trainer_kwargs[loader_name] = loader

        def load_generic(
            self,
            kwarg_name: str,
            *dynamic_args,
            optional: bool,
            only_class: bool = False,
            dynamic_kwargs: Optional[Dict[str, Any]] = None,
        ) -> None:
            if kwarg_name in self.init_data:
                self.trainer_kwargs[kwarg_name] = self.init_data[kwarg_name]
                return

            this_cls = self.init_data.get(f"{kwarg_name}_class", None)
            if this_cls is None:
                if optional:
                    return
                else:
                    raise RuntimeError(f"Could not find init data for {kwarg_name} in {self.save_path}")

            assert isinstance(this_cls, str), this_cls
            assert "." in this_cls, this_cls
            cls_p, cls_m = this_cls.rsplit(".", 1)
            this_cls = getattr(import_module(cls_p), cls_m)
            if only_class:
                self.trainer_kwargs[kwarg_name] = this_cls
            else:
                self.trainer_kwargs[kwarg_name] = this_cls(
                    *dynamic_args, **self.init_data.get(f"{kwarg_name}_kwargs", {}), **(dynamic_kwargs or {})
                )

        def load_name(self, kwarg_name: str, optional: bool):
            self.trainer_kwargs[kwarg_name] = os.path.split(os.path.dirname(self.save_path))[1]

        def load_optimizer(self, kwarg_name: str, optional: bool):
            self.load_generic(kwarg_name, self.trainer_kwargs["model"].parameters(), optional=optional)

        def load_lr_scheduler(self, kwarg_name: str, optional: bool):
            self.load_generic(kwarg_name, self.trainer_kwargs["optimizer"], optional=optional)

        # todo: remove and rename kwarg 'logger' to 'logger_class'
        def load_logger(self, kwarg_name: str, optional: bool):
            assert kwarg_name == "logger"
            self.load_generic("logger", optional=optional, only_class=True)

    @staticmethod
    def _get_save_dict(save_path, device):
        if not os.path.exists(save_path):
            raise ValueError(f"Cannot find checkpoint {save_path}")
        return torch.load(save_path, map_location=device)

    @classmethod
    def from_checkpoint(cls, checkpoint_folder, name="best", device=None):
        save_path = os.path.join(checkpoint_folder, f"{name}.pt")
        # make sure the correct device is set if we don't have access to CUDA
        if not torch.cuda.is_available():
            device = "cpu"
        save_dict = cls._get_save_dict(save_path, device)
        deserializer = cls.Deserializer(save_dict["init"], save_path, device)

        has_kwargs = False
        deserialized = []
        for name, parameter in inspect.signature(cls).parameters.items():
            if name == "kwargs":
                has_kwargs = True
                continue
            deserializer.load(name, optional=parameter.default is not inspect.Parameter.empty)
            deserialized.append(name)

        # to deserialze kwargs we can't rely on inspecting the signature, so we
        # go through the remaning kwarg names in init data instead
        if has_kwargs:
            kwarg_names = list(set(deserializer.init_data.keys()) - set(deserialized))
            for name in kwarg_names:
                if name.endswith("_kwargs"):
                    continue
                elif name.endswith("_dataset"):
                    deserializer.load(name.replace("dataset", "loader"), optional=False)
                elif name.endswith("_class"):
                    deserializer.load(name.replace("_class", ""), optional=False)
                else:
                    deserializer.load(name, optional=False)

        trainer = cls(**deserializer.trainer_kwargs)
        trainer._initialize(0, save_dict)
        trainer._is_initialized = True
        return trainer

    class Serializer:
        """Implements how to serialize trainer kwargs from a trainer instance

        Examples:
            To extend the serialization process you can inherite from this Serializer in a derived Trainer class.
            Note that the methods `dump_generic_builtin()`, `dump_generic_class()` and `dump_generic_instance()`
            called by the `dump()` method when appropriate cover most cases already.

            This example adds `the_answer` kwarg, which requires extra steps on dumping only because we don't keep a
            'the_answer' attribute:
            >>> class MyTrainer(DefaultTrainer):
            >>>     def __init__(self, *args, the_answer: int, **kwargs):
            >>>         super().__init__(*args, **kwargs)
            >>>         # self.the_answer = the_answer  # this would allow the default Serializer to save the new kwarg,
            >>>         # but let's make things more interesting...
            >>>         self.the = the_answer // 10
            >>>         self.answer = the_answer % 10
            >>>
            >>>     class Serializer(DefaultTrainer.Serializer):
            >>>         trainer: MyTrainer
            >>>         def dump_the_answer(self, kwarg_name: str) -> None:  # custom dump method for 'the_answer' kwarg
            >>>             assert kwarg_name == "the_answer"
            >>>             # populate self.init_data with the serialized data required by Deserializer
            >>>             # to restore the trainer kwargs
            >>>             self.init_data["the_answer"] = self.trainer.the * 10 + self.trainer.answer

            This example with both Serializer and Deserializer adds `the_answer` kwarg,
            while saving it in two separate entries 'the' and 'answer'
            >>> class MyTrainer(DefaultTrainer):
            >>>     def __init__(self, *args, the_answer: int, **kwargs):
            >>>         super().__init__(*args, **kwargs)
            >>>         self.the_answer = the_answer
            >>>
            >>>     class Serializer(DefaultTrainer.Serializer):
            >>>         trainer: MyTrainer
            >>>         def dump_the_answer(self, kwarg_name: str):
            >>>             assert kwarg_name == "the_answer"
            >>>             self.init_data.update({
            >>>                 "the": self.trainer.the_answer // 10,
            >>>                 "answer": self.trainer.the_answer % 10
            >>>             })
            >>>
            >>>     class Deserializer(DefaultTrainer.Deserializer):
            >>>         def load_the_answer(self, kwarg_name: str, optional: bool):
            >>>             assert kwarg_name == "the_answer"
            >>>             # 'optional' is True if MyTrainer.__init__ specifies a default value for 'kwarg_name'
            >>>             self.trainer_kwargs[kwarg_name] = self.init_data["the"] * 10 + self.init_data["answer"]
        """

        def __init__(self, trainer: DefaultTrainer):
            self.trainer = trainer
            self.init_data = {}  # to be populated during serialization process

        def dump(self, kwarg_name: str) -> None:
            dumper = getattr(self, f"dump_{kwarg_name}", None)
            if dumper is not None:
                dumper(kwarg_name)
            elif kwarg_name.endswith("_loader"):
                self.dump_data_loader(kwarg_name)
            elif kwarg_name.endswith("_class"):
                self.dump_generic_class(kwarg_name)
            elif not hasattr(self.trainer, kwarg_name):
                raise AttributeError(
                    f"{self.trainer.__class__} missing attribute '{kwarg_name}' "
                    f"or special dump method {self.trainer.__class__}.Serializer.dump_{kwarg_name}()"
                )
            else:
                assert hasattr(self.trainer, kwarg_name)
                obj = getattr(self.trainer, kwarg_name)
                if obj is None or type(obj) in (
                    bool,
                    bytearray,
                    bytes,
                    dict,
                    float,
                    frozenset,
                    int,
                    list,
                    set,
                    str,
                    tuple,
                ):
                    self.dump_generic_builtin(kwarg_name)
                else:
                    self.dump_generic_instance(kwarg_name)

        def dump_generic_builtin(self, kwarg_name: str) -> None:
            assert hasattr(self.trainer, kwarg_name)
            self.init_data[kwarg_name] = getattr(self.trainer, kwarg_name)

        def dump_generic_class(self, kwarg_name: str) -> None:
            assert hasattr(self.trainer, kwarg_name)
            assert kwarg_name.endswith("_class")
            obj = getattr(self.trainer, kwarg_name)
            self.init_data[kwarg_name] = None if obj is None else f"{obj.__module__}.{obj.__name__}"

        def dump_generic_instance(self, kwarg_name: str) -> None:
            assert hasattr(self.trainer, kwarg_name)
            instance = getattr(self.trainer, kwarg_name)
            self.init_data.update(
                {
                    f"{kwarg_name}_class": f"{instance.__class__.__module__}.{instance.__class__.__name__}",
                    f"{kwarg_name}_kwargs": get_constructor_arguments(instance),
                }
            )

        def dump_device(self, kwarg_name: str):
            assert hasattr(self.trainer, kwarg_name)
            self.init_data[kwarg_name] = str(getattr(self.trainer, kwarg_name))

        def dump_data_loader(self, kwarg_name: str) -> None:
            assert hasattr(self.trainer, kwarg_name)
            loader = getattr(self.trainer, kwarg_name)
            if loader is None:
                return
            self.init_data.update(
                {
                    f"{kwarg_name.replace('_loader', '_dataset')}": loader.dataset,
                    f"{kwarg_name}_kwargs": get_constructor_arguments(loader),
                }
            )

        def dump_logger(self, kwarg_name: str):  # todo: remove and rename kwarg 'logger' to 'logger_class'
            self.dump_generic_class(f"{kwarg_name}_class")

        def dump_model(self, kwarg_name: str):
            if is_compiled(self.trainer.model):
                self.init_data.update(
                    {
                        "model_class": self.trainer._model_class,
                        "model_kwargs": self.trainer._model_kwargs,
                    }
                )
            else:
                self.dump_generic_instance("model")

    def _build_init(self) -> Dict[str, Any]:
        serializer = self.Serializer(self)
        for name in inspect.signature(self.__class__).parameters:
            # special rules to serialize kwargs
            # if a trainer class inherits from DefaultTrainer and has **kwargs
            # they need to be saved in self._kwargs
            if name == "kwargs":
                if not hasattr(self, "_kwargs"):
                    msg = "The trainer class has **kwargs in its signature, but is missing the _kwargs attribute. " +\
                          "Please add self._kwargs to its __init__ function"
                    raise RuntimeError(msg)
                kwargs = getattr(self, "_kwargs")
                for kwarg_name in kwargs:
                    serializer.dump(kwarg_name)
                continue
            serializer.dump(name)

        return serializer.init_data

    def _initialize(self, iterations, load_from_checkpoint, epochs=None):
        assert self.train_loader is not None
        assert self.val_loader is not None
        assert self.model is not None
        assert self.loss is not None
        assert self.optimizer is not None
        assert self.metric is not None
        assert self.device is not None

        if load_from_checkpoint is not None:
            self.load_checkpoint(load_from_checkpoint)

        if sum((iterations is not None, epochs is not None)) != 1:
            raise ValueError(
                "Exactly one of 'iterations' or 'epochs' has to be specified to initialize the trainer."
                f"You have passed 'iterations'={iterations} and 'epochs'={epochs}"
            )

        if epochs is None:
            epochs = int(np.ceil(float(iterations) / len(self.train_loader)))
        else:
            iterations = epochs * len(self.train_loader)

        self.max_iteration = self._iteration + iterations
        self.max_epoch = self._epoch + epochs

        if not getattr(self, "_is_initialized", False):
            # check if we compile the model (only supported by pytorch 2)
            # to enable (de)serialization of compiled models, we keep track of the model class and kwargs
            if is_compiled(self.model):
                warnings.warn(
                    "You have passed a compiled model to the trainer."
                    "It will not be possible to (de)serialize the trainer with it."
                    "If you want to be able to do this please pass the normal model."
                    "It can be automatically compiled by setting 'compile_model' to True"
                )
            self._model_class = f"{self.model.__class__.__module__}.{self.model.__class__.__name__}"
            self._model_kwargs = get_constructor_arguments(self.model)
            self.model = auto_compile(self.model, self.compile_model)

            self.model.to(self.device)
            self.loss.to(self.device)

            # this saves all the information that is necessary
            # to fully load the trainer from the checkpoint
            self.init_data = self._build_init()

            if self.logger_class is None:
                self.logger = None
            else:
                # may set self.name if self.name is None
                save_root = getattr(self, "save_root", None)
                self.logger = self.logger_class(self, save_root, **(self.logger_kwargs or {}))

            try:
                os.makedirs(self.checkpoint_folder, exist_ok=True)
            except PermissionError:
                warnings.warn(
                    f"The checkpoint folder at {self.checkpoint_folder} could not be created."
                    "The most likely reason for this is that you copied the checkpoint somewhere else,"
                    "so we skip this error to enable loading the model from this checkpoint."
                )
                pass

        best_metric = np.inf
        return best_metric

    def save_checkpoint(self, name, best_metric, **extra_save_dict):
        save_path = os.path.join(self.checkpoint_folder, f"{name}.pt")
        extra_init_dict = extra_save_dict.pop("init", {})
        save_dict = {
            "iteration": self._iteration,
            "epoch": self._epoch,
            "best_epoch": self._best_epoch,
            "best_metric": best_metric,
            "model_state": self.model.state_dict(),
            "optimizer_state": self.optimizer.state_dict(),
            "init": self.init_data | extra_init_dict,
        }
        save_dict.update(**extra_save_dict)
        if self.scaler is not None:
            save_dict.update({"scaler_state": self.scaler.state_dict()})
        if self.lr_scheduler is not None:
            save_dict.update({"scheduler_state": self.lr_scheduler.state_dict()})
        torch.save(save_dict, save_path)

    def load_checkpoint(self, checkpoint="best"):
        if isinstance(checkpoint, str):
            save_path = os.path.join(self.checkpoint_folder, f"{checkpoint}.pt")
            if not os.path.exists(save_path):
                warnings.warn(f"Cannot load checkpoint. {save_path} does not exist.")
                return
            save_dict = torch.load(save_path)
        elif isinstance(checkpoint, dict):
            save_dict = checkpoint
        else:
            raise RuntimeError

        self._iteration = save_dict["iteration"]
        self._epoch = save_dict["epoch"]
        self._best_epoch = save_dict["best_epoch"]
        self.best_metric = save_dict["best_metric"]

        model_state = save_dict["model_state"]
        # to enable loading compiled models
        compiled_prefix = "_orig_mod."
        model_state = OrderedDict(
            [(k[len(compiled_prefix):] if k.startswith(compiled_prefix) else k, v) for k, v in model_state.items()]
        )
        self.model.load_state_dict(model_state)
        # we need to send the network to the device before loading the optimizer state!
        self.model.to(self.device)

        self.optimizer.load_state_dict(save_dict["optimizer_state"])
        if self.scaler is not None:
            self.scaler.load_state_dict(save_dict["scaler_state"])
        if self.lr_scheduler is not None:
            self.lr_scheduler.load_state_dict(save_dict["scheduler_state"])

        return save_dict

    def fit(self, iterations=None, load_from_checkpoint=None, epochs=None, save_every_kth_epoch=None):
        """Run neural network training.

        Exactly one of 'iterations' or 'epochs' has to be passed.

        Parameters:
            iterations [int] - how long to train, specified in iterations (default: None)
            load_from_checkpoint [str] - path to a checkpoint from where training should be continued (default: None)
            epochs [int] - how long to train, specified in epochs (default: None)
            save_every_kth_epoch [int] - save checkpoints after every kth epoch separately.
                The corresponding checkpoints will be saved with the naming scheme 'epoch-{epoch}.pt'. (default: None)
        """
        best_metric = self._initialize(iterations, load_from_checkpoint, epochs)
        print(
            "Start fitting for",
            self.max_iteration - self._iteration,
            "iterations / ",
            self.max_epoch - self._epoch,
            "epochs",
        )
        print("with", len(self.train_loader), "iterations per epoch")

        if self.mixed_precision:
            train_epoch = self._train_epoch_mixed
            validate = self._validate_mixed
            print("Training with mixed precision")
        else:
            train_epoch = self._train_epoch
            validate = self._validate
            print("Training with single precision")

        progress = tqdm(
            total=epochs * len(self.train_loader) if iterations is None else iterations,
            desc=f"Epoch {self._epoch}", leave=True
        )
        msg = "Epoch %i: average [s/it]: %f, current metric: %f, best metric: %f"

        train_epochs = self.max_epoch - self._epoch
        for _ in range(train_epochs):

            # run training and validation for this epoch
            t_per_iter = train_epoch(progress)
            current_metric = validate()

            # perform all the post-epoch steps:

            # apply the learning rate scheduler
            if self.lr_scheduler is not None:
                self.lr_scheduler.step(current_metric)

            # save this checkpoint as the new best checkpoint if
            # it has the best overall validation metric
            if current_metric < best_metric:
                best_metric = current_metric
                self._best_epoch = self._epoch
                self.save_checkpoint("best", best_metric)

            # save this checkpoint as the latest checkpoint
            self.save_checkpoint("latest", best_metric)

            # if we save after every k-th epoch then check if we need to save now
            if save_every_kth_epoch is not None and (self._epoch + 1) % save_every_kth_epoch == 0:
                self.save_checkpoint(f"epoch-{self._epoch + 1}", best_metric)

            # if early stopping has been specified then check if the stopping condition is met
            if self.early_stopping is not None:
                epochs_since_best = self._epoch - self._best_epoch
                if epochs_since_best > self.early_stopping:
                    print("Stopping training because there has been no improvement for", self.early_stopping, "epochs")
                    break

            self._epoch += 1
            progress.set_description(msg % (self._epoch, t_per_iter, current_metric, best_metric), refresh=True)

        print(f"Finished training after {self._epoch} epochs / {self._iteration} iterations.")
        print(f"The best epoch is number {self._best_epoch}.")

        if self._generate_name:
            self.name = None

        # TODO save the model to wandb if we have the wandb logger
        if isinstance(self.logger, WandbLogger):
            self.logger.get_wandb().finish()

    def _backprop(self, loss):
        loss.backward()
        self.optimizer.step()

    def _backprop_mixed(self, loss):
        self.scaler.scale(loss).backward()
        self.scaler.step(self.optimizer)
        self.scaler.update()

    def _train_epoch(self, progress):
        return self._train_epoch_impl(progress, contextlib.nullcontext, self._backprop)

    def _train_epoch_mixed(self, progress):
        return self._train_epoch_impl(progress, amp.autocast, self._backprop_mixed)

    def _forward_and_loss(self, x, y):
        pred = self.model(x)
        if self._iteration % self.log_image_interval == 0:
            if pred.requires_grad:
                pred.retain_grad()

        loss = self.loss(pred, y)
        return pred, loss

    def _train_epoch_impl(self, progress, forward_context, backprop: Callable[[torch.Tensor], None]):
        self.model.train()

        n_iter = 0
        t_per_iter = time.time()
        for x, y in self.train_loader:
            x, y = x.to(self.device), y.to(self.device)

            self.optimizer.zero_grad()

            with forward_context():
                pred, loss = self._forward_and_loss(x, y)

            backprop(loss)

            lr = [pm["lr"] for pm in self.optimizer.param_groups][0]
            if self.logger is not None:
                self.logger.log_train(self._iteration, loss, lr, x, y, pred, log_gradients=True)

            self._iteration += 1
            n_iter += 1
            if self._iteration >= self.max_iteration:
                break
            progress.update(1)

        t_per_iter = (time.time() - t_per_iter) / n_iter
        return t_per_iter

    def _validate(self):
        return self._validate_impl(contextlib.nullcontext)

    def _validate_mixed(self):
        return self._validate_impl(amp.autocast)

    def _validate_impl(self, forward_context):
        self.model.eval()

        metric_val = 0.0
        loss_val = 0.0

        with torch.no_grad():
            for x, y in self.val_loader:
                x, y = x.to(self.device), y.to(self.device)
                with forward_context():
                    pred, loss = self._forward_and_loss(x, y)
                    metric = self.metric(pred, y)

                loss_val += loss.item()
                metric_val += metric.item()

        metric_val /= len(self.val_loader)
        loss_val /= len(self.val_loader)
        if self.logger is not None:
            self.logger.log_validation(self._iteration, metric_val, loss_val, x, y, pred)
        return metric_val


import numpy as np
from tqdm import tqdm

flip_inst = {}
flip_inst['e'] = [1, 0]
flip_inst['w'] = [-1, 0]
flip_inst['se'] = [0, -1]
flip_inst['sw'] = [-1, -1]
flip_inst['ne'] = [1, 1]
flip_inst['nw'] = [0, 1]

def flip_tile(instr, tiles):
    tile = np.array([0, 0])
    while instr:
        for fi, dir in flip_inst.items():
            if instr.startswith(fi):
                tile += dir
                instr = instr[len(fi):]
                continue

    if list(tile) in tiles:
        del tiles[tiles.index(list(tile))]
    else:
        tiles.append(list(tile))
    return tiles

def count_neighbors(matrix, x, y):
    num = 0
    for _, dir in flip_inst.items():
        if matrix[x + dir[0], y + dir[1]] == 1:
            num += 1
    return num

def evolve(matrix):
    to_flip = np.zeros(matrix.shape)
    for x in range(1, matrix.shape[0]-1):
        for y in range(1, matrix.shape[1]-1):
            neighbors = count_neighbors(matrix, x, y)
            # Any black tile with zero or more than 2 black tiles immediately
            # adjacent to it is flipped to white.
            if matrix[x, y] == 1:
                if neighbors == 0 or neighbors > 2:
                    to_flip[x, y] = 1
            # Any white tile with exactly 2 black tiles immediately adjacent to
            # it is flipped to black.
            else:
                if neighbors == 2:
                    to_flip[x, y] = 1
    matrix[to_flip == 1] = (matrix[to_flip == 1] + 1) % 2
    return matrix

if __name__ == "__main__":
    file_name = "test_24.txt"
    file_name = "input_24.txt"

    tiles = []
    for line in open(file_name):
        tiles = flip_tile(line.strip(), tiles)
    print(len(tiles))  # 521

    tiles = np.array(tiles)
    delx = max(tiles[:, 0]) - min(tiles[:, 0])
    dely = max(tiles[:, 1]) - min(tiles[:, 1])
    matrix = np.zeros([200 + delx, 200 + dely], np.int)
    for t in tiles:
        matrix[t[0]+100, t[1]+100] = 1

    for _ in tqdm(range(100)):
        matrix = evolve(matrix)
    print(np.sum(matrix))  # 4242