import random
import json
import os

import numpy as np

def get_unique_tokens(list_fin):
    """
    list of json files --> list of unique scene tokens
    """
    list_token_scene = []

    # Open one json file at a time
    for name_fin in list_fin:
        with open(name_fin, 'r') as f:
            dict_fin = json.load(f)

        # Check if the token scene is already in the list and if not add it
        if dict_fin['token_scene'] not in list_token_scene:
            list_token_scene.append(dict_fin['token_scene'])

    return list_token_scene

def split_scenes(list_token_scene, train, val, dir_main, save=False, load=True):
    """
    Split the list according tr, val percentages (test percentage is a consequence) after shuffling the order
    """

    path_split = os.path.join(dir_main, 'scenes', 'split_scenes.json')

    if save:
        random.seed(1)
        random.shuffle(list_token_scene)  # it shuffles in place
        n_scenes = len(list_token_scene)
        n_train = round(n_scenes * train / 100)
        n_val = round(n_scenes * val / 100)
        list_train = list_token_scene[0: n_train]
        list_val = list_token_scene[n_train: n_train + n_val]
        list_test = list_token_scene[n_train + n_val:]

        dic_split = {'train': list_train, 'val': list_val, 'test': list_test}
        with open(path_split, 'w') as f:
            json.dump(dic_split, f)

    if load:
        with open(path_split, 'r') as f:
            dic_split = json.load(f)

    return dic_split

def select_categories(cat):
    """
    Choose the categories to extract annotations from
    """
    assert cat in ['person', 'all', 'car', 'cyclist']

    if cat == 'person':
        categories = ['human.pedestrian']
    elif cat == 'all':
        categories = ['human.pedestrian', 'vehicle.bicycle', 'vehicle.motorcycle']
    elif cat == 'cyclist':
        categories = ['vehicle.bicycle']
    elif cat == 'car':
        categories = ['vehicle']
    return categories

def update_with_tokens(dict_gt, nusc, token_sd):
    """
    Update with tokens corresponding to the token_sd
    """

    table_sample_data = nusc.get('sample_data', token_sd)  # Extract the whole record to get the sample token
    token_sample = table_sample_data['sample_token']  # Extract the sample_token from the table
    table_sample = nusc.get('sample', token_sample)  # Get the record of the sample
    token_scene = table_sample['scene_token']
    dict_gt['token_sample_data'] = token_sd
    dict_gt['token_sample'] = token_sample
    dict_gt['token_scene'] = token_scene
    return dict_gt

def update_with_box(dict_gt, box):

    bbox = np.zeros(7, )
    flag_child = False

    # Save the 3D bbox
    bbox[0:3] = box.center
    bbox[3:6] = box.wlh
    bbox[6] = box.orientation.degrees
    dict_gt['boxes'].append(bbox.tolist()) # Save as list to be serializable by a json file
    if box.name == 'human.pedestrian.child':
        flag_child = True

    return dict_gt, flag_child


from pathlib import Path
from datetime import datetime
import csv

class Inventory():
    def __init__(self):
        self.inventory_file = Path(
            __file__).absolute().parent / "csv-data/inventory.csv"

    def last_current_id(self):
        with open(self.inventory_file, "r") as csv_inventory:
            dictreader = csv.DictReader(csv_inventory)
            last_line = ""
            for line in dictreader:
                last_line = line
            last_id = int(last_line["id"])
            return last_id

    def show(self, report_date="", report_second_date=""):
        with open(self.inventory_file, "r") as csv_inventory:
            dictreader = csv.DictReader(csv_inventory)
            list_of_dics = list(dictreader)
            if report_second_date and report_date:
                format = "%Y-%m-%d"
                first_date = datetime.strptime(report_date, format)
                second_date = datetime.strptime(report_second_date, format)
                filtered_list = []
                for dic in list_of_dics:
                    current_dic_date = datetime.strptime(dic["buy_date"], format)
                    if current_dic_date <= second_date and current_dic_date >= first_date:
                        filtered_list.append(dic)
                return filtered_list
            elif report_date:
                filtered_list = []
                for dic in list_of_dics:
                    if dic["buy_date"] == report_date:
                        filtered_list.append(dic)
                return filtered_list
            return list_of_dics

    def check(self, item_dict):
        item_id = str(item_dict["id"])
        item_quantity = int(item_dict["quantity"])
        with open(self.inventory_file, "r") as csv_inventory:
            dictreader = csv.DictReader(csv_inventory)
            for line in dictreader:
                if line["id"] == item_id:
                    if int(line["quantity"]) >= item_quantity:
                        return {"stock": True,
                                "product": line["product"],
                                "buy_price": line["buy_price"],
                                "exp_date": line["exp_date"]
                                }
        return {"stock": False}

    def add(self, add_dict):
        with open(self.inventory_file, "a", newline="") as csv_inventory:
            fieldnames = ["id", "product", "quantity",
                          "buy_price", "exp_date", "buy_date"]
            dictwriter = csv.DictWriter(csv_inventory, fieldnames=fieldnames)
            add_dict["id"] = str(self.last_current_id()+1)
            dictwriter.writerow(add_dict)

    def remove(self, remove_dict):
        item_id = str(remove_dict["id"])
        item_quantity = int(remove_dict["quantity"])
        with open(self.inventory_file, "r+") as csv_inventory:
            dictreader = csv.DictReader(csv_inventory)

            dicts = []
            for dic in dictreader:
                if dic["id"] == item_id:
                    dic["quantity"] = int(dic["quantity"]) - item_quantity
                if int(dic["quantity"]) > 0:
                    dicts.append(dic)
                else:
                    pass

            with open(self.inventory_file, "w", newline="") as csv_inventory_write:
                fieldnames = ["id", "product", "quantity",
                              "buy_price", "exp_date", "buy_date"]
                dictwriter = csv.DictWriter(
                    csv_inventory_write, fieldnames=fieldnames)
                dictwriter.writeheader()
                dictwriter.writerows(dicts)


import requests, os, glob, json, base64, time
import openai

def delete():
    """Deletes all images in the image/ directory"""
    files = glob.glob("images/*")
    for f in files:
      os.remove(f)

def upload_file(file_path):
    """Upload a file to a CDN"""
    # Set the API endpoint
    API_ENDPOINT = "https://upload.connerow.dev/upload-base64"

    # Read the image file and encode it in base64
    with open(file_path, "rb") as image_file:
        encoded_image = base64.b64encode(image_file.read()).decode("utf-8")

    # Set the headers for the request
    headers = {
        "Content-Type": "application/json",
        "accept": "*/*"
    }

    # Set the payload for the request
    payload = {
        "image": f"data:image/png;base64,{encoded_image}"
    }

    # Make the POST request
    response = requests.post(API_ENDPOINT, headers=headers, json=payload)

    # Get the response data
    response_data = response.json()

    # Check if the request was successful
    if response_data["success"]:
        # The request was successful, return the URL
        return response_data["url"]
    else:
        # The request was not successful, return the error message
        return response_data["message"]

def gpt(prompt):
  """change the prompt with openai's text completion"""

  openai.api_key = os.getenv("API_KEY")

  start_sequence = "\nA:"
  restart_sequence = "\n\nQ: "

  response = openai.Completion.create(
    model="text-davinci-003",
    prompt=f"I am a highly intelligent question answering bot. If you ask me a question that is rooted in truth, I will give you the answer. If you ask me a question that is nonsense, trickery, or has no clear answer, I will respond with \"Question mark\".\n\nQ: Give me a iconic icon for the following sentence. This is used for a logo: \"{prompt}\"\nA: ",
    temperature=0,
    max_tokens=100,
    top_p=1,
    frequency_penalty=0,
    presence_penalty=0,
    stop=["\n"]
  )
  #print(response.content)
  #print(response)
  if 'choices' in response:
    answer = response['choices'][0]['text']
  else:
    answer = 'Unknown'
  print("prompt: "+answer)
  return answer+"digital art, emblem"

def dalle1(num, icon):
  """Generate an image using the openAI v1 image generater"""
  num +=1
  # Set up the API endpoint and API key
  endpoint = "https://api.openai.com/v1/images/generations"
  api_key = os.environ["API_KEY"]

  # Set up the text prompt for the logo

  # Set up the parameters for the API request
  params = {
      "model": "image-alpha-001",
      "prompt": icon,
      "num_images": 1,
      "size": "256x256",
      "response_format": "url"
  }
  urls = []
  for i in range(1,int(num)):

    # Make the API request
    response = requests.post(endpoint, json=params, headers={"Authorization": f"Bearer {api_key}"})

    # Get the URL of the generated image
    image_url = response.json()["data"][0]["url"]

    # Download the image
    response = requests.get(image_url)
    open(f"./images/logo{i}.png", "wb").write(response.content)
    urls.append(upload_file(f'images/logo{i}.png'))
    print(f"Dalle1 done {i}/{num-1}")
  #print(urls)
  return urls

def dalle2(number, icon):
  """Generate an image using DALL-E 2"""
  number += 3
  arr = []
  for i in range(3, int(number)):
    response = openai.Image.create( # generate image
      prompt=icon,
      n=1,
      size="256x256",
    )
    print("Dalle2 Done")
    img_data = requests.get(response["data"][0]["url"]).content
    with open(f'images/logo{i}.jpg', 'wb') as handler:
      handler.write(img_data)
    arr.append(upload_file(f'images/logo{i}.jpg'))
  #print(arr)
  return arr

def convertFromSeconds(seconds):
    seconds = seconds % (24 * 3600)
    hour = seconds // 3600
    seconds %= 3600
    minutes = seconds // 60
    seconds %= 60

    return "%d:%02d:%02d" % (hour, minutes, seconds)

print("\033[32m[Utils] loaded\033[0m")


"""Add many to many relationship user-permission

Revision ID: c90db4410eab
Revises: 8f0af5b4181a
Create Date: 2023-08-21 21:48:25.175618

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'c90db4410eab'
down_revision = '8f0af5b4181a'
branch_labels = None
depends_on = None

def upgrade() -> None:
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('user_permission',
    sa.Column('user_id', sa.Integer(), nullable=False),
    sa.Column('permission_id', sa.Integer(), nullable=False),
    sa.ForeignKeyConstraint(['permission_id'], ['permissions.id'], ),
    sa.ForeignKeyConstraint(['user_id'], ['users.id'], ),
    sa.PrimaryKeyConstraint('user_id', 'permission_id')
    )
    op.drop_constraint('permissions_owner_id_fkey', 'permissions', type_='foreignkey')
    op.drop_column('permissions', 'test')
    op.drop_column('permissions', 'owner_id')
    # ### end Alembic commands ###

def downgrade() -> None:
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('permissions', sa.Column('owner_id', sa.INTEGER(), autoincrement=False, nullable=True))
    op.add_column('permissions', sa.Column('test', sa.VARCHAR(), autoincrement=False, nullable=True))
    op.create_foreign_key('permissions_owner_id_fkey', 'permissions', 'users', ['owner_id'], ['id'])
    op.drop_table('user_permission')
    # ### end Alembic commands ###


from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Tuple
import torch
import os
from torch import Tensor
import torch.nn as nn
from fairseq import options, utils, checkpoint_utils
from fairseq.dataclass import ChoiceEnum, FairseqDataclass
from fairseq.models import (
    transformer,
    FairseqLanguageModel,
    register_model,
    register_model_architecture,
    FairseqEncoder, FairseqIncrementalDecoder,
    BaseFairseqModel,FairseqEncoderDecoderModel
)
from fairseq.data import Dictionary
from fairseq.models.transformer import (
    Embedding, TransformerDecoder,
    TransformerEncoder,
    TransformerModel
)
from fairseq.models.transformer import base_architecture as transformer_architecture
from fairseq.modules import (
    FairseqDropout,
    LayerNorm,Fp32LayerNorm,
    LayerDropModuleList,
    TransformerEncoderLayer,
    TransformerDecoderLayer
)
from fairseq.modules.checkpoint_activations import checkpoint_wrapper
from fairseq.modules.quant_noise import quant_noise as apply_quant_noise_
from omegaconf import II
from rain.layers import (
    get_available_convs, get_conv,
    PositionalEmbedding,AudioTransformerEncoder,
    UnidirectAudioTransformerEncoder, UnidirectTransoformerEncoder,
    WaitkDecoder
)
from fairseq.dataclass.utils import (
    convert_namespace_to_omegaconf,
    gen_parser_from_dataclass,
)
from .speech_transformer import (
    SpeechTransformerModelConfig, AudioTransformer,audio_transformer_s,randpos_audio_transformer_s
)
from .posemb_transformer import RandposTransformer,randpos_transformer_small

@dataclass
class OnlineTextTransformerConfig(SpeechTransformerModelConfig):
    max_source_positions: Optional[int] = II("task.max_text_positions")
    max_target_positions:Optional[int] = II("task.max_text_positions")
    main_context:int = field(
        default= 1, metadata={"help":"main context frame"}
    )
    right_context :int = field(
        default= 0, metadata={"help":"right context frame"}
    )
    decoder_delay_blocks :int = field(
        default= 4, metadata={"help":"wait-k delay"}
    )
    decoder_blocks_per_token :int = field(
        default= 1, metadata={"help":"wait k token blocks"}
    )
    online_type:ChoiceEnum(["offline", "waitk"])="offline"

@register_model("online_text_transformer", dataclass = OnlineTextTransformerConfig)
class OnlineTextTransformer(TransformerModel):

    @classmethod
    def add_args(cls, parser):
        dc = getattr(cls, "__dataclass", None)
        if dc is not None:
            gen_parser_from_dataclass(parser, dc(), delete_default=True)

    @classmethod
    def build_encoder(cls, args, src_dict, embed_tokens):
        model = UnidirectTransoformerEncoder(args, src_dict, embed_tokens)
        embed_dim= embed_tokens.embedding_dim
        if model.embed_positions is not None and args.rand_pos_encoder >0:
            model.embed_positions= PositionalEmbedding(
                model.max_source_positions,
                embed_dim,
                model.padding_idx,
                rand_max = args.rand_pos_encoder,
                learned=args.decoder_learned_pos,
            )
        return model

    @classmethod
    def build_decoder(cls, args, tgt_dict, embed_tokens):
        if args.online_type == "offline":
            model = transformer.TransformerDecoder(
                args,
                tgt_dict,
                embed_tokens,
                no_encoder_attn=False
            )
        elif args.online_type == "waitk":
            model= WaitkDecoder(
                args,
                tgt_dict,
                embed_tokens
            )
        else:
            raise NotImplementedError(f"unknown online type {args.online_type}")
        if model.embed_positions is not None and args.rand_pos_decoder >0:
            model.embed_positions= PositionalEmbedding(
                model.max_target_positions,
                model.embed_dim,
                model.padding_idx,
                rand_max = args.rand_pos_decoder,
                learned=args.decoder_learned_pos,
            )
        return model

@dataclass
class OnlineAudioTransformerConfig(OnlineTextTransformerConfig):
    max_source_positions: Optional[int] = II("task.max_audio_positions")
    max_target_positions:Optional[int] = II("task.max_text_positions")
    main_context:int = field(
        default= 16, metadata={"help":"main context frame"}
    )
    right_context :int = field(
        default= 16, metadata={"help":"right context frame"}
    )
    decoder_delay_blocks :int = field(
        default= 16, metadata={"help":"wait-k delay"}
    )
    decoder_blocks_per_token :int = field(
        default= 4, metadata={"help":"wait k token blocks"}
    )
    online_type:ChoiceEnum(["offline", "waitk"])="offline"

@register_model("online_audio_transformer", dataclass = OnlineAudioTransformerConfig)
class OnlineAudioTransformer(AudioTransformer):
    @classmethod
    def build_encoder(cls, args):
        return UnidirectAudioTransformerEncoder(args)

    @classmethod
    def add_args(cls, parser):
        dc = getattr(cls, "__dataclass", None)
        if dc is not None:
            gen_parser_from_dataclass(parser, dc(), delete_default=True)

    @classmethod
    def build_decoder(cls, args, tgt_dict, embed_tokens):
        if args.online_type == "offline":
            model = transformer.TransformerDecoder(
                args,
                tgt_dict,
                embed_tokens,
                no_encoder_attn=False
            )
        elif args.online_type == "waitk":
            model= WaitkDecoder(
                args,
                tgt_dict,
                embed_tokens
            )
        else:
            raise NotImplementedError(f"unknown online type {args.online_type}")
        if model.embed_positions is not None and args.rand_pos_decoder >0:
            model.embed_positions= PositionalEmbedding(
                model.max_target_positions,
                model.embed_dim,
                model.padding_idx,
                rand_max = args.rand_pos_decoder,
                learned=args.decoder_learned_pos,
            )
        return model

    def forward(
        self,
        fbank,
        fbk_lengths,
        prev_source=None,
        prev_target= None
    ):

        encoder_out = self.encoder(fbank, fbk_lengths)

        prev_tokens= prev_source if self.task_type=="asr" else prev_target
        assert prev_tokens is not None, f"no prev_tokens for {self.task_type}"
        decoder_out = self.decoder(
            prev_tokens,
            encoder_out=encoder_out,
        )
        logits= decoder_out[0]

        return decoder_out

@register_model_architecture("online_audio_transformer", "online_audio_transformer_offline")
def offline_audio(args):
    args.rand_pos_encoder = getattr(args, "rand_pos_encoder", 300)
    args.rand_pos_decoder= getattr(args, "rand_pos_decoder", 30)
    args.main_context=getattr(args,"main_context",16)
    args.right_context=getattr(args,"right_context",16)
    args.decoder_delay_blocks=getattr(args,"decoder_delay_blocks",32)
    args.decoder_blocks_per_token= getattr(args,"decoder_blocks_per_token",8)
    args.online_type=getattr(args,"online_type","offline")
    randpos_audio_transformer_s(args)

@register_model_architecture("online_audio_transformer", "online_audio_transformer_waitk")
def waitk_audio(args):
    args.online_type=getattr(args,"online_type","waitk")
    offline_audio(args)

@register_model_architecture("online_text_transformer", "online_text_transformer_offline")
def offline_text(args):
    args.rand_pos_encoder = getattr(args, "rand_pos_encoder", 30)
    args.rand_pos_decoder= getattr(args, "rand_pos_decoder", 30)
    args.main_context=getattr(args,"main_context",1)
    args.right_context=getattr(args,"right_context",0)
    args.decoder_delay_blocks=getattr(args,"decoder_delay_blocks", 4)
    args.decoder_blocks_per_token= getattr(args,"decoder_blocks_per_token",1)
    args.online_type=getattr(args,"online_type","offline")
    randpos_transformer_small(args)

@register_model_architecture("online_text_transformer", "online_text_transformer_waitk")
def waitk_text(args):
    args.online_type=getattr(args,"online_type","waitk")
    offline_text(args)


# Definition for a binary tree node.
# class TreeNode:
#     def __init__(self, val=0, left=None, right=None):
#         self.val = val
#         self.left = left
#         self.right = right
from collections import deque
class Solution:
    def isUnivalTree(self, root: Optional[TreeNode]) -> bool:
        if not root:
            return None
        q = deque()
        q.append(root)
        p = root.val
        while q:
            node = q.popleft()
            if node.val != p:
                return False
            if node.left:
                q.append(node.left)
            if node.right:
                q.append(node.right)
        return True


'''
Itay dali 204711196
David Toubul 342395563
Chen Azulay 201017159
'''
import pandas as pd
from sklearn import preprocessing

def getColumnTitles(table):
    """

    :param table: pandas DataFrame
    :return: list of columns
    """
    return list(table.columns)

def Discretize(num, table, structureTextFile):
    """

    :param num: number of bins
    :param table: pandas DataFrame
    :param structureTextFile: path of structure file
    :return: pandas DataFrame after discretize
    """
    def numericCol(table, structureTextFile):
        """

        :param table:
        :param structureTextFile: path
        :return: list of column which is numeric by the structure file
        """
        structure = pd.read_csv(structureTextFile, sep=" ", names=['type', 'feature', 'data'])
        column = []
        headers = getColumnTitles(table)
        for i in range(structure.shape[0]):
            if 'NUMERIC' in structure.loc[i]['data']:
                column += [headers[i]]
        return column
    column = numericCol(table, structureTextFile)
    table = table.applymap(lambda s: s.lower() if type(s) == str else s)
    table=table.dropna(subset=['class'])#drop the rows with the NaN values in 'class' column
    for col in column:
        table[col] = pd.qcut(table[col], num, labels=False, duplicates='drop')#Discretize variable into equal-sized buckets
    table = table.fillna(table.mode().iloc[0])#fillna with value that appears most often of Each Column
    # table.fillna(-1, inplace=True)
    # table.apply(lambda x: x.astype(str).str.lower())
    #table = table.applymap(lambda s: s.lower() if type(s) == str else s)
    return table

def p_xy(table,column_x,value_x,column_class,value_class):
    """

    :param table:
    :param column_x:
    :param value_x:
    :param column_class:
    :param value_class:
    :return:
    """
    length=table[column_x].value_counts()[value_x]
    try:
        p=table.loc[table[column_x]==value_x][column_class].value_counts()[value_class]
    except:
        p=1
    p=round(p/length,3)
    return p

def valuesType(table,column):
    """

    :param table:
    :param column:
    :return: return a list with the name of values in column
    """
    columnValues=table[column].unique().tolist()
    if -1 in columnValues:
        columnValues.remove(-1)
    return columnValues

def pArrayByFeature(table,featureCol,classValue,classCol):
    """
    calculate p(x|y=value)
    :param table:
    :param featureCol:
    :param classValue:y
    :param classCol:
    :return: array of probabilities of the values in the column by each value in 'class' column
    """
    array=[]
    for i in valuesType(table,featureCol):
        array+=[p_xy(table,featureCol,i,classCol,classValue)]
    return array

def fit_transforms(table):
    """

    :param table: pandas DataFrame
    :return: turn string values to int
    """
    le=preprocessing.LabelEncoder()
    columns = getColumnTitles(table)
    for col in columns:
        try:
            table[col] = le.fit_transform(table[col])
        except:
            continue
    return table


import speech_recognition as sr
import pyttsx3
import openai
import configparser

config = configparser.ConfigParser()
config.read('config.ini') #Config.ini (Please specify the file path
# config.ini (Open AI Api key)
openai.api_key = config.get('API','openai_api_key')
# G ❤
def get_openai_response(question):
    response = openai.Completion.create(
        engine='text-davinci-003',
        prompt=question,
        max_tokens=100,
        n=1,
        stop=None,
        temperature=0.7
    )
    return response.choices[0].text.strip()

def speak(text):
    engine = pyttsx3.init()
    voices = engine.getProperty('voices')

    # Türkçe sesi seçin (örneğin: Türkiye Türkçesi için 0. index)
    turkish_voice_id = 0
    engine.setProperty('voice', voices[turkish_voice_id].id)

    engine.setProperty('rate', 150)  # Konuşma hızı
    engine.setProperty('volume', 1)  # Ses seviyesi (0.0 ile 1.0 arasında)
    engine.say(text)
    engine.runAndWait()

#Keyword
listen_keyword = "beni dinle" #run
exit_keyword = "çıkış" #exit

# Mikrofon nesnesini oluşturun
r = sr.Recognizer()

# Ses kaynağı olarak varsayılan mikrofonu kullanın
mic = sr.Microphone()

while True:
    # Kullanıcıdan sesli giriş alın
    with mic as source:
        print("Lütfen bir şeyler söyleyin:")
        audio = r.listen(source)

    # Sesli girişi metne dönüştürün
    try:
        user_input = r.recognize_google(audio, language='tr-TR')
        print("Siz: ", user_input)
    except sr.UnknownValueError:
        print("Ses anlaşılamadı")
        continue

    # Anahtar kelimeyi kontrol edin ve programı etkinleştirin
    if listen_keyword in user_input:
        print("Program etkinleştirildi. Sorularınızı sorabilirsiniz.")
        while True:
            # Kullanıcıdan soruyu sesli olarak alın
            with mic as source:
                print("Sorunuzu sesli olarak sorun:")
                audio = r.listen(source)

            # Sesli girişi metne dönüştürün
            try:
                user_question = r.recognize_google(audio, language='tr-TR')
                print("Sorunuz: ", user_question)
            except sr.UnknownValueError:
                print("Ses anlaşılamadı")
                continue

            # Çıkış anahtar kelimesini kontrol edin
            if exit_keyword in user_question.lower():
                break

            # OpenAI'ya soruyu gönderin
            response = get_openai_response(user_question)
            print("OpenAI'nin cevabı: ", response)

            # Cevabı sesli olarak okuyun
            speak(response)

    else:
        print("Anahtar kelime algılanmadı. Programı etkinleştirmek için 'beni dinle' deyin.")


import cPickle
from glob import glob
from itertools import permutations
import numpy as np
from numpy import log
from os import path
import pandas as pd
import statsmodels.formula.api as smf
from utils import scale

# **********Processing*********************************
structuredata = []
valuedata = []
taskdata = {}
winning = {}
communities = {0: [0,1,2,3,4],
              2: [5],
              1: [6,7,8,9,10]}
node_lookup = {i:k for k,v in communities.items() for i in v }
f = lambda x: [k for k,v in communities.items() if x in v][0]

datafiles = sorted(glob('Data/RawData/*pkl'))
for exclude in ['/CH', '/GL', '/YH']:
    datafiles = [i for i in datafiles if exclude not in i]

for filey in datafiles:
    subj = path.basename(filey).split('_')[0]
    data = cPickle.load(open(filey,'rb'))
    subj_structuredata = pd.DataFrame(data['structuredata'])
    subj_structuredata.loc[:,'subjid'] = subj
    subj_valuedata = pd.DataFrame(data['valuedata'])
    subj_valuedata.loc[:,'subjid'] = subj

    # add subject specific variables
    # for structuredata
    subj_structuredata.loc[:,'correct_shift'] = subj_structuredata.correct.shift()
    subj_structuredata.loc[:,'congruent_rot'] = subj_structuredata.rotation==subj_structuredata.rotation.shift()
    # add community columns
    subj_structuredata.loc[:,'community'] = subj_structuredata.stim_index.apply(f)
    subj_structuredata.loc[1:,'community_cross'] = (subj_structuredata.community.diff()!=0)[1:]
    # calculate steps within the community
    steps_in_community = []
    steps = 0
    for cross in subj_structuredata.community_cross:
        if cross==False:
            steps+=1
        else:
            steps=1
        steps_in_community.append(steps)
    subj_structuredata.loc[:,'steps_within_community'] = steps_in_community
    # trials since last seen
    steps_since_seen = []
    last_seen = {}
    # ...for value data
    for i, stim in enumerate(subj_structuredata.stim_index):
        if stim in last_seen.keys():
            steps_since = i-last_seen[stim]
        else:
            steps_since = 0
        last_seen[stim] = i
        steps_since_seen.append(steps_since)
    subj_structuredata.loc[:,'steps_since_seen'] = steps_since_seen

    # for value data
    # regress out accuracy
    structure_acc = subj_structuredata[200:].groupby('stim_index').correct.mean()
    stim_freq = subj_structuredata.groupby('stim_index').rt.count()
    subj_valuedata.loc[:,'stim_acc'] = subj_valuedata.stim_index.apply(lambda x: structure_acc[x])
    subj_valuedata.loc[:,'stim_freq'] = subj_valuedata.stim_index.apply(lambda x: stim_freq[x])
    rs = smf.ols('rating ~ stim_acc', subj_valuedata).fit()
    subj_valuedata.loc[:,'reg_rating'] = rs.resid
    # add stim repetitions
    n_stim = len(np.unique(subj_valuedata.stim_index))
    n_repeats = (len(subj_valuedata))//n_stim
    repeat_array = np.hstack([[i]*n_stim for i in range(n_repeats)])
    subj_valuedata.loc[:, 'stim_repetition'] = repeat_array
    # node values
    node_values = data['taskdata']['node_values']
    value_labels = subj_valuedata.stim_index. \
                    apply(lambda x: node_values.get(x,np.nan))
    subj_valuedata.loc[:,'labeled_value'] = value_labels
    # add community
    subj_valuedata.loc[:,'community'] = subj_valuedata.stim_index.apply(f)

    # concatenate with group
    structuredata.append(subj_structuredata)
    valuedata.append(subj_valuedata)
    taskdata[subj] = data['taskdata']
    winning[subj] = data['total_win']

structuredata = pd.concat(structuredata, axis=0)
valuedata = pd.concat(valuedata, axis=0)

# Remove subjects who had no variability on their ratings
rating_variability = valuedata.groupby(['subjid']).rating.std()
drop_subjects = list(rating_variability.index[rating_variability<.1])
valuedata = valuedata.query('subjid not in %s' % drop_subjects)
structuredata = structuredata.query('subjid not in %s' % drop_subjects)

# **********Analysis*********************************
results = {}

regress_data = structuredata.query('rt!=-1').dropna(subset=['correct_shift'])
#regress_data = structuredata.query('rt!=-1 and correct==True').dropna(subset=['correct_shift'])

# structure reaction time data
models = {}
for DV in ['rt']:
    rs = smf.mixedlm("%s ~ community_cross + steps_since_seen" % DV,
                     data=regress_data, groups='subjid',
                     re_formula="~community_cross")
    # larger model
    rs_full = smf.mixedlm("%s ~ community_cross + steps_since_seen \
                          + correct_shift + C(rotation) + congruent_rot" % DV,
                     data=regress_data, groups=regress_data["subjid"],
                     re_formula="~community_cross")
    DV_name = DV[-7:]
    models[DV_name] = rs
    models[DV_name + '_full'] = rs_full

# individual difference variables
rt_raneffects = models['rt_full'].fit().random_effects
structure_coefficients = {k:v.loc['community_cross[T.True]'] for k,v in rt_raneffects.items()}
results['structure_coefficients'] = structure_coefficients

# value effect as a function of community
valuedata.loc[:,'structure_coefficient'] = valuedata.subjid.apply(lambda x: structure_coefficients[x])
value_rs = smf.mixedlm('rating ~ C(community)+ stim_acc + C(stim_file)',
                 data = valuedata.query('community!=2'),
                 groups='subjid', re_formula="~C(community)").fit()

interaction_value_rs = smf.mixedlm('rating ~ C(community)*structure_coefficient + stim_acc + C(stim_file)',
                 data = valuedata.query('community!=2'),
                 groups='subjid').fit()

value_raneffects = value_rs.random_effects
value_coefficients = {k:v.loc['C(community)[T.1]'] for k,v in value_raneffects.items()}
results['value_coefficients'] = value_coefficients

# calculate standard deviation of clusters within community
group_stats = {}
for subj in valuedata.subjid.unique():
    subj_stats = {}
    permutes = list(permutations(range(7),7))
    ratings = valuedata.query('subjid == "%s"' % subj) \
                        .groupby(['subjid','stim_index','community']) \
                        .mean().loc[:,['rating','reg_rating']].reset_index()

    std_community = (ratings.query('community != 2').query('community != 2') \
                    .groupby(['subjid','community']) \
                    .reg_rating.std()).groupby('community').mean().mean()
    subj_stats['community_std'] = std_community
    permute_stds = []
    for permute in permutes:
        n = len(ratings.subjid.unique())
        permute_pop = ratings.copy().reset_index()
        permute_pop.loc[:, 'community'] = permute_pop.community.iloc[list(permute)].reset_index()
        permute_std = (permute_pop.groupby(['subjid','community']).reg_rating.std()).groupby('community').mean().mean()
        permute_stds.append(permute_std)
    subj_stats['permuted_array']  = permute_stds
    subj_stats['p_val'] = np.mean([std_community>=i for i in permute_stds])
    group_stats[subj] = subj_stats

results['clustering'] = {k:v['p_val'] for k,v in group_stats.items()}
results['example_clustering_dist'] = group_stats.values()[0]['permuted_array']

# structure data win stay/lose switch (or just change in attention following error?)
for rot in [0,90]:
    structuredata[(structuredata.rotation.shift()==rot)
                    & (structuredata.rt!=-1)] \
                .groupby(['correct_shift','rotation']).correct.mean()

    structuredata[(structuredata.rotation.shift()==rot)
                & (structuredata.rt!=-1)] \
            .groupby(['correct_shift','rotation']).rt.median()

# save data
save_loc = path.join('Data','ProcessedData')
structuredata.to_csv(path.join(save_loc, 'structuredata.csv'))
valuedata.to_csv(path.join(save_loc, 'valuedata.csv'))
cPickle.dump(taskdata,open(path.join(save_loc,'taskdata.pkl'),'wb'))
cPickle.dump(results, open(path.join('Analysis_Results','analysis.pkl'), 'wb'))


""" Corrected CC between APs and presynaptic events
Author: Drew B. Headley

"""
import sys

sys.path.append(".")  # have to do this for relative imports to work consistently
import numpy as np
import pandas as pd
import pdb
from .load_spike_h5 import load_spike_h5
from .load_spike_aux_h5 import load_spike_aux_h5
from .cc_ptpt import cc_ptpt

def cc_presyn_phdep_files(presyn_fname, ap_fname, rhym_ser, step_len, win, **kwargs):
    """
    Uses the files for the preynaptic spikes and phase of an afferent rhythm,
    calculate a phase stratified cross-correlation between presynaptic spikes
    and action potentials.

    Parameters
    ----------
    presyn_fname : string
        file path for presynaptic spike events h5
    ap_fname : numpy array
        file path for action potentials
    rhym_ser : numpy array
        inhibitory rhythm
    step_len : numeric
        the bin size for the CC in units of seconds
    win : [int, int] (default is [-10, 10])
        a 2 element list-like specifying the number of bins for either edge
        of the cross correlation function
    kwargs: arguments for cc_ptpt

    Returns
    ----------
    cc : dictionary
        cross correlation for each phase. Keys are 't' for trough and 'p' for peak

    Examples
    ----------

    """
    ph_mask = rhym_ser >= 0

    # load action potential events
    spk_pts = load_spike_h5(ap_fname)

    # load presynaptic events into
    pre_pts = load_spike_aux_h5(presyn_fname)

    # create spike series and keep only those with the right phase
    p_inds = np.where(ph_mask)[0]
    t_inds = np.where(~ph_mask)[0]
    # find set of presynaptic spikes in troughs and peaks
    pre_pts_t = pre_pts[np.isin(pre_pts, t_inds)]
    pre_pts_p = pre_pts[np.isin(pre_pts, p_inds)]

    # calculate cross-correlation
    cc = {}
    cc["t"] = cc_ptpt(spk_pts, pre_pts_t, step_len, win, **kwargs)
    cc["p"] = cc_ptpt(spk_pts, pre_pts_p, step_len, win, **kwargs)

    return cc

if __name__ == "__main__":
    from prettytable import PrettyTable
    import pdb

    print("Testing presynaptic spikes with cc_presyn_phdep_files.py")

    samps_per_ms = 10
    sim_win = [0, 200]
    cc_win = [-15, 15]  # multiply by step to get window in milliseconds
    step = 2 * samps_per_ms  # number of simulation steps for creating the dendritic
    # event occurrence series
    t_ser = np.linspace(
        sim_win[0], sim_win[1], samps_per_ms * 1000 * sim_win[1]
    )  # seconds
    sin_ser = np.sin(t_ser * 16 * 2 * np.pi)

    pre_fpath_test = "Z:\\DendOscSub\\output_clust_16Hz_conc\\exc_stim_aux_spikes2.h5"
    ap_fpath_test = "Z:\\DendOscSub\\output_clust_16Hz_conc\\spikes.h5"

    # pdb.set_trace()
    cc_test = cc_presyn_phdep_files(
        pre_fpath_test, ap_fpath_test, sin_ser, step, cc_win
    )

    # use prettytable to print the results with each row a cc lag
    tbl = PrettyTable()
    tbl.field_names = ["Lag (ms)", "Trough", "Peak"]
    for lags, t_cc, p_cc in zip(
        cc_test["t"]["lags"], cc_test["t"]["values"], cc_test["p"]["values"]
    ):
        tbl.add_row([lags, t_cc, p_cc])
    print(tbl)


from django.shortcuts import render
import requests
import json

# Create your views here.
def home_page(request):
    api_news= requests.get('https://min-api.cryptocompare.com/data/v2/news/?lang=EN')
    api= json.loads(api_news.content)

    api_price= requests.get('https://min-api.cryptocompare.com/data/pricemultifull?fsyms=BTC,ETH,USDT,BNB,ADA,DOGE,XRP,USDC,DOT,UNI&tsyms=USD')
    price= json.loads(api_price.content)

    context= {'api': api, 'price': price}
    return render(request, 'crypto/home_page.html', context )

def prices(request):
    if request.method == 'POST':
        q= request.POST['q']
        q= q.upper()
        q_price= requests.get(f'https://min-api.cryptocompare.com/data/pricemultifull?fsyms={q}&tsyms=USD')
        q_data= json.loads(q_price.content)
        return render(request,'crypto/prices_page.html', {'q': q, 'q_data': q_data})
    else:
        no= 'رسالة بان العملة غير موجودة من الاساس'
        return render(request, 'crypto/prices_page.html', {'no': no} )

def news(request):
    api_news= requests.get('https://min-api.cryptocompare.com/data/v2/news/?lang=EN')
    api= json.loads(api_news.content)
    context= { 'api': api}
    return render(request, 'crypto/news_page.html', context)

def all_price(request):
    api_price= requests.get('https://min-api.cryptocompare.com/data/pricemultifull?fsyms=BTC,ETH,USDT,BNB,ADA,DOGE,XRP,USDC,DOT,UNI&tsyms=USD')
    price= json.loads(api_price.content)
    context= {'price': price}
    return render(request, 'crypto/all_price_page.html', context)

def daily_Pair (request):
    labels= []
    datas= []
    api_daily= requests.get('https://min-api.cryptocompare.com/data/v2/histoday?fsym=BTC&tsym=USD&limit=20')
    daily= json.loads(api_daily.content)
    context= {
        'daily': daily,
    }
    return render(request, 'crypto/daily_pair.html', context)


import functools
import random
from itertools import accumulate
from bisect import bisect

from .random_selection import random_selection
from .rank import rank
from .roulette import roulette
from .tournament import tournament
from .best import best
from structopt.tools import root, single_core, parallel

class Selections(object):
    """ """

    @single_core
    def __init__(self, parameters):
        self.parameters = parameters
        self.selections = {getattr(self, name): self.parameters[name]['probability'] for name in self.parameters}
        self.kwargs = {getattr(self, name): self.parameters[name]['kwargs'] for name in self.parameters}
        total_probability = sum(self.selections.values())
        assert total_probability <= 1.0
        self.selections[None] = 1.0 - total_probability
        self.selected_selection = None

    @single_core
    def select_selection(self):
        # Implementation from https://docs.python.org/3/library/random.html -- Ctrl+F "weights"
        choices, weights = zip(*self.selections.items())
        cumdist = list(accumulate(weights))
        x = random.random() * cumdist[-1]
        self.selected_selection = choices[bisect(cumdist, x)]

    @single_core
    def select(self, population):
        fits = [individual.fitness for individual in population]
        if self.selected_selection is None:
            return []
        kwargs = self.kwargs[self.selected_selection]
        pairs = self.selected_selection(population=population, fits=fits, **kwargs)
        self.post_processing(pairs)
        return pairs

    @single_core
    def post_processing(self, pairs):
        pass

    @staticmethod
    @functools.wraps(random_selection)
    def random_selection(population, fits):
        return random_selection(population, fits)

    @staticmethod
    @functools.wraps(rank)
    def rank(population, fits, p_min=None, unique_pairs=False, unique_parents=False):
        return rank(population, fits, p_min, unique_pairs, unique_parents)

    @staticmethod
    @functools.wraps(roulette)
    def roulette(population, fits, unique_pairs=False, unique_parents=False):
        return roulette(population, fits, unique_pairs, unique_parents)

    @staticmethod
    @functools.wraps(tournament)
    def tournament(population, fits, tournament_size=5, unique_pairs=False, unique_parents=False, keep_best=False):
        return tournament(population, fits, tournament_size, unique_pairs, unique_parents, keep_best)

    @staticmethod
    @functools.wraps(best)
    def best(population, fits):
        return best(population, fits)


import pyupbit
import os
from tqdm import tqdm
import time
import datetime
import argparse

def calc_count(interval) :
    interval_count_min = \
        {
            'minute1': 1440,
            'minute3': 480,
            'minute5': 288,
            'minute10': 144,
            'minute15': 96,
            'minute30': 48,
            'minute60': 24,
            'minute240': 6
        }
    interval_count_year = \
        {
            'day': 365,
            'week': 52,
            'month': 12
        }

    if interval in interval_count_min :
        count = interval_count_min[interval]
        interval_term = 'min'
    else :
        count = interval_count_year[interval]
        interval_term = 'year'

    return count, interval_term

def calc_last_day(year, month):
    next_month = datetime.date(int(year), month, 1).replace(day=28) + datetime.timedelta(days=4)
    return next_month - datetime.timedelta(days=next_month.day)

def int_to_str(num) :
    if len(str(num)) < 2:
        num = "0" + str(num)
    else:
        num = str(num)

    return num

def coin_data(ticker, interval, count, date_y, date_m='01', date_d='01'):
    time.sleep(0.25)

    to = f'{date_y}-{date_m}-{date_d} 00:00'
    df = pyupbit.get_ohlcv(ticker=ticker, interval=interval, to=to, count=count)

    try:
        save_path = "./data/" + str(date_y) + "_" + date_m
        save_name = to.split(' ')[0]
        os.makedirs(save_path, exist_ok=True)
        df.to_csv(save_path + '/' + str(save_name) + '.csv')

    except:
        print("None data : ", to)

def minutes_interval(ticker, interval, count, years) :
    over = False
    for year in years:
        print("current year : " + year)
        start_month = 10 if year == '2017' else 1

        for month in tqdm(range(start_month, 13)):
            last_day = calc_last_day(year, month)
            month = int_to_str(month)

            for day in range(1, last_day.day + 1):
                day = int_to_str(day)

                if str(datetime.date.today()) == f'{year}-{month}-{day}':
                    over = True
                    break

                coin_data(ticker, interval, count, year, date_m=month, date_d=day)

            if over == True:
                break

def year_interval(ticker, interval, count, years) :
    for year in years :
        print("current year : " + year)
        coin_data(ticker, interval, count, year)

if __name__ == '__main__' :

    parser = argparse.ArgumentParser()
    parser.add_argument('--ticker', type=str, required=True,
                        help='name of coin')
    parser.add_argument('--interval', type=str, required=True,
                        help='interval')

    args = parser.parse_args()

    ticker = args.ticker
    interval = args.interval
    count, interval_term = calc_count(interval)

    current_year = datetime.datetime.today().year
    years = [str(year) for year in range(2017, current_year + 1)]

    if interval_term == 'min' :
        minutes_interval(ticker, interval, count, years)

    else :
        year_interval(ticker, interval, count, years)


import datetime
import glob
import itertools
import os
import logging
import bottleneck as bn
import click
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from configobj import ConfigObj
from obspy import read, UTCDateTime, Stream
from obspy.signal.util import next_pow_2
from obspy.signal.filter import envelope as obspy_envelope

logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG)

# from click_plugins import with_plugins
def to_idds(net, sta, loc, chan, year, jday, hour):
    SDS="YEAR/NET/STA/CHAN.TYPE/JDAY/NET.STA.LOC.CHAN.TYPE.YEAR.JDAY.HOUR"
    file=SDS.replace('YEAR', "%04i"%year)
    file=file.replace('NET', net)
    file=file.replace('STA', sta)
    file=file.replace('LOC', loc)
    file=file.replace('CHAN', chan)
    file=file.replace('JDAY', jday)
    file = file.replace('HOUR', hour)
    file=file.replace('TYPE', "D")
    return file

def to_agung(net, sta, loc, chan, year, jday, hour):
    SDS="YEAR/NET/STA/CHAN.TYPE/JDAY/"
    file=SDS.replace('YEAR', "%04i"%year)
    file=file.replace('NET', net)
    file=file.replace('STA', sta)
    file=file.replace('LOC', loc)
    file=file.replace('CHAN', chan)
    file=file.replace('JDAY', jday)
    file = file.replace('HOUR', hour)
    file=file.replace('TYPE', "D")
    file += "NET.STA.*.*.*.JDAY.HOUR"
    file=file.replace('NET', net)
    file = file.replace('STA', sta)
    file = file.replace('YEAR', "%04i"%year)
    file = file.replace('JDAY', jday)
    file = file.replace('HOUR', hour)
    return file

def to_sds(net, sta, loc,chan, year, jday):
    SDS="YEAR/NET/STA/CHAN.TYPE/NET.STA.LOC.CHAN.TYPE.YEAR.JDAY"
    file=SDS.replace('YEAR', "%04i"%year)
    file=file.replace('NET', net)
    file=file.replace('STA', sta)
    file=file.replace('LOC', loc)
    file=file.replace('CHAN', chan)
    file=file.replace('JDAY', jday)
    file=file.replace('TYPE', "D")
    return file

@click.group()
@click.pass_context
def cli(ctx):
    config_file = os.path.join(os.getcwd(), "config.ini")
    config = ConfigObj(config_file)
    ctx.obj['config'] = config

    pass

@click.command()
@click.pass_context
def init(ctx):
    filelist = os.path.join(os.getcwd(), "filelist.csv")
    if os.path.isfile(filelist):
        logging.debug("Init seems to be already done")
        return
    df = pd.DataFrame([["0","0",datetime.date(2010,1,1),"0","1"],],
                      columns=["net","sta","date","size","process_step"],
                      index=["nopath",])

    df.to_csv(filelist, index_label="path")
    pass

@click.command()
@click.option('--path',  help='Scan all files in specific folder, overrides the'
                              ' default workflow step.')
@click.pass_context
def scan_archive(ctx, path):
    config = ctx.obj["config"]["msara"]
    filelist = os.path.join(os.getcwd(), "filelist.csv")
    filedb = pd.read_csv(filelist, index_col="path")

    startdate = UTCDateTime(config["startdate"])
    enddate = UTCDateTime(config["enddate"])
    data_folder = config["data_folder"]
    data_format = config["data_format"]
    network = config["network"]
    stations = config["stations"]
    location = config["location"]
    channel = config["channel"]
    bandpass = config["bandpass"]
    new_data = False

    if path:
        for root, dirs, _ in os.walk(path):
            for d in dirs:
                tmppath = os.path.join(root, d)
                _ = os.listdir(tmppath)
                if not len(_):
                    continue
                for file in sorted(_):
                    path = os.path.join(tmppath, file)
                    if os.path.isfile(path):
                        print(path)
                        st = read(path)
                        tr = st[0]
                        if tr.stats.station in stations:
                            if tr.stats.sampling_rate < (2* float(bandpass[1])):
                                print("Trace sps too small, skiping")
                                continue
                            if (tr.stats.endtime <= startdate) or (tr.stats.starttime >= enddate):
                                print("Data outside startdate-enddate")
                                continue
                            filesize = os.stat(path).st_size
                            if path in filedb.index:
                                tmp = filedb.loc[path]
                                if tmp["size"] != filesize:
                                    logging.debug(tmp["size"], filesize)
                                    logging.debug("file changed")
                                    filedb.loc[path, "size"] = filesize
                                    filedb.loc[path, "process_step"] = "E"
                                    new_data = True
                                else:
                                    logging.debug(
                                        "path is in the df already, and no size change")
                                    new_data = False
                            else:
                                logging.debug(
                                    "Adding file to the process list")
                                new = pd.Series({
                                    "net": tr.stats.network,
                                    "sta": tr.stats.station,
                                    "date": tr.stats.starttime.date,
                                    "size": filesize,
                                    "process_step": "E"
                                }, name=path)
                                filedb = filedb.append(new)
                                new_data = True
        if new_data:
            filedb.to_csv(filelist)

        return

    if data_format == "SDS":
        dates = pd.date_range(UTCDateTime(config["startdate"]).datetime,
                              UTCDateTime(config["enddate"]).datetime, freq="D")
    elif data_format in ["IDDS", "AGUNG"]:
        dates = pd.date_range(UTCDateTime(config["startdate"]).datetime,
                              UTCDateTime(config["enddate"]).datetime, freq="H")
    else:
        logging.debug("data_format %s not supported" % data_format)
        return

    for station in stations:
        for date in dates:

            if data_format == "SDS":
                path = to_sds(network, station, location, channel, date.year, "%03i"% date.dayofyear)
                path = os.path.join(data_folder, path)
            elif data_format == "IDDS":
                path = to_idds(network, station, location, channel, date.year, "%03i"% date.dayofyear, "%02i" % date.hour)
                path = os.path.join(data_folder, path)
            elif data_format == "AGUNG":
                path = to_agung(network, station, location, channel, date.year, "%03i"% date.dayofyear, "%02i" % date.hour)
                path = os.path.join(data_folder, path)
                path = glob.glob(path)
                if len(path) != 0:
                    path = path[0]
                else:
                    continue

            if not os.path.isfile(path):
                continue
            filesize = os.stat(path).st_size
            if path in filedb.index:
                tmp = filedb.loc[path]
                if tmp["size"] != filesize:
                    logging.debug(tmp["size"], filesize)
                    logging.debug("file changed")
                    filedb.loc[path, "size"] = filesize
                    filedb.loc[path, "process_step"] = "E"
                    new_data = True
                else:
                    logging.debug("path is in the df already, and no size change")
                    new_data = False
            else:
                logging.debug("Adding file to the process list")
                new = pd.Series({
                    "net": network,
                    "sta": station,
                    "date": date.date(),
                    "size": filesize,
                    "process_step": "E"
                }, name=path)
                filedb = filedb.append(new)
                new_data = True

    if new_data:
        filedb.to_csv(filelist)

@click.command()
@click.pass_context
def envelope(ctx):
    config = ctx.obj["config"]["msara"]
    data_folder = config["data_folder"]
    filelist = os.path.join(os.getcwd(), "filelist.csv")
    filedb = pd.read_csv(filelist, index_col="path")
    tmp = filedb[filedb.process_step == "E"]
    if len(tmp) == 0:
        logging.debug("No files to process")
        return

    logging.debug("Will process envelope for: %s files "% len(tmp))
    for file in tmp.index:
        if not os.path.isfile(file):
            logging.debug("%s NOT FOUND")
            continue

        outfile = file.replace(data_folder, os.path.join(os.getcwd(),"ENV"))
        logging.debug("will output to %s"%outfile)
        if not os.path.isdir(os.path.split(outfile)[0]):
            os.makedirs(os.path.split(outfile)[0])

        # PREPROCESS
        logging.debug("Loading")
        st = read(file)
        logging.debug("Detrend")
        st.detrend("demean")
        logging.debug("Merge")
        st.merge(method=1, fill_value="interpolate")
        logging.debug("Taper")
        st.taper(max_percentage=None, max_length=1.0)
        logging.debug("Filter")
        freqmin, freqmax = config["bandpass"]
        st.filter("bandpass", freqmin=float(freqmin), freqmax=float(freqmax),corners=8)
        logging.debug(st)
        trace = st[0]
        n = int(config["env_sampling_rate"])
        sps = int(trace.stats.sampling_rate)

        logging.debug("Envelope")
        from scipy.signal import hilbert
        N = next_pow_2(trace.stats.npts)
        trace.data = np.abs(hilbert(trace.data, N=N)[:trace.stats.npts])
        logging.debug("Moving Median")
        trace.data = bn.move_median(trace.data, sps * n)
        trace.data = trace.data[n * sps - 1::sps * n]
        trace.stats.sampling_rate = 1. / float(n)
        trace.data = np.require(trace.data, np.float32)
        logging.debug("Writing")
        trace.write(outfile, format="MSEED")
        del st, trace
        filedb.loc[file, "process_step"] = "R"

    filedb.to_csv(filelist)

@click.command()
@click.pass_context
def ratio(ctx):
    config = ctx.obj["config"]["msara"]
    data_folder = config["data_folder"]
    data_format = config["data_format"]
    filelist = os.path.join(os.getcwd(), "filelist.csv")
    filedb = pd.read_csv(filelist, index_col="path", parse_dates=["date",])
    tmp = filedb[filedb.process_step == "R"]
    if len(tmp) == 0:
        logging.debug("No files to process")
        return
    groups = filedb.groupby("date")
    for date, group in groups:
        # date = pd.datetime(date)
        if "R" not in group["process_step"].values:
            logging.debug("Nothing do to for %s"%date)
            continue

        logging.debug(date)
        traces = []
        stations = np.unique(sorted(group["sta"]))
        for station in stations:
            logging.debug("Will load %s" % station)
            if data_format == "SDS":
                path = to_sds("*", station, "*", "*", date.year,
                              "%03i" % date.dayofyear)
            elif data_format == "IDDS":
                path = to_idds("*", station, "*", "*", date.year,
                               "%03i" % date.dayofyear, "*")
            elif data_format == "AGUNG":
                path = to_agung("*", station, "*", "*", date.year,
                                "%03i" % date.dayofyear, "*")
            data_folder = os.path.join(os.getcwd(), "ENV")
            path = os.path.join(data_folder, path)
            logging.debug(path)
            for file in glob.glob(path):
                traces.append(read(file)[0])

        st = Stream(traces)
        stations = np.unique([tr.stats.station for tr in st])
        for sta1, sta2 in itertools.combinations(stations,2):
            st1 = st.select(station=sta1)
            st1.merge(method=1, fill_value="interpolate")

            st2 = st.select(station=sta2)
            st2.merge(method=1, fill_value="interpolate")

            max_start = max([tr.stats.starttime for tr in st1],[tr.stats.starttime for tr in st2])[0]
            min_end = min([tr.stats.endtime for tr in st1],
                            [tr.stats.endtime for tr in st2])[0]

            st1 = st1.trim(max_start, min_end)
            st2 = st2.trim(max_start, min_end)

            stR = st1[0].copy()
            stR.data = st1[0].data / st2[0].data
            stR.stats.station = "RATIO"
            pair = "%s_%s" % (sta1, sta2)
            env_output_dir = os.path.join(os.getcwd(), 'RATIO', pair)
            if not os.path.isdir(env_output_dir):
                os.makedirs(env_output_dir)
            stR.write(os.path.join(env_output_dir, str(date.date()) + '.MSEED'),
                        format="MSEED", encoding="FLOAT32")
            del stR
        for file in group.index:
            filedb.loc[file, "process_step"] = "P"
    filedb.to_csv(filelist)

@click.command()
@click.pass_context
def plot(ctx):
    config = ctx.obj["config"]["msara"]
    stations = config["stations"]
    smoothing = config["smoothing"]
    if type(smoothing) in [float, int, str]:
        smoothing = [smoothing,]
    print(smoothing)
    # plt.figure()
    # for sta1, sta2 in itertools.combinations(stations, 2):
    #     pair = "%s_%s" % (sta1, sta2)
    #     path = os.path.join(os.getcwd(), 'RATIO', pair, "*")
    #     st = read(path, format="MSEED")
    #     st.merge(method=1, fill_value="interpolate")
    #     tr = st[0]
    #     times = pd.date_range(tr.stats.starttime.datetime, tr.stats.endtime.datetime, freq="%ims"%(tr.stats.delta*1e3))
    #     plt.plot(times, tr.data, label=pair)
    # plt.legend()
    # plt.title("Raw Ratios")
    # plt.grid(True)

    for smooth in smoothing:
        plt.figure()
        for sta1, sta2 in itertools.combinations(stations, 2):
            pair = "%s_%s" % (sta1, sta2)
            path = os.path.join(os.getcwd(), 'RATIO', pair, "*")
            st = read(path, format="MSEED")
            st.merge(method=1, fill_value="interpolate")
            tr = st[0]
            sps_ratio = int(int(smooth) / tr.stats.delta)

            # tr.data = pd.rolling_median(tr.data, window=sps_ratio)
            idx = pd.date_range(tr.stats.starttime.datetime,
                                  tr.stats.endtime.datetime,
                                  freq="%ims" % (tr.stats.delta * 1e3))
            df = pd.Series(tr.data, index=idx)
            rs = df.resample("%iS" % int(smooth))
            xmedian = rs.median()
            xmean = rs.mean()
            xmean.to_csv("%s_%s_mean.csv" % (pair, smooth))
            xmedian.to_csv("%s_%s_median.csv" % (pair, smooth))
            plt.plot(xmedian.index, xmedian, label="%s - median" % pair)
            plt.plot(xmean.index, xmean, label="%s - mean" % pair)

        plt.legend()
        plt.title("Smoothed ratios (smoothing=%is)"%int(smooth))
        plt.grid(True)
    plt.show()

cli.add_command(init)
cli.add_command(scan_archive)
cli.add_command(envelope)
cli.add_command(ratio)
cli.add_command(plot)

def main():
    cli(obj={})


import requests
import os
from twilio.rest import Client

account_sid = os.environ.get("ACC_VAR")
auth_token = os.environ.get("AUTH_TOKE")
OWM_Endpoint = "https://api.openweathermap.org/data/3.0/onecall"
api_key = os.environ.get("WEATHER_API")

weather_params = {
    "lat": 54.95,
    "lon" : -7.73333,
    "appid" :api_key,
    "exclude" : "comment,minutely,daily"
    }

response = requests.get(OWM_Endpoint, weather_params)
response.raise_for_status()
weather_data = response.json()
weather_slice = weather_data["hourly"][:12]

will_rain = True

for hour_data in weather_slice:
    condition_code =(hour_data["weather"][0]["id"])
    if int(condition_code) < 700:
        will_rain = True

if will_rain:
    client = Client(account_sid, auth_token)

    message = client.messages.create(
    from_='+15417222051',
    body='You are a machine, better keep going',
    to='+353873407070'
    )

    print(message.sid)

# print(weather_data["hourly"][0]["weather"][0]["id"])


#!/usr/bin/env python
# vim: set fileencoding=utf-8 ts=4 sts=4 sw=4 et tw=80 :
#
# Extract sources from Spitzer images for UCD project.
#
# Rob Siverd
# Created:       2023-07-06
# Last modified: 2023-07-06
#--------------------------------------------------------------------------
#**************************************************************************
#--------------------------------------------------------------------------

## Logging setup:
import logging
#logging.basicConfig(level=logging.DEBUG)
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
#logger.setLevel(logging.DEBUG)
logger.setLevel(logging.INFO)

## Current version:
__version__ = "0.1.0"

## Python version-agnostic module reloading:
try:
    reload                              # Python 2.7
except NameError:
    try:
        from importlib import reload    # Python 3.4+
    except ImportError:
        from imp import reload          # Python 3.0 - 3.3

## Modules:
import os
import sys
import time
import numpy as np
from numpy.lib.recfunctions import append_fields
#from functools import partial
#from collections import OrderedDict
from collections.abc import Iterable
#import multiprocessing as mp
#np.set_printoptions(suppress=True, linewidth=160)
#import itertools as itt
_have_np_vers = float('.'.join(np.__version__.split('.')[:2]))

##--------------------------------------------------------------------------##

## Various from astropy:
try:
#    import astropy.io.ascii as aia
    import astropy.io.fits as pf
#    import astropy.table as apt
#    import astropy.time as astt
    import astropy.wcs as awcs
except ImportError:
    logger.error("astropy module not found!  Install and retry.")
#    sys.stderr.write("\nError: astropy module not found!\n")
    sys.exit(1)

### LACOSMIC cosmic ray removal:
#try:
#    from lacosmic import lacosmic
#except ImportError:
#    logger.error("failed to import lacosmic module!")
#    sys.exit(1)

##--------------------------------------------------------------------------##
## Storage structure for analysis results:
try:
    import extended_catalog
    reload(extended_catalog)
    ec = extended_catalog
except ImportError:
    logger.error("failed to import extended_catalog module!")
    sys.exit(1)

## Star extraction:
try:
    import easy_sep
    reload(easy_sep)
except ImportError:
    logger.error("easy_sep module not found!  Install and retry.")
    sys.stderr.write("Error: easy_sep module not found!\n\n")
    sys.exit(1)

### Spitzer pixel phase correction (cheesy version, IRAC ch1):
#try:
#    import spitz_pixphase
#    reload(spitz_pixphase)
#    iracfix = spitz_pixphase.IRACFix()
#except ImportError:
#    logger.error("failed to import spitz_pixphase module!")
#    sys.exit(1)

## WIRCam polynomial routines:
try:
    import wircam_poly
    reload(wircam_poly)
    wcp = wircam_poly.WIRCamPoly()
except ImportError:
    logger.error("failed to import wircam_poly module!")
    sys.exit(1)

##--------------------------------------------------------------------------##
##------------------   CFHT WIRCam Star Extraction Class    ----------------##
##--------------------------------------------------------------------------##

#_lacos_defaults = {
#                  'contrast'  :  12.0,
#              'cr_threshold'  :   5.0,
#        'neighbor_threshold'  :   4.0,
#        }

_wircam_defaults = {
                'minpixels'   :     5,
               'pix_origin'   :     1,  # MUST BE INTEGER
                'win_sigma'   :   2.0,
                'calc_wpos'   :  True,
        }

class WIRCamFind(object):

    def __init__(self):
        self._pse = easy_sep.EasySEP()
        self._pse.set_options_test(**_wircam_defaults)
        # science image:
        self._reset_iparams()
        # cosmic ray cleaning:
        self._cdata = None
        self._cmask = None
        # uncertainty image:
        self._reset_uparams()
        # miscellany:
        self._confirmed = False
        return

    # Re-initialize image values:
    def _reset_iparams(self):
        self._ipath = None
        self._idata = None
        self._ihdrs = None
        self._imwcs = None
        self._imask = None
        return

    # Re-initialize error-image values:
    def _reset_uparams(self):
        self._upath = None
        self._udata = None
        self._uhdrs = None
        self._have_err_image = False
        return

    # Reset everything:
    def _reset_everything(self):
        self._reset_iparams()
        self._reset_uparams()
        return

    # ----------------------------------------

    def set_pse_options(self, **kwargs):
        return self._pse.set_options_test(**kwargs)
        #return self._pse.set_options(**kwargs)

    # ----------------------------------------

    def use_images(self, ipath=None, upath=None):
        """Load images for analysis. Inputs:
        ipath   --  path to image for analysis
        upath   --  path to uncertainty image
        """
        #self._confirmed = False
        self._reset_everything()

        # data image:
        if ipath:
            logger.info("Loading data image %s" % ipath)
            try:
                self._idata, self._ihdrs = self._get_data_and_header(ipath)
                self._imask = np.isnan(self._idata)
                self._imwcs = awcs.WCS(self._ihdrs)
                self._ipath = ipath
                self._pse.set_image(self._idata, _docopy=False)
                self._pse.set_mask(self._imask)
                self._pse.set_imwcs(self._imwcs.all_pix2world)
            except:
                logger.error("Failed to load file: %s" % ipath)
                #self._ipath, self._idata, self._ihdrs = None, None, None
                self._reset_iparams()

        # error image:
        if upath:
            logger.info("Loading error image %s" % upath)
            try:
                self._udata, self._uhdrs = self._get_data_and_header(upath)
                self._upath = upath
                self._have_err_image = True
                self._pse.set_errs(self._udata, _docopy=False)
            except:
                logger.error("Failed to load file: %s" % ipath)
                #self._upath, self._udata, self._uhdrs = None, None, None
                #self._have_err_image = False
                self._reset_uparams()
        return

    @staticmethod
    def _get_data_and_header(filename):
        rdata, rhdrs = pf.getdata(filename, header=True)
        return rdata.astype('float32'), rhdrs.copy(strip=True)

    ## include distortion corrected pixel positions:
    #def _wir_added_value(self, dataset, wparams):
    #    filt = self._ihdrs['FILTER']
    #    #crval1 = self._ihdrs['CRVAL1']
    #    #crval2 = self._ihdrs['CRVAL2']
    #    cdmat  = wparams[:4]
    #    crval1 = wparams[4]
    #    crval2 = wparams[5]

    #    # adjusted standard positions:
    #    tx = np.atleast_1d(dataset['x'])
    #    ty = np.atleast_1d(dataset['y'])
    #    sx_dewarp, sy_dewarp = wcp.calc_corrected_xy(tx, ty, filt)
    #    pred_ra, pred_de = wcp.predicted_radec(None, tx, ty, crval1, crval2)
    #    dataset = append_fields(dataset, ('xdw', 'ydw', 'dradw', 'ddedw'),
    #            (sx_dewarp, sy_dewarp, pred_ra, pred_de), usemask=False)

    #    # adjusted windowed positions:
    #    tx = np.atleast_1d(dataset['wx'])
    #    ty = np.atleast_1d(dataset['wy'])
    #    wx_dewarp, wy_dewarp = wcp.calc_corrected_xy(tx, ty, filt)
    #    pred_ra, pred_de = wcp.predicted_radec(None, tx, ty, crval1, crval2)
    #    dataset = append_fields(dataset, ('wxdw', 'wydw', 'wdradw', 'wddedw'),
    #            (wx_dewarp, wy_dewarp, pred_ra, pred_de), usemask=False)

    #    # also try calculating
    #    return dataset

    def _wir_distortion_correction(self, dataset):
        # the models we know about:
        _have_models = wcp.get_dist_models()

        # adjust non-windowed positions:
        tx = np.atleast_1d(dataset['x'])
        ty = np.atleast_1d(dataset['y'])
        for model in _have_models:
            xnudge, ynudge = wcp.calc_xy_nudges(tx, ty, model)
            xcorr = tx + xnudge
            ycorr = ty + ynudge
            xname = 'xdw_%s' % model
            yname = 'ydw_%s' % model
            dataset = append_fields(dataset, (xname, yname),
                    (xcorr, ycorr), usemask=False)
            pass

        # adjust windowed positions:
        tx = np.atleast_1d(dataset['wx'])
        ty = np.atleast_1d(dataset['wy'])
        for model in _have_models:
            xnudge, ynudge = wcp.calc_xy_nudges(tx, ty, model)
            xcorr = tx + xnudge
            ycorr = ty + ynudge
            xname = 'wxdw_%s' % model
            yname = 'wydw_%s' % model
            dataset = append_fields(dataset, (xname, yname),
                    (xcorr, ycorr), usemask=False)
            pass

        # return updated catalog:
        return dataset

    # a more sophisticated "hot row" detector/corrector:
    def _prune_hot_rows(self, dataset, thresh):
        cleaned  = np.copy(dataset)
        rpk_pop  = np.bincount(dataset['ypeak'])
        row_num  = np.arange(rpk_pop.size)
        bad_rows = row_num[(rpk_pop >= thresh)]
        for rr in bad_rows:
            drop_me = (cleaned['ypeak'] == rr)
            cleaned = cleaned[~drop_me]
        return cleaned

    # include filter as part of instrument tag:
    def _append_instrument_tag(self, dataset):
        filt = self._ihdrs['FILTER']
        itag = 'wircam_%s' % filt
        nsrc = len(dataset)
        dataset = append_fields(dataset, ('filter', 'instrument'),
                (nsrc*[filt], nsrc*[itag]), usemask=False)
        return dataset

    # ----------------------------------------
    def find_stars(self, thresh, keepall=False, use_err_img=True,
            prune_horiz=True, include_poly=False, **fskw):
        """Driver routine for star extraction. Required inputs:
        thresh       --  significance threshold for star extraction
        keepall      --  keep all detections (skip pruning)
        use_err_img  --  False disables use of error-image
        prune_horiz  --  eliminate objects spanning 1 pixel in Y (hot rows)
        include_poly --  include polynomial distortion correction(s)

        Results are reported in an ExtendedCatalog container.
        """
        _err_mode = use_err_img and self._have_err_image
        self._pse.analyze(thresh, rel_err=_err_mode)
        dataset = self._pse.allobjs if keepall else self._pse.useobjs

        # create instrument tag for this data set:
        dataset = self._append_instrument_tag(dataset)

        # snip out single-row detections:
        if prune_horiz:
            n_dirty = len(dataset)
            dataset = self._prune_hot_rows(dataset, thresh=10)
            n_clean = len(dataset)
            #horiz   = dataset['ymin'] == dataset['ymax']
            #dataset = dataset[~horiz]

        # tack on pixel-phase corrected coordinates:
        pix_origin = self._pse.settings['pix_origin']
        #ppx, ppy = iracfix.fix_centroid(dataset['x'], dataset['y'])
        #ppra, ppde = self._pse._wcs_func(ppx, ppy, pix_origin)
        #dataset = append_fields(dataset, ('ppx', 'ppy', 'ppdra', 'ppdde'),
        #        (ppx, ppy, ppra, ppde), usemask=False)

        # distortion correction:
        if include_poly:
            dataset = self._wir_distortion_correction(dataset)
        #if include_poly:
        #    wparams = fskw.pop('wpars')
        #    sys.stderr.write("Got wparams: %s\n" % str(wparams))
        #    dataset = self._wir_added_value(dataset, wparams)

        ## Adam Kraus polynomial dephase/dewarp:
        #if include_akp:
        #    #success = self._akp_added_value(dataset)
        #    dataset = self._akp_added_value(dataset)

        #channel  = self._ihdrs['CHNLNUM']        # SHA provides this in FITS header
        #jdtdb    = self._ihdrs['OBS_TIME']       # mid-exposure JDTDB, added to headers previously
        #mission  = akspoly.mission_from_jdtdb(jdtdb)
        #aks_inst = akspoly._chmap[channel]
        #x_dephase, y_dephase = akp.dephase(np.atleast_1d(dataset['x']),
        #                                np.atleast_1d(dataset['y']), mission, aks_inst)
        #x_dewarp, y_dewarp = akp.xform_xy(x_dephase, y_dephase, aks_inst)
        #dataset = append_fields(dataset, ('xdp', 'ydp', 'xdw', 'ydw'),
        #        (x_dephase, y_dephase, x_dewarp, y_dewarp), usemask=False)

        ## sky coords from Adam Kraus polynomials:
        #this_padeg = self._ihdrs['PA']
        #this_crv_1 = self._ihdrs['CRVAL1']
        #this_crv_2 = self._ihdrs['CRVAL2']
        #aks_ra, aks_de = akspoly.xy2radec(this_padeg, x_dewarp, y_dewarp,
        #        this_crv_1, this_crv_2, aks_inst)
        #dataset = append_fields(dataset, ('akra', 'akde'),
        #        (aks_ra, aks_de), usemask=False)

        # encapsulate result:
        ecopts = {'name':os.path.basename(self._ipath), 'header':self._ihdrs}
        if self._have_err_image:
            ecopts['uname']   = os.path.basename(self._upath)
            ecopts['uheader'] = self._uhdrs
        result = ec.ExtendedCatalog(data=dataset, **ecopts)
        return result

##--------------------------------------------------------------------------##
##--------------------------------------------------------------------------##

######################################################################
# CHANGELOG (wircam_extract.py):
#---------------------------------------------------------------------
#
#  2023-07-06:
#     -- Increased __version__ to 0.1.0.
#     -- First created wircam_extract.py.
#


import argparse
import time
import torch
from typing import *
from util import initialize_seq2seq_attention_params, build_seq2seq_attention_model, load_data
from time import time
from torch import optim
from core import Seq2SeqAttentionModel, encode_all
from math import exp
from core import SOS_token, EOS_token
from operator import itemgetter

def decode(prev_hidden: torch.tensor, source_hiddens: torch.tensor, prev_context: torch.tensor,
           input: int, model: Seq2SeqAttentionModel) -> (
        torch.tensor, torch.tensor, torch.tensor, torch.tensor):
    """ Run the decoder AND the output layer for a single step.

    :param: prev_hidden: tensor of shape [L, hidden_dim] - the decoder's previous hidden state, denoted H^{dec}_{t-1}
                          in the assignment
    :param: source_hiddens: tensor of shape [source sentence length, L, hidden_dim] - the encoder's hidden states,
                            denoted H^{enc}_1 ... H^{enc}_T in the assignment
    :param: prev_context: tensor of shape [hidden_dim], denoted c_{t-1} in the assignment
    :param input: int - the word being inputted to the decoder.
                            during log-likelihood computation, this is y_{t-1}
                            during greedy decoding, this is yhat_{t-1}
    :param model: a Seq2SeqAttention model
    :return: (1) a tensor `probs` of shape [target_vocab_size], denoted p(y_t | x_1 ... x_S, y_1 .. y_{t-1})
             (2) a tensor `hidden` of shape [L, hidden_size], denoted H^{dec}_t in the assignment
             (3) a tensor `context` of shape [hidden_size], denoted c_t in the assignment
             (4) a tensor `attention_weights` of shape [source_sentence_length], denoted \alpha in the assignment
    """

    composition_1=torch.cat((model.target_embedding_matrix[input],prev_context),dim=0)
    h=model.decoder_gru.forward(composition_1,prev_hidden)
    alpha=model.attention.forward(source_hiddens[:,-1,:],h[-1])
    c_t=torch.sum(torch.mm(torch.unsqueeze(alpha, 0), source_hiddens[:,-1,:]),0)
    composition_2=torch.cat((h[-1],c_t),dim=0)
    pd=model.output_layer.forward(composition_2)
    return pd,h,c_t,alpha

def log_likelihood(source_sentence: List[int],
                   target_sentence: List[int],
                   model: Seq2SeqAttentionModel) -> torch.Tensor:
    """ Compute the log-likelihood for a (source_sentence, target_sentence) pair.

    :param source_sentence: the source sentence, as a list of words
    :param target_sentence: the target sentence, as a list of words
    :return: log-likelihood of the (source_sentence, target_sentence) pair
    """
    res=0
    source_h=encode_all(source_sentence, model)
    prev_h=source_h[-1]
    c_0=torch.zeros((model.hidden_dim))
    for i in range(len(target_sentence)):
        if i==0:
            pd, prev_h,c_t,_=decode(prev_h,source_h,c_0,SOS_token,model)
            p=pd[target_sentence[i]]
        else:
            pd, prev_h,c_t,_=decode(prev_h,source_h,c_t,target_sentence[i-1],model)
            p=pd[target_sentence[i]]
        res+=torch.log(p)
    return res

def perplexity(sentences: List[Tuple[List[int], List[int]]], model: Seq2SeqAttentionModel) -> float:
    """ Compute the perplexity of an entire dataset under a seq2seq model.  Refer to the write-up for the
    definition of perplexity.

    :param sentences: list of (source_sentence, target_sentence) pairs
    :param model: seq2seq attention model
    :return: perplexity of the translation
    """
    LL=0
    T=0
    for i in range(len(sentences)):
        LL+=log_likelihood(sentences[i][0],sentences[i][1],model)
        T+=len(sentences[i][1])
    return torch.exp(-LL/T).detach()

def translate_greedy_search(source_sentence: List[int],
                            model: Seq2SeqAttentionModel, max_length=10) -> (List[int], torch.tensor):
    """ Translate a source sentence using greedy decoding.

    :param source_sentence: the source sentence, as a list of words
    :param model: a Seq2Seq model
    :param max_length: the maximum length that the target sentence could be
    :return: (1) the translated sentence as a list of ints
             (2) the attention matrix, a tensor of shape [target_sentence_length, source_sentence_length]

    """
    res=[]
    source_h=encode_all(source_sentence, model)
    prev_h=source_h[-1]
    c_0=torch.zeros((model.hidden_dim))
    for i in range(max_length):
        if i==0:
            pd, prev_h,c_t,alpha=decode(prev_h,source_h,c_0,SOS_token,model)
            y_hat=torch.argmax(pd)
            res.append(y_hat.item())
            atten=alpha
        else:
            pd, prev_h,c_t,alpha=decode(prev_h,source_h,c_t,y_hat,model)
            y_hat=torch.argmax(pd)
            res.append(y_hat.item())
            atten=torch.vstack((atten,alpha))
        if y_hat==EOS_token:
            break
    return res, torch.Tensor(atten)

def translate_beam_search(source_sentence: List[int], model: Seq2SeqAttentionModel,
                          beam_width: int, max_length=10) -> Tuple[List[int], float]:
    """ Translate a source sentence using beam search decoding.

    :param source_sentence: the source sentence, as a list of words
    :param model: a Seq2Seq model
    :param beam_width: the number of translation candidates to keep at each time step
    :param max_length: the maximum length that the target sentence could be
    :return: (1) the target sentence (translation),
             (2) sum of conditional log-likelihood of the translation, i.e., log p(target sentence|source sentence)
    """
    if beam_width==1:
        res=translate_greedy_search(source_sentence,model,max_length)[0]
        ll=log_likelihood(source_sentence,res,model)
        return res,ll
    count=0
    res_final=[]
    res=[]
    source_h=encode_all(source_sentence, model)
    prev_h=source_h[-1]
    c_0=torch.zeros((model.hidden_dim))
    while count < max_length:
        if beam_width==0:
            break
        if count==0:
            pd, prev_h,c_t,_=decode(prev_h,source_h,c_0,SOS_token,model)
            temp=torch.topk(pd,beam_width)
            for i in range(beam_width):
                res.append([[temp.indices[i].item()],prev_h,c_t,torch.log(temp.values[i]).item()])
        else:
            res_temp=[]
            for d in res:
                pd, prev_h,c_t,_=decode(d[1],source_h,d[2],d[0][-1],model)
                temp=torch.topk(pd,beam_width)
                for i in range(beam_width):
                    idx=d[0][:]
                    idx.append(temp.indices[i].item())
                    res_temp.append([idx,prev_h,c_t,((d[3]*count+torch.log(temp.values[i]))/(count+1)).item()])
            res_temp.sort(key=lambda x: x[3],reverse=True)
            res=res_temp[:beam_width]
        for d in res:
            if d[0][-1]==EOS_token:
                res_final.append(d)
                beam_width-=1
        count+=1
    #res_final+=res
    res=max(res_final, key=lambda x: x[3])
    return res[0], res[3]*len(res[0])

def train_epoch(sentences: List[Tuple[List[int], List[int]]], model: Seq2SeqAttentionModel,
                epoch: int, print_every: int = 100, learning_rate: float = 0.0001, gradient_clip=5):
    """ Train the model for an epoch.

    :param sentences: list of (source_sentence, target_sentence) pairs
    :param model: a Seq2Seq model
    :param epoch: which epoch we're at
    """
    print("epoch\titer\tavg loss\telapsed secs")
    total_loss = 0
    start_time = time()
    optimizer = optim.Adam(model_params.values(), lr=learning_rate)
    for i, (source_sentence, target_sentence) in enumerate(sentences):
        optimizer.zero_grad()
        theloss = -log_likelihood(source_sentence, target_sentence, model)
        total_loss += theloss
        theloss.backward()

        torch.nn.utils.clip_grad_norm_(model_params.values(), gradient_clip)

        optimizer.step()

        if i % print_every == 0:
            avg_loss = total_loss / print_every
            total_loss = 0
            elapsed_secs = time() - start_time
            print("{}\t{}\t{:.3f}\t{:.3f}".format(epoch, i, avg_loss, elapsed_secs))

    return model_params

def print_translations(sentences: List[Tuple[List[int], List[int]]], model: Seq2SeqAttentionModel,
                       source_vocab: Dict[int, str], target_vocab: Dict[int, str], beam_width):
    """ Iterate through a dataset, printing (1) the source sentence, (2) the actual target sentence, and (3)
    the translation according to our model.

    :param sentences: a list of (source sentence, target sentence) pairs
    :param model: a Seq2Seq model
    :param source_vocab: the mapping from word index to word string, in the source language
    :param target_vocab: the mapping from word index to word string, in the target language
    """
    for (source_sentence, target_sentence) in sentences:
        if beam_width > 0:
            translation, _ = translate_beam_search(source_sentence, model, beam_width)
        else:
            translation, _ = translate_greedy_search(source_sentence, model)

        print("source sentence:" + " ".join([source_vocab[word] for word in source_sentence]))
        print("target sentence:" + " ".join([target_vocab[word] for word in target_sentence]))
        print("translation:\t" + " ".join([target_vocab[word] for word in translation]))
        print("")

def beam_search_ll(sentences: List[Tuple[List[int], List[int]]], model: Seq2SeqAttentionModel,
                   beam_width: int) -> float:
    """ Computes the average of conditional log-likelihood of beam search translation.

    :param sentences: list of (source_sentence, target_sentence) pairs
    :param model: a Seq2Seq model
    :param beam_width: the number of translation candidates to keep at each time step
    :return: the average of the sum of conditional log-likelihood of the translations,
             i.e., sum_i (log p(target sentence i|source sentence i)) / len(sentences)
    """
    assert beam_width > 0, "beam_width must be set positive"

    ll_sum = 0.0
    total_cnt = 0
    for (source_sentence, target_sentence) in sentences:
        _, ll = translate_beam_search(source_sentence, model, beam_width)
        ll_sum += ll
        total_cnt += 1
    return ll_sum / total_cnt

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Seq2Seq Homework Assignment')
    parser.add_argument("action", type=str,
                        choices=["train", "finetune", "train_perplexity", "test_perplexity", "test_beam_search_ll",
                                 "print_train_translations", "print_test_translations", "visualize_attention"])
    parser.add_argument("--beam_width", type=int, default=-1,
                        help="number of translation candidates to keep at each time step. "
                             "this option is used for beam search translation (greedy search decoding is used by default).")
    parser.add_argument("--load_model", type=str,
                        help="path to saved model on disk.  if this arg is unset, the weights are initialized randomly")
    parser.add_argument("--save_model_prefix", type=str, help="prefix to save model with, if you're training")
    args = parser.parse_args()

    # load train/test data, and source/target vocabularies
    train_sentences, test_sentences, source_vocab, target_vocab = load_data()

    # load model weights (if path is specified) or else initialize weights randomly
    model_params = initialize_seq2seq_attention_params() if args.load_model is None \
        else torch.load(args.load_model)  # type: Dict[str, torch.Tensor]

    # build a Seq2SeqAttentionModel object
    model = build_seq2seq_attention_model(model_params)  # type: Seq2SeqAttentionModel

    if args.action == 'train':
        for epoch in range(10):
            train_epoch(train_sentences, model, epoch)
            torch.save(model_params, '{}_{}.pth'.format(args.save_model_prefix, epoch))
    elif args.action == 'finetune':
        train_epoch(train_sentences[:1000], model, 0, learning_rate=1e-5)
        torch.save(model_params, "{}.pth".format(args.save_model_prefix))
    elif args.action == "print_train_translations":
        print_translations(train_sentences, model, source_vocab, target_vocab, args.beam_width)
    elif args.action == "print_test_translations":
        print_translations(test_sentences, model, source_vocab, target_vocab, args.beam_width)
    elif args.action == "train_perplexity":
        print("perplexity: {}".format(perplexity(train_sentences[:1000], model)))
    elif args.action == "test_perplexity":
        print("perplexity: {}".format(perplexity(test_sentences, model)))
    elif args.action == "test_beam_search_ll":
        print("beam width: {} average LL: {}".format(args.beam_width,
                                                     beam_search_ll(test_sentences[:100], model, args.beam_width)))
    elif args.action == "visualize_attention":
        from plotting import visualize_attention

        # visualize the attention matrix for the first 5 test set sentences
        for i in range(5):
            source_sentence = test_sentences[i][0]
            predicted_sentence, attention_matrix = translate_greedy_search(source_sentence, model)
            source_sentence_str = [source_vocab[w] for w in source_sentence]
            predicted_sentence_str = [target_vocab[w] for w in predicted_sentence]
            visualize_attention(source_sentence_str, predicted_sentence_str,
                                attention_matrix.detach().numpy(), "images/{}.png".format(i))


from tellr.db import db
from tellr.models.question_category import QuestionCategoryModel
from sqlalchemy.sql.expression import func

Model = db.Model

class QuestionModel(Model):

    __tablename__ = "questions"

    id = db.Column(db.Integer, primary_key=True)
    content = db.Column(db.String(50), nullable=False, unique=True)
    answers = db.relationship(
        "AnswerModel", lazy="dynamic", uselist=True, cascade="all, delete-orphan"
    )
    closed = db.Column(db.Boolean, default=False)

    category_id = db.Column(
        db.Integer, db.ForeignKey("question_categories.id"), nullable=False
    )
    category = db.relationship("QuestionCategoryModel")

    def save_to_db(self):
        db.session.add(self)
        db.session.commit()

    def delete_from_db(self):
        db.session.delete(self)
        db.session.commit()

    @classmethod
    def find_by_id(cls, _id):
        return cls.query.filter_by(id=_id).first()

    @classmethod
    def get_questions(cls):
        return cls.query.all()

    @classmethod
    def find_random(cls, list):
        return cls.query.filter(~(cls.id.in_(list))).order_by(func.random()).first()


from sqlalchemy.orm import relationship

from aps_app import db
import datetime
from sqlalchemy import Column, Integer, DateTime, Date, String, Boolean, Float, Text, JSON, ForeignKey

class Dogs(db.Model):

    __tablename__ = "dogs"

    dog_id = Column(Integer, primary_key=True, unique=True, autoincrement=True)
    coordinator_id = Column(Integer, ForeignKey("users.user_id"), nullable=False)
    name = Column(String(20), nullable=False)
    birthdate = Column(Date, nullable=False)
    sex = Column(String(1), nullable=False)
    breed = Column(String(100), nullable=False, default="Unknown")
    size = Column(String(20), nullable=False)
    color = Column(String(100), nullable=False)
    house_trained = Column(Boolean, nullable=False)
    good_with_kids = Column(Boolean, nullable=False)
    intake_date = Column(Date, default=lambda: datetime.date.today())
    adoption_fee = Column(Float, nullable=False)
    status = Column(Integer, nullable=False, default=1)
    display_status = Column(Integer, nullable=False, default=0)
    data = Column(JSON, nullable=False)
    created_at = Column(DateTime, default=lambda: datetime.datetime.utcnow())

    coordinator = relationship("Users")

    @classmethod
    def create(cls, coordinator_id, name, birthdate, sex, breed, color, size, house_trained, good_with_kids, intake_date, adoption_fee, status, display_status, data):
        new_dog = Dogs(
            coordinator_id=coordinator_id,
            name=name,
            birthdate=birthdate,
            sex=sex,
            breed=breed,
            color=color,
            size=size,
            house_trained=house_trained,
            good_with_kids=good_with_kids,
            intake_date=intake_date,
            adoption_fee=adoption_fee,
            status=status,
            display_status=display_status,
            data=data
        )

        db.session.add(new_dog)
        db.session.commit()

        return new_dog

    @classmethod
    def __repr__(cls):
        return f'<Dog: {cls.name}, Age: {cls.age}, Sex: {cls.sex}, ID: {cls.dog_id}>'  # function ensures readable print statement for debugging

class DogImages(db.Model):

    __tablename__ = "dog_images"

    image_id = Column(Integer, primary_key=True, unique=True, autoincrement=True)
    dog_id = Column(Integer, ForeignKey("dogs.dog_id"), nullable=False)
    asset_id = Column(String(100), nullable=False)  # NOT SURE IF THIS LENGTH IS OK
    image_url = Column(Text, nullable=False)
    original_filename = Column(Text, nullable=False)
    format = Column(String(20), nullable=False)
    public_id = Column(Text, nullable=False)
    secure_url = Column(Text, nullable=False)
    signature = Column(Text, nullable=False)
    created_at = Column(DateTime, nullable=False)

    dog = relationship("Dogs")

    @classmethod
    def store_dog_image(cls, dog_id, image):
        new_img = DogImages(
            dog_id=dog_id,
            asset_id=image["asset_id"],
            image_url=image["image_url"],
            original_filename = image["original_filename"],
            format=image["format"],
            public_id=image["public_id"],
            secure_url=image["secure_url"],
            signature=image["signature"],
            created_at=image["created_at"],
        )

        db.session.add(new_img)
        db.session.commit()

        return new_img