import xadmin
from .models import UserCourse,UserCoupon

class UserCourseAdmin(object):

    list_display = ['name','user', 'course','add_time']
    search_fields = ['name','user', 'course']
    list_filter = ['name','user', 'course','add_time']

class UserCouponAdmin(object):
    list_display = ['user','coupon','add_time']
    search_fields = ['user','coupon']
    list_filter = ['user','coupon','add_time']

xadmin.site.register(UserCourse,UserCourseAdmin)
xadmin.site.register(UserCoupon,UserCouponAdmin)


# create a 300x300 canvas.
# create a square drawing function that takes 2 parameters:
# the square size, and the fill color,
# and draws a square of that size and color to the center of the canvas.
# create a loop that fills the canvas with rainbow colored squares.

from tkinter import *
import random

def draw_rainbow(a, steps): #a = size of the centered square

    root = Tk()

    w = 300
    h = 300

    canvas = Canvas(root, width=w, height=h)
    canvas.pack()

    m = w/2
    b = a

    for i in range(((w-a)//steps), -1, -1):
        a = a - i*steps
        b = a
        RR=("%02x"%random.randint(0,255))
        GG=("%02x"%random.randint(0,255))
        BB=("%02x"%random.randint(0,255))
        ge="#"
        color=ge+RR+GG+BB
        line = canvas.create_polygon(m-(a/2), m-(b/2), m+(a/2), m-(b/2), m+(a/2), m+(b/2), m-(a/2), m+(b/2), fill=color)
    root.mainloop()

draw_rainbow(50, 6)


import tkinter as tk  # graphical user interface
from tkinter import *
import easygui  # to open the filebox
import cv2  # for image processing
import matplotlib.pyplot as plt  # for visualization and plotting

import os  # to read and save path
import sys

# Making the GUI main window
top = tk.Tk()
top.geometry("450x450")
top.title("Toonify Your Image")
top.configure(background="gray")
label = Label(top, background="#cdcdcd", font=("ubuntu mono", 20, "bold"))

def upload():
    image_path = easygui.fileopenbox()
    toonify(image_path)

# the below code will covert our imported image into a numpy array
def toonify(image_path):
    # read_image
    original_image = cv2.imread(image_path)
    original_image = cv2.cvtColor(original_image, cv2.COLOR_BGR2RGB)
    print(original_image)  # this will be stored in form of numbers

    if original_image is None:
        print("Can't find any image. Choose approciate file")
        sys.exit()

    # resizzing the image after each transformation
    resize_image1 = cv2.resize(original_image, (960, 540))
    # plt.imshow(resize_image1, cmap="gray")

    # Image transformation to grayscale
    grayscale_image = cv2.cvtColor(original_image, cv2.COLOR_BGR2GRAY)

    resize_image2 = cv2.resize(grayscale_image, (960, 540))
    # plt.imshow(resize_image2, cmap="gray")

    # applying median blur to smoothen an image
    smooth_grayscale_image = cv2.medianBlur(grayscale_image, 5)

    resize_image3 = cv2.resize(smooth_grayscale_image, (960, 540))
    # plt.imshow(resize_image3, cmap="gray")

    # Extracting the edges in the image (Highlighted Edges)
    get_edge = cv2.adaptiveThreshold(
        smooth_grayscale_image, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 9, 9
    )

    resize_image4 = cv2.resize(get_edge, (960, 540))
    # plt.imshow(resize_image4, cmap="gray")

    # applying bilateral filter to remove noise and keep edge sharp as required
    color_image = cv2.bilateralFilter(original_image, 9, 300, 300)

    resize_image5 = cv2.resize(color_image, (960, 540))
    # plt.imshow(resize_image5, cmap="gray")

    # Giving a Cartoon Effect
    cartoon_image = cv2.bitwise_and(color_image, color_image, mask=get_edge)

    resize_image6 = cv2.resize(cartoon_image, (960, 540))
    # plt.imshow(resize_image6, cmap="gray")

    # Plotting all the transitions together
    images = [
        resize_image1,
        resize_image2,
        resize_image3,
        resize_image4,
        resize_image5,
        resize_image6,
    ]
    fig, axes = plt.subplots(
        3,
        2,
        figsize=(8, 8),
        subplot_kw={"xticks": [], "yticks": []},
        gridspec_kw=dict(hspace=0.1, wspace=0.1),
    )
    for i, ax in enumerate(axes.flat):
        ax.imshow(images[i], cmap="gray")

    # Making a Save button in the GUI main window
    save1 = Button(
        top,
        text="save an image",
        command=lambda: save(resize_image6, image_path),
        padx=30,
        pady=5,
    )
    save1.configure(
        background="#364153", foreground="white", font=("ubuntu mono", 20, "bold")
    )
    save1.pack(side=TOP, pady=50)

    # save button code
    plt.show()

# Enhancing the save button
def save(resize_image6, image_path):
    # saving an image using imwrite function
    new_name = "cartoonified_Image"
    path1 = os.path.dirname(image_path)
    extension = os.path.splitext(image_path)[1]
    path = os.path.join(path1, new_name + extension)
    cv2.imwrite(path, cv2.cvtColor(resize_image6, cv2.COLOR_RGB2BGR))
    I = "Image saved by name " + new_name + " at " + path
    tk.messagebox.showinfo(title=None, message=I)

# Making the Cartoonify button in the GUI main window
upload = Button(top, text="Cartoonify an Image", command=upload, padx=10, pady=5)
upload.configure(
    background="#374256", foreground="wheat", font=("ubuntu mono", 10, "bold")
)
upload.pack(side=TOP, pady=50)

# Main function to build the GUI window
top.mainloop()


import tempfile
from typing import Dict, Iterable, List, Tuple
from overrides import overrides

import torch

import allennlp
from allennlp.data import DataLoader, DatasetReader, Instance, Vocabulary
from allennlp.data.fields import LabelField, TextField, MetadataField
from allennlp.data.token_indexers import TokenIndexer, SingleIdTokenIndexer
from allennlp.data.tokenizers import Token, Tokenizer, WhitespaceTokenizer
from allennlp.models import Model
from allennlp.modules import TextFieldEmbedder, Seq2VecEncoder
from allennlp.modules.text_field_embedders import BasicTextFieldEmbedder
from allennlp.modules.token_embedders import Embedding
from allennlp.modules.seq2vec_encoders import BagOfEmbeddingsEncoder, CnnEncoder

'''transformer stuff'''
from allennlp.data.tokenizers import PretrainedTransformerTokenizer
from allennlp.data.token_indexers import PretrainedTransformerIndexer
# from allennlp.modules.text_field_embedders import
from allennlp.modules.token_embedders import PretrainedTransformerEmbedder
from allennlp.modules.seq2vec_encoders import BertPooler

from allennlp.nn import util
from allennlp.training.metrics import CategoricalAccuracy, Auc
from allennlp.training.optimizers import AdamOptimizer
from allennlp.training.trainer import Trainer, GradientDescentTrainer
from allennlp.training.util import evaluate

# import the regularization
from allennlp.nn.regularizers import L2Regularizer, RegularizerApplicator

import pandas as pd
import os
import gc
from tqdm.auto import tqdm
import sys
sys.path.append("/scratch/gobi1/johnchen/new_git_stuff/multimodal_fairness/")
from src.preprocessing.text_preprocessing import preprocess_mimic

'''
We should throw out the X, where X is not good
'''

'''
Reimplementation of the AUC metric. However, we are simply not calling it correctly.
We need to actually do it in the forward pass
'''
import logging
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
logger.debug("hello")

@DatasetReader.register("MortalityReader")
class MortalityReader(DatasetReader):
    def __init__(self,
                 lazy: bool = True,
                 tokenizer: Tokenizer = None,
                 token_indexers: Dict[str, TokenIndexer] = None,
                 max_tokens: int = 768*4,
                 listfile: str = "/scratch/gobi1/johnchen/new_git_stuff/multimodal_fairness/data/in-hospital-mortality/train/listfile.csv",
                 notes_dir: str = "/scratch/gobi1/johnchen/new_git_stuff/multimodal_fairness/data/extracted_notes",
                 skip_patients_file: str ="/scratch/gobi1/johnchen/new_git_stuff/multimodal_fairness/data/extracted_notes/null_patients.txt",
                 stats_write_dir: str="/scratch/gobi1/johnchen/new_git_stuff/multimodal_fairness/data/extracted_notes/",
                 all_stays: str = "/scratch/gobi1/johnchen/new_git_stuff/multimodal_fairness/data/root/all_stays.csv"

    ):
        super().__init__(lazy)
        self.tokenizer = tokenizer or WhitespaceTokenizer()
        self.token_indexers = token_indexers or {'tokens': SingleIdTokenIndexer()}
        self.max_tokens = max_tokens
        self.listfile = listfile
        self.notes_dir = notes_dir
        logger.critical(f"we are getting the max tokens {self.max_tokens}")
        self.null_patients = []
        with open(skip_patients_file, "r") as file:
            for line in file:
                self.null_patients.append(line.strip())
        self.stats_write_dir = stats_write_dir
        self.all_stays_path = all_stays
        self.all_stays_df = self.get_all_stays()
        # self.null_patients

    def get_label_stats(self, file_path: str):
        '''
        Gets label (mortality) stats
        '''
        # get stats on the dataset listed at _path_
        from collections import defaultdict
        self.stats = defaultdict(int)

        with open(file_path, "r") as file:
            file.readline() # could also pandas readcsv and ignore first line
            for line in file:
                info_filename, _, label = line.split(",")
                self.stats[int(label)] +=1
        return self.stats

    '''
    Gets stats for the data listed at the datapath
    '''
    def get_note_stats(self, file_path):
        self.note_stats = {}
        exclusions = 0
        with open(file_path, "r") as file, \
                open(os.path.join(self.stats_write_dir, "note_lengths.txt") , "a") as note_length_file:
            file.readline() # could also pandas readcsv and ignore first line
            for line in tqdm(file):
                info_filename, _, label = line.split(",")
                info = info_filename.split("_")
                patient_id = info[0]

                # verify string inside a list of string
                if patient_id not in self.null_patients: # could also just do try except here

                    eps = int("".join([c for c in info[1] if c.isdigit()]))
                    notes = pd.read_pickle(os.path.join(self.notes_dir, patient_id, "notes.pkl"))
                    notes[["CHARTTIME", "STORETIME", "CHARTDATE"]] = notes[["CHARTTIME", "STORETIME", "CHARTDATE"]].apply(pd.to_datetime)
                    # fill in the time, do two passes. Any not caught in the first pass will get helped by second
                    notes["CHARTTIME"] = notes["CHARTTIME"].fillna(notes["STORETIME"])
                    notes["CHARTTIME"] = notes["CHARTTIME"].fillna(value=notes["CHARTDATE"].map(lambda x: pd.Timestamp(x) + pd.Timedelta(days=1) - pd.Timedelta(seconds=1)))

                    assert len(notes[notes["CHARTTIME"].isnull()]) == 0 # all of them should have been filled in.

                    # now, let's sort the notes
                    episode_specific_notes = notes[notes["EPISODES"] == eps].copy(deep=True)

                    # hadm_id = episode_specific_notes.groupby(["HADM_ID"]).agg({""}) # hadm ids seem to 1-to1 correspond to episodes
                    hadm_id = episode_specific_notes["HADM_ID"]
                    one_hadm_id = hadm_id.unique()
                    logger.info(type(one_hadm_id))
                    assert (one_hadm_id.shape[0]) == 1
                    assert len(one_hadm_id) == 1

                    icu_intime = self.all_stays_df[ self.all_stays_df["HADM_ID"] == one_hadm_id[0]]

                    # we are assuming that the intime is not null
                    intime_date = pd.Timestamp(icu_intime["INTIME"].iloc[0]) # iloc will automatically extract once you get to the base element
                    logger.info(type(intime_date))
                    intime_date_plus_two_days = pd.Timestamp(intime_date) + pd.Timedelta(days=2)

                    # all notes up to two days. Including potentially previous events.
                    mask = ( episode_specific_notes["CHARTTIME"] > intime_date) & (episode_specific_notes["CHARTTIME"] <= intime_date_plus_two_days)
                    all_mask = (episode_specific_notes["CHARTTIME"] <= intime_date_plus_two_days)

                    time_episode_specific_notes = episode_specific_notes[mask].copy(deep=True)

                    logger.debug("Went from {} to {} notes\n".format(len(episode_specific_notes), len(time_episode_specific_notes)))
                    # filter all of them, based on the all stays info
                    # do the episode and patient, associate to a specific hadm? i.e. are there 1 to 1 mappings here

                    # first, get the icu intime
                    # then, get the
                    # icu_intime = self.all_stays_df[]

                    # with open(os.path.join(self.stats_write_dir, "num_notes.txt"), "a") as notes_dir:

                    if len(time_episode_specific_notes) > 0:

                        text_df = time_episode_specific_notes
                        text_df.sort_values("CHARTTIME", ascending=True, inplace=True)  # we want them sorted by increasing time

                        # unlike the other one, we found our performance acceptable. Therefore, we use only the first note.
                        text = text_df["TEXT"].iloc[0] #assuming sorted order
                        tokens = self.tokenizer.tokenize(text)
                        if patient_id in self.note_stats:
                            logger.warning("Encountering the patient another time, for another episode {} {}".format(patient_id, eps))
                        self.note_stats[patient_id] = len(tokens)
                        if int(patient_id)%1000==0:
                            logger.info("text for patient {} \n: {}".format(patient_id,text))
                            logger.info("end of text for patient {} \n".format(patient_id))

                        xs = ""
                        if len(time_episode_specific_notes) > 1:

                        # lets try to join both of them
                            xs = text_df["TEXT"].iloc[1] #assuming sorted order
                        else:
                            logger.info("pat, eps: {} {} had only one note".format(patient_id, eps))

                    else:
                        logger.warning("No text found for patient {}. This is with the 48 hour\n. ".format(patient_id))
                        exclusions +=1
            sorted_dict = sorted(self.note_stats.items(), key=lambda tup: tup[1])
            note_length_file.write("For this file {}\n".format(file_path))
            for tup in sorted_dict:
                note_length_file.write("{} {}\n".format(tup[1], tup[0]))

        logger.critical("With 48 hour windowing, removed {}\n".format(exclusions))
        return self.note_stats

    def get_all_stays(self):
        my_stays_df = pd.read_csv(self.all_stays_path)
        return my_stays_df

    @overrides
    def _read(self, file_path: str) -> Iterable[Instance]:
        '''Expect: one instance per line'''

        use_preprocessing = True
        with open(file_path, "r") as file:
            file.readline() # could also pandas readcsv and ignore first line
            for line in file:
                cur_tokens = 0
                info_filename, _, label = line.split(",")
                info = info_filename.split("_")
                patient_id = info[0]

                # verify string inside a list of string
                if patient_id not in self.null_patients: # could also just do try except here

                    eps = int("".join([c for c in info[1] if c.isdigit()]))
                    notes = pd.read_pickle(os.path.join(self.notes_dir, patient_id, "notes.pkl"))
                    notes[["CHARTTIME", "STORETIME", "CHARTDATE"]] = notes[["CHARTTIME", "STORETIME", "CHARTDATE"]].apply(pd.to_datetime)
                    # fill in the time, do two passes. Any not caught in the first pass will get helped by second
                    notes["CHARTTIME"] = notes["CHARTTIME"].fillna(notes["STORETIME"])
                    notes["CHARTTIME"] = notes["CHARTTIME"].fillna(value=notes["CHARTDATE"].map(lambda x: pd.Timestamp(x) + pd.Timedelta(days=1) - pd.Timedelta(seconds=1)))

                    assert len(notes[notes["CHARTTIME"].isnull()]) == 0 # all of them should have been filled in.

                    # now, let's sort the notes
                    episode_specific_notes = notes[notes["EPISODES"] == eps]

                    hadm_id = episode_specific_notes["HADM_ID"]
                    one_hadm_id = hadm_id.unique()

                    icu_intime = self.all_stays_df[self.all_stays_df["HADM_ID"] == one_hadm_id[0]]

                    # we are assuming that the intime is not null
                    intime_date = pd.Timestamp(icu_intime["INTIME"].iloc[
                                                   0])  # iloc will automatically extract once you get to the base element
                    intime_date_plus_two_days = pd.Timestamp(intime_date) + pd.Timedelta(days=2)

                    # all notes up to two days. Including potentially previous events.
                    mask = (episode_specific_notes["CHARTTIME"] > intime_date) & (
                                episode_specific_notes["CHARTTIME"] <= intime_date_plus_two_days)

                    time_episode_specific_notes = episode_specific_notes[mask].copy(deep=True)
                    # with open(os.path.join(self.stats_write_dir, "num_notes.txt"), "a") as notes_dir:

                    if len(time_episode_specific_notes) > 0:

                        text_df = time_episode_specific_notes
                        text_df.sort_values("CHARTTIME", ascending=False, inplace=True)  # we want them sorted by increasing time

                        # unlike the other one, we found our performance acceptable. Therefore, we use only the first note.
                        text = " ".join(text_df["TEXT"].tolist())

                        #
                        # iloc[0] #assuming sorted order
                        #
                        # xs = ""
                        # if len(time_episode_specific_notes) > 1:
                        #
                        # # lets try to join both of them
                        #     xs = text_df["TEXT"].iloc[1] #assuming sorted order
                        # else:
                        #     logger.info("pat, eps: {} {} had only one note".format(patient_id, eps))
                        # text = xs + " " + text

                        # join the texts together, or simply use the first one (according to starttime)
                        # tokens = self.tokenizer.tokenize(text)[:self.max_tokens]
                        if use_preprocessing:
                            token_sent_stream = preprocess_mimic(text)
                            tokens = []
                            cur_tokens = 0
                            for i,token_sent in enumerate(token_sent_stream):
                                if cur_tokens > self.max_tokens: break
                                cur_tokens += len(token_sent.split())
                                tokens.append(token_sent)

                            text = " ".join(tokens) #overwrite the text!
                        tokens = self.tokenizer.tokenize(text)[:self.max_tokens]

                        text_field = TextField(tokens, self.token_indexers)
                        label_field = LabelField(label)
                        meta_data_field = MetadataField({"patient_id": patient_id,
                                                         "episode": eps,
                                                         "hadm_id": hadm_id,
                                                         })
                        fields = {'text': text_field, 'label': label_field, "metadata": meta_data_field}
                        yield Instance(fields)
                    else:
                        logger.warning("No text found for patient {}".format(patient_id))
                        # in this case, we ignore the patient

'''
We can also look at dropout and other techniques which we do here!
'''

@Model.register("MortalityClassifier")
class MortalityClassifier(Model):
    def __init__(self,
                 vocab: Vocabulary,
                 embedder: TextFieldEmbedder,
                 encoder: Seq2VecEncoder,
                 regularizer_applicator: RegularizerApplicator = None
                 ):
        super().__init__(vocab, regularizer_applicator)
        self.embedder = embedder
        self.encoder = encoder
        num_labels = vocab.get_vocab_size("labels")
        self.classifier = torch.nn.Linear(encoder.get_output_dim(), num_labels)
        self.accuracy = CategoricalAccuracy()
        self.auc = Auc()
        self.reg_app = regularizer_applicator

    def forward(self,
                text: Dict[str, torch.Tensor],
                label: torch.Tensor) -> Dict[str, torch.Tensor]:
        # Shape: (batch_size, num_tokens, embedding_dim)
        embedded_text = self.embedder(text)
        # Shape: (batch_size, num_tokens)
        mask = util.get_text_field_mask(text)
        # Shape: (batch_size, encoding_dim)
        encoded_text = self.encoder(embedded_text, mask) #horizontal; vertical (partial depth) might be good
        # Shape: (batch_size, num_labels)
        logits = self.classifier(encoded_text)
        # Shape: (batch_size, num_labels)
        probs = torch.nn.functional.softmax(logits, dim=-1)
        # reg_loss = self.get_regularization_penalty() # should not have to manually apply the regularization
        # Shape: (1,)
        loss = torch.nn.functional.cross_entropy(logits, label)
        self.accuracy(logits, label)
        preds = logits.argmax(-1)
        self.auc(preds, label)
        output = {'loss': loss, 'probs': probs}
        return output

    '''this is called'''
    def get_metrics(self, reset: bool = False) -> Dict[str, float]:
        return {"accuracy": self.accuracy.get_metric(reset),
                "auc":self.auc.get_metric(reset)}
#
#
def build_dataset_reader(**kwargs) -> DatasetReader:
    return MortalityReader(**kwargs, lazy=False)
#

# "/scratch/gobi1/johnchen/new_git_stuff/multimodal_fairness/data/in-hospital-mortality/train/listfile.csv"
'''
The issue is that this may for somereason read everything into memory
'''
def read_data(
    reader: DatasetReader,
        train_data_path: str = "/scratch/gobi1/johnchen/new_git_stuff/multimodal_fairness/data/in-hospital-mortality/train/listfile.csv",
        valid_data_path: str = "/scratch/gobi1/johnchen/new_git_stuff/multimodal_fairness/data/in-hospital-mortality/test/listfile.csv"
) -> Tuple[Iterable[Instance], Iterable[Instance]]:
    print("Reading data")
    training_data = reader.read(train_data_path)
    validation_data = reader.read(valid_data_path)
    return training_data, validation_data

# def read_data(
#     reader: DatasetReader
# ) -> Tuple[Iterable[Instance], Iterable[Instance]]:
#     print("Reading data")
#     training_data = reader.read("quick_start/data/movie_review/train.tsv")
#     validation_data = reader.read("quick_start/data/movie_review/dev.tsv")
#     return training_data, validation_data

#
def build_vocab(instances: Iterable[Instance]) -> Vocabulary:
    print("Building the vocabulary")
    return Vocabulary.from_instances(instances)
#

def build_model_Transformer(vocab: Vocabulary,
                use_reg: bool = True) -> Model:
    print("Building the model")
    vocab_size = vocab.get_vocab_size("tokens")
    EMBED_DIMS = 300
    # turn the tokens into 300 dim embedding. Then, turn the embeddings into encodings
    embedder = BasicTextFieldEmbedder(
        {"tokens": Embedding(embedding_dim=EMBED_DIMS, num_embeddings=vocab_size)})
    encoder = CnnEncoder(embedding_dim=EMBED_DIMS, ngram_filter_sizes = (2,3,4,5),
                         num_filters=5) # num_filters is a tad bit dangerous: the reason is that we have this many filters for EACH ngram f
    # encoder = BertPooler("bert-base-cased")
    # the output dim is just the num filters *len(ngram_filter_sizes)

    #     construct the regularizer applicator
    regularizer_applicator = None
    if use_reg :
        l2_reg = L2Regularizer()
        regexes = [("embedder", l2_reg),
                   ("encoder", l2_reg),
                   ("classifier", l2_reg)
                   ]
        regularizer_applicator = RegularizerApplicator(regexes)

    return MortalityClassifier(vocab, embedder, encoder,regularizer_applicator)

def build_model(vocab: Vocabulary,
                use_reg: bool = True) -> Model:
    print("Building the model")
    vocab_size = vocab.get_vocab_size("tokens")
    EMBED_DIMS = 300
    # turn the tokens into 300 dim embedding. Then, turn the embeddings into encodings
    embedder = BasicTextFieldEmbedder(
        {"tokens": Embedding(embedding_dim=EMBED_DIMS, num_embeddings=vocab_size)})
    encoder = CnnEncoder(embedding_dim=EMBED_DIMS, ngram_filter_sizes = (2,3,4,5),
                         num_filters=5) # num_filters is a tad bit dangerous: the reason is that we have this many filters for EACH ngram f
    # encoder = BertPooler("bert-base-cased")
    # the output dim is just the num filters *len(ngram_filter_sizes)

    #     construct the regularizer applicator
    regularizer_applicator = None
    if use_reg :
        l2_reg = L2Regularizer()
        regexes = [("embedder", l2_reg),
                   ("encoder", l2_reg),
                   ("classifier", l2_reg)
                   ]
        regularizer_applicator = RegularizerApplicator(regexes)

    return MortalityClassifier(vocab, embedder, encoder,regularizer_applicator)
#
def build_data_loaders(
    train_data: torch.utils.data.Dataset,
    dev_data: torch.utils.data.Dataset, batch_size=64
) -> Tuple[allennlp.data.DataLoader, allennlp.data.DataLoader]:
    # Note that DataLoader is imported from allennlp above, *not* torch.
    # We need to get the allennlp-specific collate function, which is
    # what actually does indexing and batching.
    train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True)
    dev_loader = DataLoader(dev_data, batch_size=batch_size, shuffle=False)
    return train_loader, dev_loader

def build_data_loaders_from_reader(dataset_reader, vocab, batch_size=64):
    train_data, dev_data = read_data(dataset_reader)

    train_data.index_with(vocab)
    dev_data.index_with(vocab)
    train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=False)
    dev_loader = DataLoader(dev_data, batch_size=batch_size, shuffle=False)
    return train_loader, dev_loader

#
def build_trainer(
    model: Model,
    serialization_dir: str,
    train_loader: DataLoader,
    dev_loader: DataLoader
) -> Trainer:
    parameters = [
        [n, p]
        for n, p in model.named_parameters() if p.requires_grad
    ]
    optimizer = AdamOptimizer(parameters)
    trainer = GradientDescentTrainer(
        model=model,
        serialization_dir=serialization_dir,
        data_loader=train_loader,
        validation_data_loader=dev_loader,
        num_epochs=50,
        optimizer=optimizer,
        cuda_device=0,
        validation_metric="+auc",
        patience=5

    )
    return trainer

'''
we pass in the vocab to ensure that we are speaking the same language!
'''
def run_training_loop_over_dataloaders(model,train_loader,dev_loader, args, use_gpu=False, batch_size =32,
                      serialization_dir = "/scratch/gobi1/johnchen/new_git_stuff/multimodal_fairness/src/models/new_allen_nlp/experiments"):
    # move the model over, if necessary, and possible
    gpu_device = torch.device("cuda:0" if use_gpu  else "cpu")
    model = model.to(gpu_device)

    experiment_dir = os.path.join(serialization_dir, args.run_name)

    if not os.path.exists(experiment_dir):
        os.makedirs(experiment_dir, exist_ok=True)

    serialization_dir = experiment_dir

    # This is the allennlp-specific functionality in the Dataset object;
    # we need to be able convert strings in the data to integers, and this
    # is how we do it.

    # These are again a subclass of pytorch DataLoaders, with an
    # allennlp-specific collate function, that runs our indexing and
    # batching code.

    # You obviously won't want to create a temporary file for your training
    # results, but for execution in binder for this course, we need to do this.
    trainer = build_trainer(
        model,
        serialization_dir,
        train_loader,
        dev_loader
    )
    trainer.train()

    return model

'''
we pass in the vocab to ensure that we are speaking the same language!
'''
def run_training_loop(model, dataset_reader, vocab, args, use_gpu=False, batch_size =32,
                      serialization_dir = "/scratch/gobi1/johnchen/new_git_stuff/multimodal_fairness/src/models/new_allen_nlp/experiments"):
    # move the model over, if necessary, and possible
    gpu_device = torch.device("cuda:0" if use_gpu  else "cpu")
    model = model.to(gpu_device)

    experiment_dir = os.path.join(serialization_dir, args.run_name)

    if not os.path.exists(experiment_dir):
        os.makedirs(experiment_dir, exist_ok=True)

    serialization_dir = experiment_dir

    # This is the allennlp-specific functionality in the Dataset object;
    # we need to be able convert strings in the data to integers, and this
    # is how we do it.

    # These are again a subclass of pytorch DataLoaders, with an
    # allennlp-specific collate function, that runs our indexing and
    # batching code.
    gc.collect()

    train_loader, dev_loader = build_data_loaders_from_reader(dataset_reader,vocab, batch_size=batch_size)
    # You obviously won't want to create a temporary file for your training
    # results, but for execution in binder for this course, we need to do this.
    trainer = build_trainer(
        model,
        serialization_dir,
        train_loader,
        dev_loader
    )
    trainer.train()
    del train_loader
    del dev_loader
    gc.collect()
    return model, dataset_reader

def main():
    logger.setLevel(logging.CRITICAL)
    args = lambda x: None
    args.batch_size = 1024
    args.run_name = "100"
    args.train_data = "/scratch/gobi1/johnchen/new_git_stuff/multimodal_fairness/data/decompensation/train/listfile.csv"
    args.dev_data = "/scratch/gobi1/johnchen/new_git_stuff/multimodal_fairness/data/decompensation/test/listfile.csv"

    import time

    start_time = time.time()
    # mr = MortalityReader()
    # instances = mr.read("/scratch/gobi1/johnchen/new_git_stuff/multimodal_fairness/data/in-hospital-mortality/train/listfile.csv")
    # for inst in instances[:10]:
    #     print(inst)
    print("we are running with the following info")
    print("Torch version {} Cuda version {} cuda available? {}".format(torch.__version__, torch.version.cuda,
                                                                       torch.cuda.is_available()))
    # We've copied the training loop from an earlier example, with updated model
    # code, above in the Setup section. We run the training loop to get a trained
    # model.

    dataset_reader = build_dataset_reader()

    dataset_reader.get_label_stats(args.train_data)
    for key in sorted(dataset_reader.stats.keys()):
        print("{} {}".format(key,dataset_reader.stats[key]))
    dataset_reader.get_label_stats(args.dev_data)

    for key in sorted(dataset_reader.stats.keys()):
        print("{} {}".format(key,dataset_reader.stats[key]))
    # These are a subclass of pytorch Datasets, with some allennlp-specific
    # functionality added.
    train_data, dev_data = read_data(dataset_reader, args.train_data, args.dev_data)

    vocab = build_vocab(train_data + dev_data)

    # make sure to index the vocab before adding it
    train_data.index_with(vocab)
    dev_data.index_with(vocab)

    train_dataloader, dev_dataloader = build_data_loaders(train_data, dev_data)
    # del train_data
    # del dev_data

    # throw in all the regularizers to the regularizer applicators
    model = build_model(vocab,use_reg=False)
    model = run_training_loop_over_dataloaders(model, train_dataloader, dev_dataloader, args,use_gpu=True, batch_size=args.batch_size)

    logger.warning("We have finished training")
    # # Now we can evaluate the model on a new dataset.
    # test_data = dataset_reader.read(
    #     '/scratch/gobi1/johnchen/new_git_stuff/multimodal_fairness/data/in-hospital-mortality/test/listfile.csv')
    # test_data.index_with(model.vocab)
    # data_loader = DataLoader(test_data, batch_size=args.batch_size)

    # results = evaluate(model, data_loader, -1, None)
    # print(results)

    results = evaluate(model, dev_dataloader, 0, None)

    print("we succ fulfilled it")
    with open("nice_srun_time.txt", "w") as file:
        file.write("it is done\n{}\nTook {}".format(results, time.time() - start_time))

    '''
    since it is a pytorch model, we can just forward on it, to get the actual output dict.
    and in particular, we can accumulate everything
    '''

    # for batch in dev_dataloader:
        # we assume we get a read instance type thing?

'''
get the stats of the preprocessed notes
'''
def get_preprocessed_stats():
    logger.setLevel(logging.CRITICAL)
    dataset_reader = build_dataset_reader()
    train_note_stats = dataset_reader.get_note_stats("/scratch/gobi1/johnchen/new_git_stuff/multimodal_fairness/data/in-hospital-mortality/train/listfile.csv")
    test_note_stats = dataset_reader.get_note_stats("/scratch/gobi1/johnchen/new_git_stuff/multimodal_fairness/data/in-hospital-mortality/test/listfile.csv")
    # merge the dictionaries
    all_note_stats = {**train_note_stats, **test_note_stats}
    sorted_dict = sorted(all_note_stats.items(), key=lambda tup: tup[1])
    with open("/scratch/gobi1/johnchen/new_git_stuff/multimodal_fairness/data/extracted_notes/note_lengths.txt", "a") as note_length_file:
        for tup in sorted_dict:
            note_length_file.write("{} {}\n".format(tup[1], tup[0]))

if __name__ == __name__:
    main()
    # get_preprocessed_stats()
    # dataset_reader = build_dataset_reader(all_stays="/scratch/gobi1/johnchen/new_git_stuff/multimodal_fairness/data/root/all_stays.csv")
    # stays_df = dataset_reader.get_all_stays()
    # print(len(stays_df))


import cv2
import sys
import numpy as np

image = cv2.imread(sys.argv[1])
blur = cv2.GaussianBlur(image, (5,5), 0)
mean = cv2.blur(image, (5,5))

matrix3by3 = np.matrix('1 2 1; 2 4 2; 1 2 1')/16
matrix5by5 = np.matrix('1 4 7 4 1; 4 16 26 16 4; 7 26 41 26 7; 4 16 26 16 4; 1 4 7 4 1')/273

custom_filtered1 = cv2.filter2D(image, -1, matrix5by5)
custom_filtered2 = cv2.filter2D(image, -1, matrix3by3)

cv2.imshow("gaussian", blur)
cv2.imshow("mean", mean)
cv2.imshow("custom 1", custom_filtered1)
cv2.imshow("custom 2", custom_filtered2)
cv2.imshow("original", image)
cv2.waitKey(0)

cv2.destroyAllWindows()


# -*- coding: utf-8 -*-
"""
Created on Sun Mar 24 15:46:12 2019

@author: durgesh singh
"""

from torch.utils.data import Dataset
import nltk
from nltk.corpus import treebank
from nltk.corpus import brown
from collections import defaultdict
import torch

class CustomDataset(Dataset):

    def __init__(self,dname='brown'):
        super().__init__()

        data = None
        #selecting the datset
        if dname =='treebank':
            if len(treebank.words()) == 0:
                nltk.download('treebank')
            data = treebank.tagged_sents(tagset='universal')

        elif dname == 'brown':
            if len(brown.words()) == 0:
                nltk.download('brown')
            data = brown.tagged_sents(tagset='universal')

        self.data=data
        #print(data[0:1])
        vocab,tags =self._build_vocab()
        max_sent_len = max(map(len, data))
        self.max_sent_len = max_sent_len
        self.word_to_idx = defaultdict(lambda:0, {word:idx for idx,word in enumerate(vocab)})
        self.idx_to_word = {idx:word for word,idx in self.word_to_idx.items()}
        self.tag_to_idx = {tag:idx for idx,tag in enumerate(tags)}
        self.idx_to_tag = {idx:tag for tag,idx in self.tag_to_idx.items()}
        self.sen_list,self.tag_list = self._convert_to_num()

    def get_target_size(self):
        return len(self.tag_to_idx)

    def get_vocab_size(self):
        return len(self.word_to_idx)

    def _convert_to_num(self):
        data = self.data
        max_sent_len = self.max_sent_len
        sent_list=[]
        taggs_list=[]

        for sen in data:
            num_row = [self.word_to_idx[word.lower()] for word,tag in sen]
            tag_row = [self.tag_to_idx[tag] for word,tag in sen]
            num_row = num_row +[0]*(max_sent_len-len(num_row))
            tag_row = tag_row +[0]*(max_sent_len-len(tag_row))
            num_row = torch.tensor(num_row)
            tag_row = torch.tensor(tag_row)
            sent_list.append(num_row)
            taggs_list.append(tag_row)

        return sent_list,taggs_list

    def _build_vocab(self):
        data=self.data
        vocabset=set()
        tagset = set()
        all_sents_tags = [[(word.lower(), tag) for word, tag in sentence] for sentence in data]
        for sent_tag in all_sents_tags:
            sent,tags = zip(*sent_tag)
            vocabset.update(sent)
            tagset.update(tags)
        return ['<UNK>','<EOS>']+list(vocabset),list(tagset)

    def __len__(self):
        return len(self.sen_list)

    def __getitem__(self,idx):
        return self.sen_list[idx],self.tag_list[idx]

cdataset = CustomDataset(dname='brown')


from django.shortcuts import render
from django.core.paginator import Paginator

from .models import Criminal
from .forms import AdvancedSearch

def search(request):
    # If quick search
    if request.GET.get('search'):
        searchName = request.GET.get('search')
        print(searchName)
        criminals = Criminal.objects.filter(name__contains=searchName)
        form = AdvancedSearch(data={'name': searchName})
    # If advanced search
    elif (request.GET.get('name')):
            form = AdvancedSearch(request.GET)
            if (form.is_valid()):
                criminals = Criminal.objects.filter(name__contains=form.cleaned_data['name'])
            else:
                criminals = Criminal.objects.all()
    # If default view
    else:
        form = AdvancedSearch()
        criminals = Criminal.objects.all()

    # Pagination
    paginator = Paginator(criminals, 2)
    page_number = request.GET.get('page')
    page_obj = paginator.get_page(page_number)

    return render(request, 'criminals/results.html', {'criminals': page_obj, 'advancedSearch': form})


__author__ = 'shikun'
import math
from math import floor
import subprocess
import os

class ApkInfo():
    def __init__(self, apkpath):
        self.apkpath = apkpath

    # 得到app的文件大小
    def get_apk_size(self):
        size = floor(os.path.getsize(self.apkpath) / (1024 * 1000))
        return str(size) + "M"

    # 得到版本，windows用findstr，linux用grep
    def get_apk_version(self):
        cmd = "aapt dump badging " + self.apkpath + " | findstr versionName"
        result = ""
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE,
                             stdin=subprocess.PIPE, shell=True)
        (output, err) = p.communicate()
        if output != "":
            result = output.split()[3].decode()[12:]
        return result

    # 得到应用名字，windows使用findstr，linux使用grep
    def get_apk_name(self):
        cmd = "aapt dump badging " + self.apkpath + " | findstr application-label: "
        result = ""
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE,
                             stdin=subprocess.PIPE, shell=True)
        (output, err) = p.communicate()
        if output != "":
            result = output.split()[0].decode()[18:]
        return result

    # 得到包名，windows使用findstr，linux使用grep
    def get_apk_pkg(self):
        cmd = "aapt dump badging " + self.apkpath + " | findstr package:"
        result = ""
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE,
                             stdin=subprocess.PIPE, shell=True)
        (output, err) = p.communicate()
        if output != "":
            result = output.split()[1].decode()[6:-1]
        return result

    # 得到启动类，windows使用findstr，linux使用grep
    def get_apk_activity(self):
        cmd = "aapt dump badging " + self.apkpath + " | findstr launchable-activity:"
        result = ""
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE,
                             stdin=subprocess.PIPE, shell=True)
        (output, err) = p.communicate()
        if output != "":
            result = output.split()[1].decode()[6:-1]
        return result

if __name__ == '__main__':
    ApkInfo(r"D:\app\appium_study\apk\t.apk").get_apk_pkg()
    ApkInfo(r"D:\app\appium_study\apk\t.apk").get_apk_activity()


#####
#
# Module name:  rpt_bysource.py
# Purpose:      dupReport grouped by source
#
# Notes:
#
#####

# Import system modules
import datetime
import time
import email
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import smtplib
import re

# Import dupReport modules
import globs
import db
import drdatetime
import report

# Report grouped by source
def runReport(startTime):
    globs.log.write(1, 'rpt_bysource()')

    # Get header and column info
    nFields, fldDefs, reportOpts, rptCols, rptTits = report.initReportVars()

    # Print the report title
    msgHtml, msgText, msgCsv = report.rptTop(reportOpts, nFields)

    # Remove columns we don't need for this report
    # These are already part of the report logic processing & subheaders
    # We won't need to loop through them for the report fields
    rptCols.remove('source')

    # Print column titles if not printing for each section
    if reportOpts['repeatheaders'] is False:
        msgHtml, msgText, msgCsv = report.rptPrintTitles(msgHtml, msgText, msgCsv, rptCols)

    # Select sources from database
    dbCursor = globs.db.execSqlStmt("SELECT DISTINCT source FROM backupsets ORDER BY source")
    srcSet = dbCursor.fetchall()
    globs.log.write(2, 'srcSet=[{}]'.format(srcSet))

    # Loop through backupsets table and get all the potential destinations
    for srcKey in srcSet:

        # Add Source title
        subHead = globs.optionManager.getRcOption('report', 'subheading')
        if subHead is not None:
            # Substitute subheading keywords
            subHead = subHead.replace('#SOURCE#', srcKey[0])
        if subHead is None or subHead == '':
            msgHtml += '
{}: {}
\n'.format(nFields, reportOpts['subheadbg'], rptTits['source'], srcKey[0])
            msgText += '***** {}: {}*****\n'.format(rptTits['source'], srcKey[0])
            msgCsv += '\"***** {}: {}*****\",\n'.format(rptTits['source'], srcKey[0])
        else:
            msgHtml += '
{}
\n'.format(nFields, reportOpts['subheadbg'], subHead)
            msgText += '***** {} *****\n'.format(subHead)
            msgCsv += '\"***** {} *****\"\n'.format(subHead)

        # Print column titles if printing for each section
        if reportOpts['repeatheaders'] is True:
            msgHtml, msgText, msgCsv = report.rptPrintTitles(msgHtml, msgText, msgCsv, rptCols)

        sqlStmt = "SELECT destination, dupversion, timestamp, duration, examinedFiles, examinedFilesDelta, sizeOfExaminedFiles, fileSizeDelta, addedFiles, deletedFiles, modifiedFiles, filesWithError, \
            parsedResult, messages, warnings, errors, logdata FROM report WHERE source=\'{}\'".format(srcKey[0])
        if reportOpts['sortby'] == 'destination':
            sqlStmt += ' ORDER BY destination'
        else:
            sqlStmt += ' ORDER BY timestamp'

        dbCursor = globs.db.execSqlStmt(sqlStmt)
        reportRows = dbCursor.fetchall()
        globs.log.write(3, 'reportRows=[{}]'.format(reportRows))

        # Loop through each new activity for the source/destination and add to report
        for destination, dupversion, timestamp, duration, examinedFiles, examinedFilesDelta, sizeOfExaminedFiles, fileSizeDelta, \
            addedFiles, deletedFiles, modifiedFiles, filesWithError, parsedResult, messages, \
            warnings, errors, logdata in reportRows:

            # Convert duration to string
            rptDuration = drdatetime.timeDiff(duration)

            # Truncate message, warning, & error if indicated in .rc file
            messages, warnings, errors, logdata = report.truncateWarnErrMsgs(messages, reportOpts['truncatemessage'], warnings, reportOpts['truncatewarning'], errors, reportOpts['truncateerror'], logdata, reportOpts['truncatelogdata'])

            # Get date and time from timestamp
            dateStr, timeStr = drdatetime.fromTimestamp(timestamp)

            # Print report fields
            # Each field takes up one column/cell in the table
            msgHtml += '
'

            # The full list of possible fields in the report. printField() below will skip a field if it is emoved in the .rc file.
            titles = ['destination', 'dupversion', 'date','time', 'duration', 'files', 'filesplusminus', 'size', 'sizeplusminus', 'added','deleted',  'modified', 'errors', 'result']
            fields = [destination, dupversion, dateStr, timeStr, rptDuration, examinedFiles, examinedFilesDelta, sizeOfExaminedFiles, fileSizeDelta, addedFiles, deletedFiles,  modifiedFiles, filesWithError, parsedResult]

            for ttl, fld in zip(titles, fields):
                msgHtml += report.printField(ttl, fld, 'html')
                msgText += report.printField(ttl, fld, 'text')
                msgCsv += report.printField(ttl, fld, 'csv')

            msgHtml += '
\n'
            msgText += '\n'
            msgCsv += '\n'

            fields = [messages, warnings, errors, logdata ]
            options = ['displaymessages', 'displaywarnings', 'displayerrors', 'displaylogdata']
            backgrounds = ['jobmessagebg', 'jobwarningbg', 'joberrorbg', 'joblogdatabg']
            titles = ['jobmessages', 'jobwarnings', 'joberrors', 'joblogdata']
            # Print message/warning/error/logdata fields
            # Each of these spans all the table columns
            for fld, opt, bg, tit in zip(fields, options, backgrounds, titles):
                if ((fld != '') and (reportOpts[opt] == True)):
                    msgHtml += '
{}{}
\n'.format(nFields, reportOpts[bg], rptTits[tit], fld)
                    msgText += '{}: {}\n'.format(rptTits[tit], fld)
                    msgCsv += '\"{}: {}\",\n'.format(rptTits[tit], fld)

        # Show inactivity - Look for missing source/dest pairs in report
        dbCursor = globs.db.execSqlStmt("SELECT destination, lastTimestamp, lastFileCount, lastFileSize FROM backupsets WHERE source = '{}' ORDER BY source".format(srcKey[0]))
        missingRows = dbCursor.fetchall()
        for destination, lastTimestamp, lastFileCount, lastFileSize in missingRows:
            dbCursor = globs.db.execSqlStmt('SELECT count(*) FROM report WHERE source=\"{}\" AND destination=\"{}\"'.format(srcKey[0], destination))
            countRows = dbCursor.fetchone()
            if countRows[0] == 0:

                # If src/dest is known offline, skip
                srcDest = '{}{}{}'.format(srcKey[0], globs.opts['srcdestdelimiter'], destination)
                offline = globs.optionManager.getRcOption(srcDest, 'offline')
                if offline != None:
                    if offline.lower() in ('true'):
                        continue

                # Calculate days since last activity
                diff = drdatetime.daysSince(lastTimestamp)

                # See if we're past the backup interval before reporting
                result, interval = report.pastBackupInterval(srcDest, diff)
                if result is False:
                    globs.log.write(3, 'SrcDest=[{}] DaysDiff=[{}]. Skip reporting'.format(srcDest, diff))
                    msgHtml += '
'
                    msgHtml += report.printField('destination', destination, 'html')
                    msgHtml += 'Last activity {} days ago. Backup interval is {} days.	'.format(nFields-1, diff, interval)
                    msgHtml += '
\n'

                    msgText += report.printField('destination', destination, 'text')
                    msgText += '{}: Last activity {} days ago. Backup interval is {}\n'.format(destination, diff, interval)

                    msgCsv += report.printField('destination', destination, 'csv')
                    msgCsv += '\"{}: Last activity {} days ago. Backup interval is {}.\"\n'.format(destination, diff, interval)
                else:
                    lastDateStr, lastTimeStr = drdatetime.fromTimestamp(lastTimestamp)
                    msgHtml += '
'
                    msgHtml += report.printField('destination', destination, 'html')
                    msgHtml += 'No new activity. Last activity on {} at {} ({} days ago)	'.format(nFields-1, report.getLastSeenColor(reportOpts, diff, interval), lastDateStr, lastTimeStr, diff)
                    msgHtml += '
\n'

                    msgText += report.printField('destination', destination, 'text')
                    msgText += '{}: No new activity. Last activity on {} at {} ({} days ago)\n'.format(destination, lastDateStr, lastTimeStr, diff)

                    msgCsv += report.printField('destination', destination, 'csv')
                    msgCsv += '\"{}: No new activity. Last activity on {} at {} ({} days ago)\"\n'.format(destination, lastDateStr, lastTimeStr, diff)

    # Add report footer
    msgHtml, msgText, msgCsv = report.rptBottom(msgHtml, msgText, msgCsv, startTime, nFields)

    # Return text & HTML messages to main program. It can decide which one it wants to use.
    return msgHtml, msgText, msgCsv


class Arclist:

	I = [] #arc start
	J = [] #arc end
	H = [] #heads
	L = [] #links
	free = []

	count = 0

	def __init__(self, num, i_list, j_list):
		self.set_num_of_nodes(num)
		for i in range(len(i_list)):
			self.add_edge(i_list[i],j_list[i])

	def set_num_of_nodes(self, num):
		for i in range(num):
			self.H.append(-1)

	def add_edge(self, left, right):
		if len(self.free) == 0:
			self.I.append(left)
			self.J.append(right)
			self.L.append(self.H[left])
			self.H[left]=len(self.L)-1
		else:
			self.I[free[0]]=left
			self.J[free[0]]=right
			self.L[free[0]]=self.H[left]
			self.H[left]=self.free[0]
			self.free.pop(0)

	def remove_edge(self, index):
		self.free.append(index)
		it_left = self.H[self.I[index]]
		it_right = self.H[self.J[index]]
		if it_left == index:
			self.H[it_left] = self.L[self.H[it_left]]
		else:
			while(self.L[it_left]!=index):
				it_left = self.L[it_left]
				if it_left == -1: return
			self.L[it_left] = self.L[self.L[it_left]]
		if it_right == index:
			self.H[it_right] = self.L[self.H[it_right]]
		else:
			while(self.L[it_right]!=index):
				it_right = self.L[it_right]
				if it_right == -1: return
			self.L[it_right] = self.L[self.L[it_right]]

	def print_lists(self):
		print(self.I)
		print(self.J)
		print(self.H)
		print(self.L)

	def make_norm_graph(self, G):
		G.clear()
		for i in range(len(self.H)):
			G.add_node(i)
			it = self.H[i]
			while(it!=-1):
				G.add_edge(self.I[it],self.J[it])
				it = self.L[it]


"""
Group 8
CSC 256
Prof. Tonya Melvin-Bryant
Lab Intro: Full Retirement Age

Using the Full Retirement Age information at Social Security Administration, this program:
asks the user for a birth year and month,
displays the age (with additional months) for obtaining full SSA benefits
displays the year and month for obtaining full SSA benefits
"""

import sys
from retirement import *

def main():
    print('Social Security Full Retirement Age Calculator')
    x = 0
    while x == 0:
        try:
            birth_year = int(input('Enter the year of birth or Enter to exit:'))
            birth_month = int(input('Enter Birth Month as a number:'))
        except ValueError:
            sys.exit()
        retirement_age_year, retirement_age_month = calculate_retirement_age(birth_year)
        retirement_date_year, retirement_date_month = calculate_retirement_date(birth_year, birth_month, retirement_age_year, retirement_age_month)
        month = {1: 'January',
                 2: 'February',
                 3: 'March',
                 4: 'April',
                 5: 'May',
                 6: 'June',
                 7: 'July',
                 8: 'August',
                 9: 'September',
                 10: 'October',
                 11: 'November',
                 12: 'December'}
        print('Your full retirement age is ' + str(retirement_age_year) + ' and ' + str(retirement_age_month) + ' months.')
        print("This will be in {} of {}\n".format(month[retirement_date_month], retirement_date_year))

if __name__ == "__main__":
    main()


from pip._vendor.distlib.compat import raw_input
from urllib3.util import url
from general import *
from domain_name import *
from ip_address import *
from nmap import *
from robots_txt import *
from whois import *

ROOT_DIR = 'reports'
create_dir(ROOT_DIR)

def gather_info(name, url):
    print('Getting domain name...')
    domain_name = get_domain_name(url)
    print('Getting ip address...')
    ip_address = get_ip_address(domain_name)
    print('Getting nmap data...')
    nmap = get_nmap('',ip_address)
    print('Getting robots.txt...')
    robots_txt = get_robots_txt(url)
    print('Getting whois data...')
    whois = get_whois(domain_name)
    print('Writing report...')
    create_report(name,url,domain_name,ip_address,nmap,robots_txt, whois)
    print('Report created successfully !')

def create_report(name, full_url,domain_name,ip_address,nmap,robots_txt, whois):
    project_dir = ROOT_DIR + '/' + name
    create_dir(project_dir)
    write_file(project_dir + '/full_url.txt',full_url)
    write_file(project_dir + '/domain_name.txt',domain_name)
    write_file(project_dir + '/ip_address.txt',ip_address)
    write_file(project_dir + '/nmap.txt',nmap)
    write_file(project_dir + '/robots.txt',robots_txt)
    write_file(project_dir + '/whois.txt',whois)

nume = input('name of website: ')
adresa = input('full domain (ex: https://www.thenewboston.com/): ').lower()

gather_info(nume, adresa)


#!usr/bin/env python

import numpy as np
import pandas as pd
import h5py
from datetime import date,timedelta

#------------- AMSR-E vod-----------------------------#

# read the vod data
# midnight
vod_midnight_dn = '../Data/MM_VOD UMT Amazon.xlsx'

vod_midnight_df = pd.read_excel(vod_midnight_dn,skiprows=2)
vod_midnight_df = vod_midnight_df.rename(
    columns={'Unnamed: 1' : 'Date',
             'Brasilia '  : 'BSB_vod_asc',
             'Caxiuana'   : 'CAX_vod_asc',
             'Paracou'    : 'GYF_vod_asc',
             'Manaus K34 ': 'M34_vod_asc',
             'Pe-de-Gigante' : 'PDG_vod_asc',
             'Petrolina'  : 'PNZ_vod_asc',
             'Rebio Jaru' : 'RJA_vod_asc',
             'Santarem km67' : 'S67_vod_asc'})

vod_midday_dn = '../Data/MM_VOD UMT Amazon_Midday.xlsx'

vod_midday_df = pd.read_excel(vod_midday_dn,skiprows=2)
vod_midday_df = vod_midday_df.rename(
    columns={'Unnamed: 1' : 'Date',
             'Brasilia '  : 'BSB_vod_des',
             'Caxiuana'   : 'CAX_vod_des',
             'Paracou'    : 'GYF_vod_des',
             'Manaus K34 ': 'M34_vod_des',
             'Pe-de-Gigante' : 'PDG_vod_des',
             'Petrolina'  : 'PNZ_vod_des',
             'Rebio Jaru' : 'RJA_vod_des',
             'Santarem km67' : 'S67_vod_des'})

# concatenate the two dataframe
vod_df = pd.merge(vod_midnight_df,vod_midday_df,
                 on=['Year', 'Date'])

# process the datetime
date_series = pd.to_datetime(vod_df['Date'])
# get doy
doy = np.array([date_series[i].timetuple().tm_yday for i in np.arange(len(date_series))])
# get year
year = np.array([date_series[i].year for i in np.arange(len(date_series))])
# get month
month = np.array([date_series[i].month for i in np.arange(len(date_series))])

# write year and doy into the df
vod_df['Year'] = pd.Series(data=year,index=vod_df.index)
vod_df['Month'] = pd.Series(data=month,index=vod_df.index)
vod_df['doy'] = pd.Series(data=doy,index=vod_df.index)

# delete empty cols
del vod_df['Unnamed: 10_x']
del vod_df['Unnamed: 11_x']
del vod_df['94_x']
del vod_df['Unnamed: 10_y']
del vod_df['Unnamed: 11_y']
del vod_df['94_y']

# save raw data before interpolation
vod_df = vod_df.sort_values(['Year','doy'])
vod_df.to_csv('obs_vod_raw.csv')

# interpolate into daily values
data_len = vod_df.shape[0]
# loop over each entry
for irow in np.arange(data_len):
    # get the current year
    row_year = int(vod_df['Year'].values[irow])
    cur_doy = int(vod_df['doy'].values[irow])

    # get the total day within the year
    last_day = date(row_year,12,31)
    day_num = last_day.timetuple().tm_yday

    if irow == data_len - 1:
        next_doy = day_num + 1
    elif vod_df['Year'].values[irow] == row_year:
        next_doy = int(vod_df['doy'].values[irow+1])
    else:
        next_doy = day_num + 1

    # replicate the current entry for day_in_week - 1 times and append to the
    # dataframe
    for iday in np.arange(cur_doy+1,next_doy):
        test_day = date(row_year,1,1) + timedelta(days=int(iday-1))
        new_rec = vod_df.iloc[irow,:].copy()
        new_rec['Month'] = test_day.month
        new_rec['doy'] = test_day.timetuple().tm_yday
        vod_df = vod_df.append([new_rec],ignore_index=True)

# finally sort the dataframe using year and doy
vod_df = vod_df.sort_values(['Year','doy'])
vod_df.to_csv('obs_vod.csv')

#---------------- QuikScat

qs_fn = '../Data/p11_qst_weekly.xlsx'

qs_df = pd.read_excel(qs_fn,skiprows=1)

# rename the columns
qs_df = qs_df.rename(
    columns={
        'Unnamed: 0' : 'Year',
        'Unnamed: 1' : 'Week',
        "'Brasilia'"   : 'BSB_qst_asc_H',
        "'Brasilia'.1"   : 'BSB_qst_des_H',
        "'Brasilia'.2"   : 'BSB_qst_asc_V',
        "'Brasilia'.3"   : 'BSB_qst_des_V',
        "'Caxiuana'"   : 'CAX_qst_asc_H',
        "'Caxiuana'.1"   : 'CAX_qst_des_H',
        "'Caxiuana'.2"   : 'CAX_qst_asc_V',
        "'Caxiuana'.3"   : 'CAX_qst_des_V',
        "'Paracou'"   : 'GYF_qst_asc_H',
        "'Paracou'.1"   : 'GYF_qst_des_H',
        "'Paracou'.2"   : 'GYF_qst_asc_V',
        "'Paracou'.3"   : 'GYF_qst_des_V',
        "'Manaus-K34'"   : 'M34_qst_asc_H',
        "'Manaus-K34'.1"   : 'M34_qst_des_H',
        "'Manaus-K34'.2"   : 'M34_qst_asc_V',
        "'Manaus-K34'.3"   : 'M34_qst_des_V',
        "'Pe-de-Gigante'"   : 'PDG_qst_asc_H',
        "'Pe-de-Gigante'.1"   : 'PDG_qst_des_H',
        "'Pe-de-Gigante'.2"   : 'PDG_qst_asc_V',
        "'Pe-de-Gigante'.3"   : 'PDG_qst_des_V',
        "'Petrolina'"   : 'PNZ_qst_asc_H',
        "'Petrolina'.1"   : 'PNZ_qst_des_H',
        "'Petrolina'.2"   : 'PNZ_qst_asc_V',
        "'Petrolina'.3"   : 'PNZ_qst_des_V',
        "'Rebio-Jaru'"   : 'RJA_qst_asc_H',
        "'Rebio-Jaru'.1"   : 'RJA_qst_des_H',
        "'Rebio-Jaru'.2"   : 'RJA_qst_asc_V',
        "'Rebio-Jaru'.3"   : 'RJA_qst_des_V',
        "'Santarem-km67'"   : 'S67_qst_asc_H',
        "'Santarem-km67'.1"   : 'S67_qst_des_H',
        "'Santarem-km67'.2"   : 'S67_qst_asc_V',
        "'Santarem-km67'.3"   : 'S67_qst_des_V'
            })

# interpolate qst into daily values
# first add month and doy
qs_df['Month'] = pd.Series(data=np.zeros((qs_df.shape[0],)) *
                           np.nan,index=qs_df.index)
qs_df['doy'] = pd.Series(data=np.zeros((qs_df.shape[0],)) *
                           np.nan,index=qs_df.index)

data_len = qs_df.shape[0]

# loop over each entry
cur_year = np.nan
cur_doy = 1
for irow in np.arange(data_len):
    # get the current year
    if np.isnan(qs_df['Year'].values[irow]):
        row_year = int(cur_year)
        qs_df.loc[irow,'Year'] = cur_year
    else:
        row_year = int(qs_df['Year'].values[irow])
        cur_year = row_year
        # reset day of year
        cur_doy = 1

    # get the total day within the year
    last_day = date(row_year,12,31)
    day_num = last_day.timetuple().tm_yday
    # get the number of days within this week
    day_in_week = np.minimum(7,day_num-cur_doy+1)

    # get the current week
    row_week = qs_df['Week']

    # set Month and DOY
    cur_day = date(row_year,1,1) + timedelta(days=int(cur_doy)-1)
    qs_df.loc[irow,'Month'] = cur_day.month
    qs_df.loc[irow,'doy'] = cur_day.timetuple().tm_yday

    # now we have finished processing this day, increase the doy
    cur_doy += 1

    # replicate the current entry for day_in_week - 1 times and append to the
    # dataframe
    for iday in np.arange(1,day_in_week):
        test_day = cur_day + timedelta(days=int(iday))
        new_rec = qs_df.iloc[irow,:].copy()
        new_rec['Month'] = test_day.month
        new_rec['doy'] = test_day.timetuple().tm_yday
        cur_doy += 1
        qs_df = qs_df.append([new_rec],ignore_index=True)

# finally sort the dataframe using year and doy
qs_df = qs_df.sort_values(['Year','doy'])
qs_df.to_csv('obs_qst.csv')

#
#
## read the simulated cwc
#site_array = ['BSB','CAX','GYF','M34','PDG','PNZ','RJA','S67']
#
#
## date vars
#col_list = []
## add PFT-specific state variables
#state_vars = ['LAI','CWC','PSI_LEAF_MAX','PSI_LEAF_MIN','PSI_LEAF_DELTA']
#hite_list = [5,10,15,20]
#for var_name in state_vars:
#    col_list.append(var_name + '_all')
#    for hite in hite_list:
#        col_list.append(var_name + '_top{:d}'.format(hite))
#
#
#
#
## loop over site
#for isite, site_name in enumerate(site_array):
#    site_fn = '../Data/{:s}_2000s_daily.csv'.format(site_name)
#    site_df = pd.read_csv(site_fn)
#    # find the corresponding DMEAN values
#
#    # loop over years
#    for iyear,year in enumerate(np.arange(2003,2010+1)):
#        vod_year_idx = vod_df['Year'] == year
#        vod_doy = vod_df['doy'][vod_year_idx]
#
#        site_year_idx = site_df['year'] == year
#        print(site_df.head())
#
#        # loop over col_str
#        for icol, col_str in enumerate(col_list):
#            new_col_str = site_name + '_' + col_str
#            vod_df.loc[vod_year_idx,new_col_str] = \
#                site_df.loc[site_year_idx,col_str].values[vod_doy-1]
#
#
## save the combined data
#output_fn = '../Data/site_vod_vs_sim.csv'
#vod_df.to_csv(output_fn)


# -*- coding: utf-8 -*-

"""Plot a snapshot of particles"""

# -----------------------------------
# plotcloud.py
#
# Bjørn Ådlandsvik, <bjorn@imr.no>
# Institute of Marine Research
# 2009-11-01
# -----------------------------------

import numpy as np
import matplotlib.pyplot as plt
from netCDF4 import Dataset

# ----------------------
# User settings
# ----------------------

# Particle file name
fname = "FW.nc"
#fname = "BW.nc"

# Time frame
tframe = 1

# Geographical extent
lon0, lat0 = -10, 60
lon1, lat1 =  60, 80

# ------------------------
# Read data
# ------------------------

# Particle data

f = Dataset(fname)

X = f.variables['X'][tframe,:]
Y = f.variables['Y'][tframe,:]
life = f.variables['life'][tframe,:]

X = X[life==1]
Y = Y[life==1]

f.close()

# Get coast data
nc = Dataset('kyst.nc')
Xcoast = nc.variables['X'][:]
Ycoast = nc.variables['Y'][:]
S = nc.variables['S'][:]
nc.close()

# Get topography data
i0 = (lon0+180)*60
j0 = (lat0+90)*60
imax = (lon1-lon0)*60 + 1
jmax = (lat1-lat0)*60 + 1
f = Dataset('/home/bjorn/Data/etopo1/ETOPO1_Bed_g_gmt4.grd')
H = -f.variables['z'][j0:j0+jmax, i0:i0+imax]
lon = f.variables['lon'][i0:i0+imax]
lat = f.variables['lat'][j0:j0+jmax]
f.close()

# ----------------------
# Plot
# ---------------------

# Plot coast data
for i in xrange(len(S)-1):
    plt.fill(Xcoast[S[i]:S[i+1]], Ycoast[S[i]:S[i+1]], facecolor=(0.2,0.8,0.2))

# Plot bottom contours
plt.contour(lon, lat, H, levels=(100,200,400), colors='r')

# Plot cloud of particle tracks
plt.plot(X, Y, 'b.')

# Limit plot area
plt.axis((lon0, lon1, lat0, lat1))

plt.show()


# filter(function, 반복객체)

# filter() 미사용 코드
def junior(ages):
    result = []
    for age in ages:
        if age <= 19:
            result.append(age)
    return result

def senior(ages):
    result = []
    for age in ages:
        if age >= 19:
            result.append(age)
    return result

age = list(range(1, 40))
print('junior :', junior(age))
print('senior :', senior(age))

# filter() 사용 코드
age = list(range(1, 40))
print('junior :', list(filter(lambda x: x <= 19, age)))
print('senior :', list(filter(lambda x: x >= 20, age)))

# lambda 함수 개념
# x가 들어오면 x에 1을 더한값을 반환해라
# map(), filter(), reduce(), sorted() 등과 같이 씀

# def function_name( 인자 ):
#     return 반환값

# lambda 인자 : 반환값

def function_name(x):
    return x+1

lambda x: x+1

# 함수 호출

def function_name(x):
    return x+1

function_name(10)

(lambda x: x+1)(10)

# 예시 - 짝수만 출력
a, b, c = map(int, input().split())
print(*(lambda a: a % 2 == 0, [a, b, c]))

# *args, **kwargs

# 가변인자를 받을때 앞에 아스테리크를 붙임
# 디폴트값 지정 가능, 생략 가능 예) third = None
# 들어오는 인자의 갯수를 모르거나 들어오는 인자를 모두 받아 처리하고자 할때 사용

# map()맵 함수


import datetime as dt
import pandas_datareader as web
import numpy as np
import pandas as pd
import warnings
warnings.filterwarnings('ignore')

class Portfolio:
	def __init__(self, basketPortfolio, startDate, endDate, benchmarkTicker):
		"""
		Args:
		    basketPortfolio (dict): stocks in the portfolio and their corresponding number of shares
		    startDate (datetime object): start date of the lifetime of the portfolio
		    endDate (datetime object): end date of the lifetime of the portfolio
		    benchmarkTicker (str): ticker of the portfolio's benchmark index
		"""
		self.basketPortfolio = basketPortfolio

		self.startDate = startDate
		self.endDate = endDate

		self.benchmarkTicker = benchmarkTicker
		self.symbols = list(self.basketPortfolio.keys())

		# Get the historical adjusted close stock prices for the stocks in the portfolio
		self.priceData = web.DataReader(self.symbols, "yahoo" , self.startDate, self.endDate)['Adj Close']

		self.priceData['Portfolio Value'] = None
		self.priceData['Daily Returns'] = None

		totalValue = 0

		# Calculate daily portfolio value and daily returns
		for row in range(0, len(self.priceData)): # iterate through rows
			for i in self.basketPortfolio: # iterate through tickers
				totalValue += self.priceData[i].iloc[row] * self.basketPortfolio[i]
			self.priceData['Portfolio Value'].iloc[row] = totalValue
			if row != 0:
				self.priceData['Daily Returns'] = self.priceData['Portfolio Value'].pct_change(1)
			totalValue = 0

	def averageDailyReturn(self):
		"""Used CAGR to calculate the average daily return
		CAGR is a good measure to determine what an investment yields on a daily compounded basis

		CAGR = (final portfolio value/initial portfolio value) ^ (1/business days in between the start date and the end date) - 1

		Returns:
		    float: the average daily percent return of the portfolio
		"""
		# The length of the dataframe is the number of trading days in the lifespan of the portfolio
		averageDailyGain = (((self.priceData['Portfolio Value'].iloc[len(self.priceData) - 1])/self.priceData['Portfolio Value'].iloc[0])**(1/len(self.priceData)) - 1)
		return averageDailyGain

	def volatility(self):
		"""Calculate the standard deviation of the daily returns of portfolio value
		Made the variable that is returned a private class variable, so that other methods in the class can access the portfolio volatility value

		Returns:
		    float: the portfolio's volatility over the lifespan of the portfolio
		"""
		self.__portfolioVolatility = self.priceData['Daily Returns'].std()
		return self.__portfolioVolatility

	def riskRatio(self):
		"""
		Returns:
		    float: volatility ratio of the portfolio compared to the volatility of the benchmark over the lifetime of the portfolio
		"""
		# Get the historical adjusted close stock prices for the portfolio's benchmark
		self.benchmarkData = pd.DataFrame(web.DataReader(self.benchmarkTicker, "yahoo" , self.startDate, self.endDate)['Adj Close'])
		self.benchmarkData = self.benchmarkData.assign(DailyReturns = "None")

		# Calculate daily returns of the benchmark index
		for row in range(0, len(self.benchmarkData)):
			self.benchmarkData['DailyReturns'] = self.benchmarkData['Adj Close'].pct_change(1)

		# Calculate standard deviation of the benchmark's daily returns
		self.__benchmarkVolatility = self.benchmarkData['DailyReturns'].std()

		# Risk Ratio = Portfolio Volatility/Benchmark Volatility
		return self.__portfolioVolatility/self.__benchmarkVolatility

	def marginalVolatility(self, ticker, shares):
		"""
		Args:
		    ticker (str): ticker of the stock to be added to the portfolio
		    shares (int): number of shares of the stock to be added to the portfolio

		Returns:
		    float: the difference in volatility of the portfolio if the specified number of shares of the ticker was added to the
		    portfolio
		"""
		marginalPortfolio = self.basketPortfolio
		# If the stock is already in the portfolio, update the share count
		if ticker in marginalPortfolio:
			marginalPortfolio[ticker] += shares
		# Otherwise add a new key/value pair to the dictionary with the new stock ticker and number of shares
		else:
			marginalPortfolio[ticker] = shares

		marginalSymbols = list(marginalPortfolio.keys())

		# Get the historical adjusted close stock prices for the stocks in the portfolio with the included stock and the number of shares
		marginalData = web.DataReader(marginalSymbols, "yahoo", self.startDate, self.endDate)['Adj Close']

		marginalData['Portfolio Value'] = None
		marginalData['Daily Returns'] = None
		totalValue = 0

		# Calculate daily portfolio value and daily returns
		for row in range(0, len(marginalData)): # iterate through rows
			for i in marginalPortfolio: # iterate through tickers
				totalValue += marginalData[i].iloc[row] * marginalPortfolio[i]
			marginalData['Portfolio Value'].iloc[row] = totalValue
			if row != 0:
				marginalData['Daily Returns'] = marginalData['Portfolio Value'].pct_change(1)
			totalValue = 0

		# Calculate standard deviation of the portfolio's daily returns with the included stock and the number of shares
		self.__marginalPortfolioVolatility = marginalData['Daily Returns'].std()
		return self.__marginalPortfolioVolatility - self.__portfolioVolatility

	def maxDrawDown(self):
		"""
		Returns:
		    float: maximum drawdown of the portfolio (largest drop from peak to trough in the lifespan of the portfolio)
		"""
		# Initialize values to 0
		peak = 0
		trough = 0
		DDCurrent = 0
		DDTotal = 0

		# Iterate through the daily portfolio values
		for value in self.priceData['Portfolio Value']:
			# If the current value is greater than the current peak, set the peak equal to the current value
			# Reset trough value
			# Reset current dropdown value
			if value > peak:
				peak = value
				trough = 0
				DDCurrent = 0

			# If trough has not been initialized and the current value is less than the peak, set the trough equal to the current value
			elif value < peak and trough == 0:
				trough = value

			# If the current value is less than the trough, set the trough equal to the current value
			else:
				trough = value

			# When both the peak and trough are initialized, calculate current dropdown value
			if peak != 0 and trough != 0:
				DDCurrent = peak - trough

			# If the current dropdown value is greater than the previous dropdown value, update with the overall max dropdown value
			if DDCurrent > DDTotal:
				DDTotal = DDCurrent
		self.__maxDrawDownValue = DDTotal/peak
		return self.__maxDrawDownValue

basket = {"AAPL": 50, "GME": 150, "TSLA": 5, "AAL": 200, "AMZN": 1}
start = dt.date(2019, 1, 1)
end = dt.date(2021, 3, 1)
benchmark = "VOO"

portfolio = Portfolio(basket, start, end, benchmark)

print("Average Daily Return: " + str(round(portfolio.averageDailyReturn()*100, 3)) + "%")
print("Volatility: " + str(round(portfolio.volatility()*100, 3)) + "%")
print("Risk Ratio: " + str(round(portfolio.riskRatio(), 3)))
print("Marginal Volatility: " + str(round(portfolio.marginalVolatility("GME", 300)*100, 3)) + "%")
print("Max Draw Down: " + str(round(portfolio.maxDrawDown()*100, 3)) + "%")


import cv2

# Load a photo of Donald Trump
trump = cv2.imread('images/trump.jpg', cv2.IMREAD_COLOR)
# Resize Trump
trump = cv2.resize(trump,(800,450))
# Show the image of Trump
cv2.imshow("tash", trump)
# Keep the image open until a key is pressed
cv2.waitKey(0)

# Challenge: Use the openCV documentation to draw red eyes on Donald Trump
# https://docs.opencv.org/3.1.0/dc/da5/tutorial_py_drawing_functions.html


#
# @lc app=leetcode id=212 lang=python
#
# [212] Word Search II
#
class Solution(object):
    def findWords(self, board, words):
        """
        :type board: List[List[str]]
        :type words: List[str]
        :rtype: List[str]
        """

    #     # This solution is LTE
    #     # dfs + memo search
    #     if not board or not board[0] or not words: return []

    #     rows = len(board)
    #     cols = len(board[0])

    #     # prevent LTE
    #     word_set = set(words)
    #     prefix_set = set()
    #     for word in words:
    #         for i in range(len(word)):
    #             prefix_set.add(word[:i + 1])

    #     res = set()
    #     for word in words:
    #         for r in range(rows):
    #             for c in range(cols):
    #                 # check if word exists
    #                 self._dfs(board, word, r, c, rows, cols, word_set, prefix_set, [], res)

    #     return list(res)

    # # length of the word will change when recursed
    # def _dfs(self, board, word, r, c, rows, cols, word_set, prefix_set, visited, res):
    #     if word not in prefix_set:
    #         return

    #     if word in word_set:
    #         res.add(word)

    #     # search criteria
    #     # boundary check + isVisisted + cannot be self
    #     if r < 0 or r >= rows or c < 0 or c >= cols or \
    #         word[0] != board[r][c] or (r, c) in visited:
    #         return

    #     # word match, proceed next step
    #     visited.append((r, c))
    #     self._dfs(board, word[1:], r - 1, c, rows, cols, word_set, prefix_set, visited, res)
    #     self._dfs(board, word[1:], r + 1, c, rows, cols, word_set, prefix_set, visited, res)
    #     self._dfs(board, word[1:], r, c + 1, rows, cols, word_set, prefix_set, visited, res)
    #     self._dfs(board, word[1:], r, c - 1, rows, cols, word_set, prefix_set, visited, res)
    #     visited.pop()   # reset visited

        if not board or not board[0] or not words: return []

        res = []
        trie = Trie()
        root = trie.root

        # construct the trie
        for word in words:
            trie.insert(word)

        rows = len(board)
        cols = len(board[0])
        for r in range(rows):
            for c in range(cols):
                self._dfs(board, root, r, c, rows, cols, "", res)

        return res

    def _dfs(self, board, node, r, c, rows, cols, path, res):
        if node.isWord:
            res.append(path)
            node.isWord = False     # reset

        # boundary check
        if r < 0 or r >= rows or \
            c < 0 or c >= cols:
            return

        tmp = board[r][c]
        node = node.children.get(tmp)   # find cur node (char)
        if not node: return

        board[r][c] = '#'
        self._dfs(board, node, r+1, c, rows, cols, path + tmp, res)
        self._dfs(board, node, r-1, c, rows, cols, path + tmp, res)
        self._dfs(board, node, r, c+1, rows, cols, path + tmp, res)
        self._dfs(board, node, r, c-1, rows, cols, path + tmp, res)
        board[r][c] = tmp # reset

class Trie():
    def __init__(self):
        self.root = TrieNode()

    def insert(self, word):
        node = self.root
        for w in word:
            node = node.children[w]
        node.isWord = True

    def search(self, word):
        node = self.root
        for w in word:
            node = node.children.get(w)
            if not node: return False
        return node.isWord

class TrieNode():
    def __init__(self):
        # default value is a TrieNode
        self.children = collections.defaultdict(TrieNode)
        self.isWord = False


import urllib.request
import simplejson

# The request also includes the userip parameter which provides the end
# user's IP address. Doing so will help distinguish this legitimate
# server-side traffic from traffic which doesn't come from an end-user.
url = ('https://ajax.googleapis.com/ajax/services/search/web'
       '?v=1.0&q=python&userip=121.133.116.69')

url2 ='https://ajax.googleapis.com/ajax/services/search/web?v=1.0&q=python&userip=121.133.116.69'

def get_search_result(keyword, user_ip):
    url = ('https://ajax.googleapis.com/ajax/services/search/web'
           '?v=1.0&q='
           +keyword
           +'&userid='
           +user_ip)
    return url

for i in range(1,2) :
    url = get_search_result('python', '121,13,116,69')

#    request = urllib.request.Request(url, None, {'Referer': '1.1.1.12:8888'})
    request = urllib.request.Request(url)

    response = urllib.request.urlopen(request)

    print(response)

    print (i)
    print ('\n###########\n')

    # Process the JSON string.
    results = simplejson.load(response)
    decoded_data = simplejson.load(response)
    # now have some fun with the results...
    print (results)

    result_len = len(decoded_data['responseData']['results'])

    for i in range(0, result_len):
        print (i)
        print (decoded_data['responseData']['results'][i]['content'])


from TensorFloow.PCDataProc import DataSet
from Utils import *
import tensorflow.contrib.slim as slim

#region GRAPH GENERATION AND TRAINING-----------------------------------------------------------------------------------

class NetworkTF:
#  OUTPUT_LABELS=["PCadeathnew","Recur0306","Deathfup","fumos"]
#  N_INPUTS=222

  def __init__(self,dataPath,hiddenLayers,neuronsPerLayer,batchSize,weightScaleFactor=1,moreCols=False,dropLabels=None):
    #generate data
    self.dataSet=DataSet(dataPath,0.7,True,batchSize,moreCols,dropLabels)
    self.dataSet.GenerateScalers()
    self.dataSet.ScaleTraining()
    self.dataSet.ScaleTesting()
    self.sess=None
    self.grads=None
    self.graph=tf.Graph()
    with self.graph.as_default():
      #vars and constants
      inputsTF=tf.placeholder(tf.float32,[None,self.dataSet.nInputs],name='inputsPL')
      outputsTF=tf.placeholder(tf.float32,[None,self.dataSet.nOutputs],name='outputsPL')
      learnRateTF=tf.placeholder(tf.float32,name='learnRatePL')
      keepProbTF=tf.placeholder(tf.float32,name='keepProbPL')
      #building network
      #netVarsTF,netOpsTF=GenNetworkTF(inputsTF,len(OUTPUT_LABELS),[200,100,50,25,10],tf.nn.relu)
      netVarsTF,netOpsTF=self._GenLayersTF(inputsTF,self.dataSet.nOutputs,[neuronsPerLayer]*hiddenLayers,tf.nn.relu,weightScaleFactor)
      #training network
      netOutTF=netOpsTF[-1]
      lastNetOpTF=tf.sigmoid(tf.nn.dropout(netOutTF,keepProbTF))
      self.netOutProcTF,self.outputsProcTF=self._ProcOutputsTF(lastNetOpTF,outputsTF)
      #self.netOutProcTF,self.outputsProcTF=self._ProcOutputsTF(tf.nn.dropout(netOutTF,keepProbTF),outputsTF)
      #  netOutProcTF=tf.sigmoid(tf.nn.dropout(netOutTF,KEEP_PROB))
      #  outputsProcTF=outputsTF
      self.errorTF=tf.reduce_mean(tf.pow(self.netOutProcTF-self.outputsProcTF,2))
      #errorTrainTF=tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(netOutProcTF,outputsProcTF))
      self.trainTF=tf.train.AdamOptimizer(learnRateTF).minimize(self.errorTF)
      #  trainTF=tf.train.GradientDescentOptimizer(learnRateTF).minimize(errorTrainTF)
      #testing network
      #  netOutProcTestTF,outputsProcTestTF=ProcOutputsTF(netOutTF,outputsTF)
      #self.errorTestTF=tf.reduce_mean(tf.pow(self.netOutProcTF-self.outputsProcTF,2))
      self.gradTF=self._GradientFunctionTF(lastNetOpTF,inputsTF)

      #init
      self.initTF=tf.global_variables_initializer()

  @staticmethod
  def _GradientFunctionTF(outLayerTF,gradientLayerTF):
    outLen=outLayerTF.get_shape()[1]
    outputMaskTF=tf.placeholder(tf.float32,[None,outLen],name='outputMaskPL')
    #ret.append(tf.gradients(outLayerTF,gradientLayerTF,outLayer[i]))
    return tf.gradients(outLayerTF,gradientLayerTF,outputMaskTF)

  @staticmethod
  def _GenLayersTF(inputs,nOutputs,neuronsPerHiddenLayer,Activation,weightScaleFactor):
    neuronsPerHiddenLayer.insert(0,int(inputs.get_shape()[1]))
    neuronsPerHiddenLayer.append(nOutputs)
    network=[]#will contain weights and biases for each layer
    ops=[]#will contain operations that run the network, [sum,ActivationF] for each layer
    for i in range(len(neuronsPerHiddenLayer)-1):
      #xavier init
      weights=tf.Variable(tf.truncated_normal(shape=[neuronsPerHiddenLayer[i],neuronsPerHiddenLayer[i+1]],
                                              stddev=weightScaleFactor/(neuronsPerHiddenLayer[i]**0.5)))
      #biases=tf.Variable(tf.zeros(shape=[neuronsPerHiddenLayer[i+1]]))
      biases=tf.Variable(tf.zeros(shape=[neuronsPerHiddenLayer[i+1]]))
      network.append([weights,biases])
    ops.append(inputs)
    for i in range(len(network)-1):
      ops.append(Activation(tf.matmul(ops[i],network[i][0])+network[i][1]))
    #no activation function on the last layer, allows customm activation function
    ops.append(tf.matmul(ops[-1],network[-1][0])+network[-1][1])
    return network,ops[1:]

  @staticmethod
  def _ProcOutputsTF(netOutTF,outputsTF):
    outBoolsTF,outTimesTF=tf.split(outputsTF,2,1)
    netBoolsTF,netTimesTF=tf.split(netOutTF,2,1)
    netOutProcTF=tf.concat([netBoolsTF,netTimesTF],1)
    outputsProcTF=tf.concat([outBoolsTF,outTimesTF],1)
    return netOutProcTF,outputsProcTF

  def StartSession(self):
    self.sess=tf.Session(graph=self.graph)
    self.sess.run(self.initTF)

  def EndSession(self):
    self.sess.close()
    self.sess=None

  def Train(self,nIterations,learnRate,keepProb=1.0,recordError=False):
    errorRec=None
    if recordError:errorRec=[]
    for j in range(nIterations):
      batchInputs,batchOutputs=self.dataSet.GetTrainingBatches(True)
      for i in range(batchInputs.shape[0]):
        netOutProc,outProc,errorRes,TrainRes=self.sess.run((self.netOutProcTF,self.outputsProcTF,self.errorTF,self.trainTF),
                                                      feed_dict={'inputsPL:0':batchInputs[i],'outputsPL:0':batchOutputs[i],'learnRatePL:0':learnRate,'keepProbPL:0':keepProb})
        if errorRec is not None: errorRec.append(errorRes)
    return errorRec

  def GradientValues(self,inputSet,outputSet):
    ret=[]
    outputMasks=np.identity(outputSet.shape[1])
    for outputMask in outputMasks:
      #mask so gradients are per output variable
      mask=np.tile(outputMask,outputSet.shape[0]).reshape(outputSet.shape[0],outputSet.shape[1])
      gradVals=self.sess.run(self.gradTF,feed_dict={'inputsPL:0':inputSet,'outputsPL:0':outputSet,'keepProbPL:0':1,'outputMaskPL:0':mask})
      gradVals=gradVals[0]
      ret.append(gradVals)
    return np.asarray(ret)

  def Test(self,panadasPath=None):
    error,out=self.sess.run((self.errorTF,self.netOutProcTF),feed_dict={'inputsPL:0':self.dataSet.inputsTest,
                           'outputsPL:0':self.dataSet.outputsTest,'keepProbPL:0':1})
    if panadasPath:
      outUnscaled=self.dataSet.InverseScaleOutput(out)
      testUnscaled=self.dataSet.InverseScaleOutput(self.dataSet.outputsTest)
      self.dataSet.OutputToPandas(testUnscaled,outUnscaled,panadasPath)
    return error


import subprocess
import time
import logging
import os
import signal

log = logging.getLogger(__name__)

def runpreloadcommand(cmd,timeout):
    process = subprocess.Popen([cmd], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    processRc = None
    handleprocess = True
    counter = 0
    stdout = ''
    stderr = ''
    while handleprocess:
        counter += 1
        time.sleep(1)
        cout,cerr = process.communicate()
        stdout += cout
        stderr += cerr
        process.poll()
        processRc = process.returncode
        if processRc != None:
            break
        if counter == timeout:
            os.kill(process.pid, signal.SIGQUIT)
        if counter > timeout:
            os.kill(process.pid, signal.SIGKILL)
            processRc = -9
            break
    return (processRc,stdout,stderr)

class uploaderScp:
    def __init__(self):
        self.remotePrefix = None
        self.log = logging.getLogger("uploaderScp")
    def _getfilepath(self,remotePath):
        if self.remotePrefix != None:
            return self.remotePrefix + remotePath
        else:
            return remotePath
    def exists(self,remotePath):
        prefix = self.remotePrefix.split(":")
        fuill = "%s/%s" % (prefix[1],remotePath)
        cmd = "ssh %s stat %s" % (prefix[0],fuill)
        log.debug(cmd)
        timeout = 10
        return runpreloadcommand(cmd,timeout)

    def delete(self,remotePath):
        prefix = self.remotePrefix.split(":")
        fuill = "%s/%s" % (prefix[1],remotePath)

        cmd = "ssh %s rm %s" % (prefix[0],fuill)
        timeout = 10
        return runpreloadcommand(cmd,timeout)

    def upload(self,localpath,remotePath):
        self.log = logging.getLogger("uploaderScp.upload")
        cmd = "scp %s %s" % (localpath,remotePath)
        self.log.info("Attempting:%s" % (cmd))
        rc,stdout,stderr = runpreloadcommand(cmd,10)
        if rc != 0:
            self.log.debug(cmd)
            self.log.error( stderr)
            return (rc,stdout,stderr)
        return (rc,stdout,stderr)
        return self.replace(localpath,remotePath)

    def replace(self,localpath,remotePath):
        self.log = logging.getLogger("uploaderScp.replace")
        cmd = "scp %s %s" % (localpath,remotePath)
        self.log.info("Attempting:%s" % (cmd))
        rc,stdout,stderr = runpreloadcommand(cmd,10)
        if rc != 0:
            self.log.debug(cmd)
            self.log.error( stderr)
            return (rc,stdout,stderr)
        return (rc,stdout,stderr)
    def download(self,remotePath,localpath):
        self.log = logging.getLogger("uploaderScp.download")
        cmd = "scp %s %s" % (remotePath,localpath)
        self.log.info("Attempting:%s" % (cmd))
        rc,stdout,stderr = runpreloadcommand(cmd,10)
        if rc != 0:
            self.log.debug(cmd)
            self.log.error( stderr)
            return (rc,stdout,stderr)
        return (rc,stdout,stderr)


#Exampe for a bounded lookahead, using node
#this relies on other defitions such as is_min and is_max

from search_mm import node

def minimax_bl(self,node,depth):
    if goal(node):
        return node.staticvalue

    # Here we expect to have defined node.depth and keep it rolling
    if node.depth==depth:
        return node.staticvalue

    if is_min(node):
        return min(minimax(node.succ()))

    if is_max(node):
        return max(minimax(node.succ()))


# def isStringInArray(a1, a2):
#     i = len(a1)
#     j = len(a2)
#     m = 0
#     n = 0

#     while (m<i and n<j):
#         if (a1[m] == a2[n]):
#             m = m + 1
#         n = n + 1
#     if (m == i):
#         return "present"
#     else:
#         return "not present"

# print(isStringInArray("cam","cancun"))

# def chk_palindrome(A):
#     i = 0
#     j = len(A)-1
#     if len(A) == 1:
#         return "palindrome"
#     while i<j and A[i]==A[j]:
#         i += 1
#         j -= 1
#     if i>j or i==j:
#         return "palindrome"
#     else:
#         return "not palindrome"

# print(chk_palindrome("a"))

def cleanPhone(N):
    N = N.replace("-","")
    N = N.replace(" ","")
    final = ""
    if len(N)%10 == 0:
        final = N[0:3]+"-"+N[3:6]+"-"+N[6:]
    elif len(N) > 10:
        final = N[0:3]+"-"+N[3:6]+"-"+N[6:8]+"-"+N[8:]
    elif len(N) < 10:
        final = N[0:3]+"-"+N[3:6]+"-"+N[6:]
    return final

print(cleanPhone("123-456---78   "))


#! /usr/bin/env python
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
# pylint: disable=E1137,E1136,E0110
import sys
import os
import time
import datetime
import collections
import argparse

import numpy as np

import logging
LOGGER = logging.getLogger("server")

import rpyc
from rpyc.utils.server import OneShotServer, ThreadedServer

from web_tool.ModelSessionKerasExample import KerasDenseFineTune
from web_tool.ModelSessionPyTorchExample import TorchFineTuning
from web_tool.Utils import setup_logging, serialize, deserialize

class MyService(rpyc.Service):

    def __init__(self, model):
        self.model = model

    def on_connect(self, conn):
        pass

    def on_disconnect(self, conn):
        pass

    def exposed_last_tile(self):
        return serialize(self.model.last_tile)

    def exposed_run(self, tile, inference_mode=False):
        tile = deserialize(tile) # need to serialize/deserialize numpy arrays
        output = self.model.run(tile, inference_mode)
        return serialize(output) # need to serialize/deserialize numpy arrays

    def exposed_retrain(self):
        return self.model.retrain()

    def exposed_add_sample_point(self, row, col, class_idx):
        return self.model.add_sample_point(row, col, class_idx)

    def exposed_undo(self):
        return self.model.undo()

    def exposed_reset(self):
        return self.model.reset()

    def exposed_save_state_to(self, directory):
        return self.model.save_state_to(directory)

    def exposed_load_state_from(self, directory):
        return self.model.load_state_from(directory)

def main():
    global MODEL
    parser = argparse.ArgumentParser(description="AI for Earth Land Cover Worker")

    parser.add_argument("-v", "--verbose", action="store_true", help="Enable verbose debugging", default=False)

    parser.add_argument("--port", action="store", type=int, help="Port we are listenning on", default=0)
    parser.add_argument("--model", action="store", dest="model",
        choices=[
            "keras_dense",
            "pytorch"
        ],
        help="Model to use", required=True
    )
    parser.add_argument("--model_fn", action="store", dest="model_fn", type=str, help="Model fn to use", default=None)
    parser.add_argument("--fine_tune_layer", action="store", dest="fine_tune_layer", type=int, help="Layer of model to fine tune", default=-2)

    parser.add_argument("--gpu", action="store", dest="gpuid", type=int, help="GPU to use", required=False)

    args = parser.parse_args(sys.argv[1:])

    # Setup logging
    log_path = os.path.join(os.getcwd(), "tmp/logs/")
    setup_logging(log_path, "worker")

    # Setup model
    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
    os.environ["CUDA_VISIBLE_DEVICES"] = "" if args.gpuid is None else str(args.gpuid)
    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

    if args.model == "keras_dense":
        model = KerasDenseFineTune(args.model_fn, args.gpuid, args.fine_tune_layer)
    elif args.model == "pytorch":
        model = TorchFineTuning(args.model_fn, args.gpuid, args.fine_tune_layer)
    else:
        raise NotImplementedError("The given model type is not implemented yet.")

    t = OneShotServer(MyService(model), port=args.port)
    t.start()

if __name__ == "__main__":
    main()


from torch.utils.data import Dataset
from transformers import BertTokenizer
import torch
import logging

class WOSDataset(Dataset):
    logger = logging.getLogger(__name__)

    def __init__(self,
                 file_path: str = './data/WOS11967/',
                 tokenizer: str = 'bert-base-cased',
                 split: str = 'train'):
        super().__init__()
        assert split in {'training', 'validation', 'test'}
        self.file_path = file_path + split

        self.x = []
        self.y = []

        try:
            with open(self.file_path+'/X.txt', 'r') as feature_data_file:
                for line in feature_data_file:
                    self.x.append(line.strip())
        except FileNotFoundError:
            logging.error(
                f'### Failed to find data file: {self.file_path}X.txt')
            raise FileNotFoundError

        tag_set = set()
        try:
            with open(self.file_path+'/Y.txt', 'r') as feature_data_file:
                for line in feature_data_file:
                    self.y.append(int(line.strip()))
                    tag_set.add(int(line.strip()))
        except FileNotFoundError:
            logging.error(
                f'### Failed to find data file: {self.file_path}YL1.txt')
            raise FileNotFoundError
        self.num_labels = len(tag_set)
        assert len(self.x) == len(self.y)

    def __len__(self):
        return len(self.x)

    def __getitem__(self, idx):
        sample = {}
        sample['text'] = self.x[idx]
        sample['label'] = torch.LongTensor([int(self.y[idx])])
        return sample

    def __iter__(self):
        return iter(range(self.__len__()))


#
#  Copyright (c) 2011-2014 Exxeleron GmbH
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

from qpython import MetaData
from qtype import *  # @UnusedWildImport

_MILIS_PER_DAY = 24 * 60 * 60 * 1000

_EPOCH_QMONTH = numpy.datetime64('2000-01', 'M')
_EPOCH_QDATE = numpy.datetime64('2000-01-01', 'D')
_EPOCH_QDATETIME = numpy.datetime64('2000-01-01T00:00:00.000', 'ms')
_EPOCH_TIMESTAMP = numpy.datetime64('2000-01-01T00:00:00', 'ns')

_QMONTH_NULL = qnull(QMONTH)
_QDATE_NULL = qnull(QDATE)
_QDATETIME_NULL = qnull(QDATETIME)
_QMINUTE_NULL = qnull(QMINUTE)
_QSECOND_NULL = qnull(QSECOND)
_QTIME_NULL = qnull(QTIME)
_QTIMESTAMP_NULL = qnull(QTIMESTAMP)
_QTIMESPAN_NULL = qnull(QTIMESPAN)

class QTemporal(object):
    '''Represents a q temporal value.'''

    def __init__(self, dt):
        self._datetime = dt

    def meta_init(self, **meta):
        self.meta = MetaData(**meta)

    @property
    def raw(self):
        return self._datetime

    def __str__(self):
        return '%s [%s]' % (self._datetime, self.meta)

    def __eq__(self, other):
        return (isinstance(other, self.__class__)
            and self._datetime == other._datetime)

    def __ne__(self, other):
        return not self.__eq__(other)

def qtemporal(dt, **meta):
    '''Converts a numpy.datetime64 to q temporal object and enriches object instance with given meta data.'''
    result = QTemporal(dt)
    result.meta_init(**meta)
    return result

def from_raw_qtemporal(raw, qtype):
    return qtemporal(_FROM_Q[qtype](raw), qtype = qtype)

def to_raw_qtemporal(dt, qtype):
    return _TO_Q[qtype](dt)

def _from_qmonth(raw):
    if raw == _QMONTH_NULL:
        return _QMONTH_NULL
    else:
        return _EPOCH_QMONTH + numpy.timedelta64(int(raw), 'M')

def _to_qmonth(dt):
    t_dt = type(dt)
    if t_dt == numpy.int32:
        return dt
    elif t_dt == numpy.datetime64:
        return (dt - _EPOCH_QMONTH).astype(int)
    else:
        raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))

def _from_qdate(raw):
    if raw == _QDATE_NULL:
        return _QDATE_NULL
    else:
        return _EPOCH_QDATE + numpy.timedelta64(int(raw), 'D')

def _to_qdate(dt):
    t_dt = type(dt)
    if t_dt == numpy.int32:
        return dt
    elif t_dt == numpy.datetime64:
        return (dt - _EPOCH_QDATE).astype(int)
    else:
        raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))

def _from_qdatetime(raw):
    if numpy.isnan(raw) or raw == _QDATETIME_NULL:
        return _QDATETIME_NULL
    else:
        return _EPOCH_QDATETIME + numpy.timedelta64(long(_MILIS_PER_DAY * raw), 'ms')

def _to_qdatetime(dt):
    t_dt = type(dt)
    if t_dt == numpy.float64:
        return dt
    elif t_dt == numpy.datetime64:
        return (dt - _EPOCH_QDATETIME).astype(float) / _MILIS_PER_DAY
    else:
        raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))

def _from_qminute(raw):
    if raw == _QMINUTE_NULL:
        return _QMINUTE_NULL
    else:
        return numpy.timedelta64(int(raw), 'm')

def _to_qminute(dt):
    t_dt = type(dt)
    if t_dt == numpy.int32:
        return dt
    elif t_dt == numpy.timedelta64:
        return dt.astype(int)
    else:
        raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))

def _from_qsecond(raw):
    if raw == _QSECOND_NULL:
        return _QSECOND_NULL
    else:
        return numpy.timedelta64(int(raw), 's')

def _to_qsecond(dt):
    t_dt = type(dt)
    if t_dt == numpy.int32:
        return dt
    elif t_dt == numpy.timedelta64:
        return dt.astype(int)
    else:
        raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))

def _from_qtime(raw):
    if raw == _QTIME_NULL:
        return _QTIME_NULL
    else:
        return numpy.timedelta64(int(raw), 'ms')

def _to_qtime(dt):
    t_dt = type(dt)
    if t_dt == numpy.int32:
        return dt
    elif t_dt == numpy.timedelta64:
        return dt.astype(int)
    else:
        raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))

def _from_qtimestamp(raw):
    if raw == _QTIMESTAMP_NULL:
        return _QTIMESTAMP_NULL
    else:
        return _EPOCH_TIMESTAMP + numpy.timedelta64(long(raw), 'ns')

def _to_qtimestamp(dt):
    t_dt = type(dt)
    if t_dt == numpy.int64:
        return dt
    elif t_dt == numpy.datetime64:
        return (dt - _EPOCH_TIMESTAMP).astype(long)
    else:
        raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))

def _from_qtimespan(raw):
    if raw == _QTIMESPAN_NULL:
        return _QTIMESPAN_NULL
    else:
        return numpy.timedelta64(long(raw), 'ns')

def _to_qtimespan(dt):
    t_dt = type(dt)
    if t_dt == numpy.int64:
        return dt
    elif t_dt == numpy.timedelta64:
        return dt.astype(long)
    else:
        raise ValueError('Cannot convert %s of type %s to q value.' % (dt, type(dt)))

_FROM_Q = {
           QMONTH:          _from_qmonth,
           QDATE:           _from_qdate,
           QDATETIME:       _from_qdatetime,
           QMINUTE:         _from_qminute,
           QSECOND:         _from_qsecond,
           QTIME:           _from_qtime,
           QTIMESTAMP:      _from_qtimestamp,
           QTIMESPAN:       _from_qtimespan,
           }

_TO_Q = {
         QMONTH:            _to_qmonth,
         QDATE:             _to_qdate,
         QDATETIME:         _to_qdatetime,
         QMINUTE:           _to_qminute,
         QSECOND:           _to_qsecond,
         QTIME:             _to_qtime,
         QTIMESTAMP:        _to_qtimestamp,
         QTIMESPAN:         _to_qtimespan,
         }