import copy
import math

import numpy as np
import matplotlib.pyplot as plt
import sklearn
from sklearn import metrics
from sklearn.metrics import recall_score,matthews_corrcoef
from sklearn.metrics.scorer import make_scorer
from sklearn.model_selection import cross_val_score,cross_validate
from sklearn.ensemble import RandomForestClassifier,AdaBoostClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import LogisticRegression
from sklearn import svm
try:
    from sklearn.model_selection import train_test_split
except ImportError:
    from sklearn.cross_validation import train_test_split

# MutationAssessor, PolyPhen 2, SIFT, FATHMM-W, CADD, Condel
score_index = [1,3,5,7,9,10]
# CADD only have score, does not have a prediction
pred_index = [2,4,6,8,-1,11]
file_names = ["humVar_scores.csv",
              "exovar_scores.csv",
              "predictSNPSelected_scores.csv"]
validate_data = ["variBenchSelected_scores.csv"]

def load_data(file_names):
    """ load data from csv files
    Return:
        Scores of tools: n*6 matrix (n: number of variants, 6: number of tools)
        Predictions of tools: n*6 matrix (n: number of variants, 6: number of tools)
        True labels: numpy array of length n (n: number of variants)
    """

    true_label = []
    data_matrix = []
    tool_label = []

    for file_name in file_names:
        file = open(file_name)
        for i, line in enumerate(file):
            # exclude the head line
            if i == 0:
                continue
            else:
                items = line.strip().split(",")
                true_label.append(int(items[0]))
                data_line = np.ones(len(score_index))
                pred_line = np.ones(len(pred_index))
                for j in range(len(score_index)):
                    # normalize the scores into an interval [0,1]
                    # SIFT
                    if score_index[j] == 5:
                        data_line[j] = 1-float(items[score_index[j]])
                    # Mutation Accessor
                    elif score_index[j] == 1:
                        ori_score = float(items[score_index[j]])
                        if ori_score <= 1.938:
                            data_line[j] = 0.5*(ori_score+5.545)/7.483
                        else:
                            data_line[j] = 0.5 +0.5 * (ori_score -1.938) / 3.999
                    # FATHMM
                    elif score_index[j] == 7:
                        ori_score = float(items[score_index[j]])
                        if ori_score >-1.5:
                            data_line[j] = 0.5 * (10.64-ori_score) / 12.14
                        else:
                            data_line[j] = 0.5 - 0.5 * (ori_score +1.5) / 14.63
                    # CADD
                    elif score_index[j] == 9:
                        ori_score = float(items[score_index[j]])
                        if ori_score > 15:
                            pred_line[j] = 1
                        else:
                            pred_line[j] = -1
                        data_line[j] = ori_score/40
                    # PolyPhen 2 or Condel
                    else:
                        data_line[j] = float(items[score_index[j]])
                    # data_line[j] = float(items[score_index[j]])
                    if score_index[j] != 20:
                        pred_line[j] = float(items[pred_index[j]])
                data_matrix.append(data_line)
                tool_label.append(pred_line)
        file.close()
    return np.array(data_matrix),np.array(tool_label),np.array(true_label)

def plot_ROC_curve(y_true, y_score):
    """ get the x-axis and y-axis for plotting the ROC curve by varying a threshold on y_score.
    Assume that the positive class is labeled truly as 1, and that if given a threshold value,
    we can classify the examples with: (y_score > threshold).astype(int)
    Return:
        The area under your curve (AUROC)
        X-axis data and Y-axis date used to plot ROC curve
    """

    thresholds = np.unique(y_score)
    x_axis = []
    y_axis = []
    f_score = []
    for i in range(0,len(thresholds)):
        y_prediction = (y_score > thresholds[i]).astype(int)

        TP = np.sum((y_true == y_prediction) & (y_true == 1))
        FN = np.sum((y_true != y_prediction) & (y_true == 1))
        FP = np.sum((y_true != y_prediction) & (y_true == 0))
        TN = len(y_true)-TP-FN-FP

        true_positive = TP/(TP+FN)
        # precision = TP/(TP+FP)
        false_positive = FP/(TN+FP)

        x_axis.append(false_positive)
        y_axis.append(true_positive)
        # use this expression instead of recall and precision
        # to avoid the denominator of precision to be 0
        f_score.append(2*TP/(2*TP+FP+FN))

    fpr, tpr, _ = metrics.roc_curve(y_true, y_score)
    auc = sklearn.metrics.auc(fpr, tpr)

    return auc,x_axis,y_axis

def get_metrics(y_true, y_prediction):
    """ print out and return the metrics for evaluating the performance of tools
    Return:
        Accuracy
        Precision
        Recall
        Specificity
        F-score
        Negative predictive value
        Matthews Correlation Coefficient
        The area under ROC curve
    """

    TP = np.sum((y_true == y_prediction) & (y_true == 1))
    FN = np.sum((y_true != y_prediction) & (y_true == 1))
    FP = np.sum((y_true != y_prediction) & (y_true == -1))
    TN = len(y_true) - TP - FN - FP
    print(len(y_true),TP,FN,FP,TN)

    accuracy = (TP+TN)/(TP+FP+TN+FN)
    precision = TP / (TP + FP)
    # recall/sensitivity/true positive
    recall = TP/(TP+FN)
    # specificity/false positive
    specificity = TN / (TN + FP)
    f_score = 2 * TP / (2 * TP + FP + FN)
    # negative predictive value
    NPV = TN/(TN+FN)
    mcc = (TP*TN-FP*FN)/(math.sqrt(TP+FP)*math.sqrt(TP+FN)*math.sqrt(TN+FP)*math.sqrt(TN+FN))

    fpr, tpr, _ = metrics.roc_curve(y_true, y_prediction)
    auc = sklearn.metrics.auc(fpr, tpr)

    print("accuracy:",accuracy,",precision:",precision,",recall:",recall,",specificity:",specificity)
    print("f score:",f_score,",NPV:",NPV,",MCC:",mcc,",AUC",auc)

    return accuracy,precision,recall,specificity,f_score,NPV,mcc,auc

def print_metrics(accuracy,precision,recall,f_score,mcc,auc):
    print("accuracy:%0.2f" % accuracy, ",precision:%0.2f" % precision,
          ",recall:%0.2f" % recall, ",f-score:%0.2f" % f_score,
          ",mcc:%0.2f" % mcc, ",auc:%0.2f" % auc)

if __name__ == '__main__':
    data, tool_label, true_label = load_data(file_names)
    x_list = []
    y_list = []
    auc_list = []
    legend = ['Mutation Assessor', 'PolyPhen 2', 'SIFT', 'FATHMM-W', 'CADD', 'Condel'
        , 'Logistic Regression', 'SVM', 'Random Forest', 'Adaboost']
    for i in range(len(tool_label[0])):
        print("----"+legend[i]+"-----")
        y_score = data[np.logical_not(np.isnan(data[:, i])),i]
        y_true = true_label[np.logical_not(np.isnan(data[:, i]))]
        y_prediction = tool_label[np.logical_not(np.isnan(data[:, i])),i]
        y_bin = copy.copy(y_true)
        y_bin[y_bin == -1] = 0
        auc,x_axis,y_axis=plot_ROC_curve(y_bin, y_score)
        get_metrics(y_true,y_prediction)
        x_list.append(x_axis)
        y_list.append(y_axis)
        auc_list.append(auc)
    full_data = data[np.sum(np.isnan(data[:, :6]), 1) == 0, :4]
    full_label = true_label[np.sum(np.isnan(data[:, :6]), 1) == 0]

    y_bin = copy.copy(full_label)
    y_bin[y_bin==-1]=0

    # plot ROC curves
    plt.figure()
    for i in range(len(x_list)):
        plt.plot(x_list[i], y_list[i], '-')
        legend[i] += ", auc={0:.3f}".format(auc_list[i])
    plt.xlabel('False positive rate')
    plt.ylabel('True positive rate')
    plt.title('ROC curve')
    plt.legend(legend, loc='lower right')

    plt.plot(x_list[0],x_list[0],'grey',ls='--')
    plt.show()

    '''
    cross-validation
    '''
    scoring = {'accuracy': 'accuracy',
               'prec': 'precision',
               'recall': 'recall',
               'f': 'f1',
               'mcc': make_scorer(matthews_corrcoef),
               'auc:':make_scorer(sklearn.metrics.auc)}
    full_label[full_label == -1] = 0

    print("----Logistic regression-----")
    logistic = LogisticRegression()
    scores = cross_validate(logistic, full_data, full_label, scoring=scoring,
                            cv = 10, return_train_score = False)
    print_metrics(scores['test_accuracy'].mean(),scores['test_prec'].mean(),scores['test_recall'].mean(),
                  scores['test_f'].mean(),scores['test_mcc'].mean(),scores['test_auc'].mean())
    log_scores = cross_val_score(logistic, full_data, full_label, cv=10)
    auc, x_axis, y_axis = plot_ROC_curve(y_bin, log_scores)
    x_list.append(x_axis)
    y_list.append(y_axis)
    auc_list.append(auc)

    print("----svm-----")
    clf = svm.SVC()
    scores = cross_validate(clf, full_data, full_label, scoring=scoring,
                            cv = 10, return_train_score = False)
    print_metrics(scores['test_accuracy'].mean(), scores['test_prec'].mean(), scores['test_recall'].mean(),
                  scores['test_f'].mean(), scores['test_mcc'].mean(), scores['test_auc'].mean())
    svm_scores = cross_val_score(clf, full_data, full_label, cv=10)
    auc, x_axis, y_axis = plot_ROC_curve(y_bin, svm_scores)
    x_list.append(x_axis)
    y_list.append(y_axis)
    auc_list.append(auc)

    print("----Random Forest-----")
    rf = RandomForestClassifier(max_depth=5, n_estimators=1000, max_features=1)
    scores = cross_validate(rf, full_data, full_label, scoring=scoring,
                            cv = 10, return_train_score = False)
    print_metrics(scores['test_accuracy'].mean(), scores['test_prec'].mean(), scores['test_recall'].mean(),
                  scores['test_f'].mean(), scores['test_mcc'].mean(), scores['test_auc'].mean())
    rf_scores = cross_val_score(rf, full_data, full_label, cv=10)
    auc, x_axis, y_axis = plot_ROC_curve(y_bin, rf_scores)
    x_list.append(x_axis)
    y_list.append(y_axis)
    auc_list.append(auc)

    print("----Adaboost-----")
    bdt = AdaBoostClassifier(DecisionTreeClassifier(max_depth=1),
                              algorithm="SAMME.R",
                              n_estimators=1000,
                              learning_rate=0.07)
    scores = cross_validate(bdt, full_data, full_label, scoring=scoring,
                            cv = 10, return_train_score = False)
    print_metrics(scores['test_accuracy'].mean(), scores['test_prec'].mean(), scores['test_recall'].mean(),
                  scores['test_f'].mean(), scores['test_mcc'].mean(), scores['test_auc'].mean())
    bdt_scores = cross_val_score(bdt, full_data, full_label, cv=10)
    auc, x_axis, y_axis = plot_ROC_curve(y_bin, bdt_scores)
    x_list.append(x_axis)
    y_list.append(y_axis)
    auc_list.append(auc)

    # plot ROC curve
    plt.figure()
    for i in range(len(x_list)):
        plt.plot(x_list[i], y_list[i], '-')
        legend[i] += ", auc={0:.3f}".format(auc_list[i])
    plt.xlabel('False positive rate')
    plt.ylabel('True positive rate')
    plt.title('ROC curve')
    plt.legend(legend, loc='lower right')

    plt.plot(x_list[0],x_list[0],'grey',ls='--')
    plt.show()

    '''
        external validation
    '''
    data, _, true_label = load_data(validate_data)
    test_data = data[np.sum(np.isnan(data[:, :6]), 1) == 0, :4]
    test_label = true_label[np.sum(np.isnan(data[:, :6]), 1) == 0]
    test_bin = copy.copy(test_label)
    test_bin[test_bin==-1]=0

    print("----Logistic regression-----")
    logistic = LogisticRegression()
    logistic.fit(full_data, full_label)
    get_metrics(test_label, logistic.predict(test_data))
    log_score = logistic.predict_proba(test_data)[:, 1]
    auc, x_axis, y_axis = plot_ROC_curve(test_bin, log_scores)
    x_list[6]=x_axis
    y_list[6]=y_axis
    auc_list[6]=auc

    print("----svm-----")
    clf = svm.SVC()
    clf.fit(full_data, full_label)
    get_metrics(test_label, clf.predict(test_data))
    svm_score = clf.decision_function(test_data)
    auc, x_axis, y_axis = plot_ROC_curve(test_bin, svm_scores)
    x_list[7] = x_axis
    y_list[7] = y_axis
    auc_list[7] = auc

    print("----Random Forest-----")
    rf = RandomForestClassifier(max_depth=5, n_estimators=1000, max_features=1)
    rf.fit(full_data, full_label)
    get_metrics(test_label, rf.predict(test_data))
    rf_score = rf.predict_proba(test_data)[:, 1]
    auc, x_axis, y_axis = plot_ROC_curve(test_bin, rf_scores)
    x_list[8] = x_axis
    y_list[8] = y_axis
    auc_list[8] = auc

    print("----Adaboost-----")
    bdt = AdaBoostClassifier(DecisionTreeClassifier(max_depth=1),
                             algorithm="SAMME.R",
                             n_estimators=1000,
                             learning_rate=0.08)
    bdt.fit(full_data, full_label)
    get_metrics(test_label, bdt.predict(test_data))
    ada_score = bdt.predict_proba(test_data)[:, 1]
    auc, x_axis, y_axis = plot_ROC_curve(test_bin, bdt_scores)
    x_list[9] = x_axis
    y_list[9] = y_axis
    auc_list[9] = auc

    # plot ROC curve
    plt.figure()
    for i in range(len(x_list)):
        plt.plot(x_list[i], y_list[i], '-')
        legend[i] += ", auc={0:.3f}".format(auc_list[i])
    plt.xlabel('False positive rate')
    plt.ylabel('True positive rate')
    plt.title('ROC curve')
    plt.legend(legend, loc='lower right')

    plt.plot(x_list[0],x_list[0],'grey',ls='--')
    plt.show()


# -*- coding: utf-8 -*-

"""CTPN implementation based on PyTorch
"""

import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F

class VGG_16(nn.Module):
    """VGG16 without fully connected (FC) layers.

    Low-level feature extraction.
    """
    def __init__(self):
        super(VGG_16, self).__init__()
        # conv1
        self.convolution1_1 = nn.Conv2d(3, 64, 3, padding=1)
        self.convolution1_2 = nn.Conv2d(64, 64, 3, padding=1)
        # max pooling
        self.pooling1 = nn.MaxPool2d(2, stride=2)
        # conv2
        self.convolution2_1 = nn.Conv2d(64, 128, 3, padding=1)
        self.convolution2_2 = nn.Conv2d(128, 128, 3, padding=1)
        # max pooling
        self.pooling2 = nn.MaxPool2d(2, stride=2)
        # conv3
        self.convolution3_1 = nn.Conv2d(128, 256, 3, padding=1)
        self.convolution3_2 = nn.Conv2d(256, 256, 3, padding=1)
        self.convolution3_3 = nn.Conv2d(256, 256, 3, padding=1)
        # max pooling
        self.pooling3 = nn.MaxPool2d(2, stride=2)
        # conv4
        self.convolution4_1 = nn.Conv2d(256, 512, 3, padding=1)
        self.convolution4_2 = nn.Conv2d(512, 512, 3, padding=1)
        self.convolution4_3 = nn.Conv2d(512, 512, 3, padding=1)
        # max pooling
        self.pooling4 = nn.MaxPool2d(2, stride=2)
        # conv5
        self.convolution5_1 = nn.Conv2d(512, 512, 3, padding=1)
        self.convolution5_2 = nn.Conv2d(512, 512, 3, padding=1)
        self.convolution5_3 = nn.Conv2d(512, 512, 3, padding=1)

    def forward(self, x):
        x = F.relu(self.convolution1_1(x), inplace=True)
        x = F.relu(self.convolution1_2(x), inplace=True)
        x = self.pooling1(x)
        x = F.relu(self.convolution2_1(x), inplace=True)
        x = F.relu(self.convolution2_2(x), inplace=True)
        x = self.pooling2(x)
        x = F.relu(self.convolution3_1(x), inplace=True)
        x = F.relu(self.convolution3_2(x), inplace=True)
        x = F.relu(self.convolution3_3(x), inplace=True)
        x = self.pooling3(x)
        x = F.relu(self.convolution4_1(x), inplace=True)
        x = F.relu(self.convolution4_2(x), inplace=True)
        x = F.relu(self.convolution4_3(x), inplace=True)
        x = self.pooling4(x)
        x = F.relu(self.convolution5_1(x), inplace=True)
        x = F.relu(self.convolution5_2(x), inplace=True)
        x = F.relu(self.convolution5_3(x), inplace=True)
        return x

class Im2col(nn.Module):
    """Convert the feature map of the VGG16 output to a vector.

    Convert convolution operation to matrix multiplication.
    """
    def __init__(self, kernel_size, stride, padding):
        super(Im2col, self).__init__()
        self.kernel_size = kernel_size
        self.stride = stride
        self.padding = padding

    def forward(self, x):
        height = x.shape[2]
        x = F.unfold(x, self.kernel_size, padding=self.padding, stride=self.stride)
        x = x.reshape((x.shape[0], x.shape[1], height, -1))
        return x

class BLSTM(nn.Module):
    """Bi-directional Long Short-Term Memory Network
    """
    def __init__(self, channel, hidden_unit, bidirectional=True):
        super(BLSTM, self).__init__()
        self.lstm = nn.LSTM(channel, hidden_unit, bidirectional=bidirectional)

    def forward(self, x):
        x = x.transpose(1, 3)   # the batch size of x must be 1
        recurrent, _ = self.lstm(x[0])
        recurrent = recurrent[np.newaxis, :, :, :]
        recurrent = recurrent.transpose(1, 3)
        return recurrent

class CTPN(nn.Module):
    """Connectionist Text Proposal Network

    Connect VGG16 with BLSTM.

    Returns:
        pred_y
        score
        side_refinement
    """
    def __init__(self):
        super(CTPN, self).__init__()
        # VGG16 CNN
        self.cnn = nn.Sequential()
        self.cnn.add_module('VGG_16', VGG_16())
        # BLSTM RNN
        self.rnn = nn.Sequential()
        self.rnn.add_module('Im2col', Im2col((3, 3), (1, 1), (1, 1)))
        self.rnn.add_module('blstm', BLSTM(3 * 3 * 512, 128))
        # 全连接层
        self.FC = nn.Conv2d(256, 512, 1)
        # 输出2*10个参数, 2个参数分别表示anchor的height和center_y, 10表示anchor的尺寸个数
        self.vertical_coordinate = nn.Conv2d(512, 2 * 10, 1)
        # 输出2*10个分数, 2表示有无字符, 10表示anchor的尺寸个数
        self.score = nn.Conv2d(512, 2 * 10, 1)
        # 输出10个参数, 表示anchor的水平偏移, 用于调整文本框水平边缘精度, 10表示anchor的尺寸个数
        self.side_refinement = nn.Conv2d(512, 10, 1)

    def forward(self, x, val=False):
        x = self.cnn(x)
        x = self.rnn(x)
        x = self.FC(x)
        x = F.relu(x, inplace=True)
        pred_y = self.vertical_coordinate(x)
        score = self.score(x)
        if val:
            score = score.reshape((score.shape[0], 10, 2, score.shape[2], score.shape[3]))
            score = score.squeeze(0)
            score = score.transpose(1, 2)
            score = score.transpose(2, 3)
            score = score.reshape((-1, 2))
            #score = F.softmax(score, dim=1)
            score = score.reshape((10, pred_y.shape[2], -1, 2))
            pred_y = pred_y.reshape((pred_y.shape[0], 10, 2, pred_y.shape[2],
                                                   pred_y.shape[3]))
            pred_y = pred_y.squeeze(0)
        side_refinement = self.side_refinement(x)
        return pred_y, score, side_refinement


import time
import movie
import re
import json
# import csv

def get_one_page(url):
    try:
        headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.162 Safari/537.36'
        }
        response = movie.get(url, headers=headers)
        print(response.status_code)
        if response.status_code == 200:
            return response.text
        return None
    except :
        return '访问错误'

def parse_one_page(html):
    item = re.findall('<a href="(.*?)" title=', html)
    print(item)
    return item

# def write_to_file(data):
#     with open('片库电影', 'w', encoding='utf-8') as csvfile:
#         csvfile.writer = csv.writer((data, csvfile) + '\n')
def write_to_file(data):
    with open('result1.txt', 'a', encoding='utf-8') as f:
        f.write(json.dumps(data, ensure_ascii=False) + '\n')

def main(package):
    url = 'https://www.pianku.tv/mv/' +str('------')+ str(package) + str('.html')
    print(url)
    html = get_one_page(url)
    print(html)
    parse_one_page(html)
    print(item)
    write_to_file(item)

if __name__ == '__main__':
    for i in range(1, 2):
        main(str(i))
        # time.sleep(1)


# -*- coding: utf-8 -*-

"""
Import sample data for classification engine
"""

import predictionio
import argparse
import os
import sys

def import_events(client, file):
  with open(file, 'r') as f:
    count = 0
    print("Importing data...")
    for line in f:
      data = line.rstrip('\r\n').split(",")
      if len(data) == 5:
        client.create_event(
          event="$set",
          entity_type="user",
          entity_id=str(count), # use the count num as user ID
          properties= {
            "attr0" : float(data[0]),
            "attr1" : float(data[1]),
            "attr2" : float(data[2]),
            "attr3" : float(data[3]),
            "target" : data[4]
          }
        )
        count += 1
  print("%s events are imported." % count)

def download_datafile(url, path):
  if sys.version_info[0] >= 3:
    from urllib.request import urlretrieve
  else:
    from urllib import urlretrieve
  urlretrieve(url, path)

if __name__ == '__main__':
  parser = argparse.ArgumentParser(
    description="Import sample data")
  parser.add_argument('--access-key', dest='access_key', default='IRIS_TOKEN')
  parser.add_argument('--url', default="http://localhost:7070")
  parser.add_argument('--file', dest='file_path', default="./data/iris.data")
  parser.add_argument('--download-url', dest='download_url',
    default="https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data")

  args = parser.parse_args()
  print(args)

  if not os.path.exists(args.file_path):
    download_datafile(args.download_url, args.file_path)

  client = predictionio.EventClient(
    access_key=args.access_key,
    url=args.url,
    threads=5,
    qsize=500)
  import_events(client, args.file_path)


from django.http import HttpResponseRedirect
from django.contrib.auth.models import User
from rest_framework import permissions, status
from rest_framework.decorators import api_view
from rest_framework.response import Response
from rest_framework.views import APIView
from .serializers import UserSerializer, UserSerializerWithToken, \
    SittingSerializer
from .models import Sitting
from quiz.models import Quiz

@api_view(['GET'])
def current_user(request):
    """
    Determine the current user by their token, and return their data
    """

    serializer = UserSerializer(request.user)
    return Response(serializer.data)

class UserList(APIView):
    """
    Handle Access to the User Database
    """

    permission_classes = (permissions.AllowAny,)

    def post(self, request, format=None):
        """
        Add a new user to our User Table
        """
        serializer = UserSerializerWithToken(data=request.data)
        if serializer.is_valid():
            serializer.save()
            return Response(serializer.data, status=status.HTTP_201_CREATED)
        return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

class SittingAPI(APIView):
    """
    Handle Access to the Quiz sitting
    """

    permission_classes = (permissions.IsAuthenticated,)

    def get(self, request, pk=None):
        """
        Get the quiz sitting for the current user. If it does not exist,
        then create it.
        """
        if pk is None:
            return Response(serializer.errors,
                            status=status.HTTP_400_BAD_REQUEST)
        sit, created = Sitting.objects.get_or_create(user=request.user,
                                                     quiz_id=pk)
        serializer = SittingSerializer(sit)
        return Response(serializer.data)

    def post(self, request, pk=None):
        """
        Post answers in response to a quiz, where pk is the quiz id
        which the user is answering
        """
        if pk is None:
            return Response({'response': 'Invalid Quiz ID'},
                            status=status.HTTP_400_BAD_REQUEST)

        if "user_choices" not in request.data:
                return Response({'response': 'Missing User Choices'},
                                status=status.HTTP_400_BAD_REQUEST)

        data = {
            "user_choices": request.data["user_choices"],
            "quiz": pk
        }

        try:
            sitting = Sitting.objects.get(user=request.user, quiz=data["quiz"])
        except ObjectDoesNotExist:
            return Response({'response': 'Invalid Quiz or User'},
                            status=status.HTTP_400_BAD_REQUEST)

        serializer = SittingSerializer(sitting, data=data, partial=True)
        if not serializer.is_valid():
            return Response(serializer.errors,
                            status=status.HTTP_400_BAD_REQUEST)

        serializer.instance.calculate_score()
        serializer.save()
        return Response({"success": True, "response": serializer.data})


########################################################################################################################
#                                                          IMPORT                                                      #
########################################################################################################################
import torch
import sys
import os
import json
import time
import numpy as np
import argparse

from torch.utils.data import DataLoader
from torch.utils.data import WeightedRandomSampler
from umap.umap_ import find_ab_params

from singleVis.custom_weighted_random_sampler import CustomWeightedRandomSampler
from singleVis.SingleVisualizationModel import VisModel
from singleVis.losses import UmapLoss, ReconstructionLoss, TemporalLoss, DVILoss, SingleVisLoss, DummyTemporalLoss
from singleVis.edge_dataset import DVIDataHandler
from singleVis.trainer import DVITrainer
from singleVis.data import NormalDataProvider
from singleVis.spatial_edge_constructor import SingleEpochSpatialEdgeConstructor
from singleVis.projector import DVIProjector
from singleVis.eval.evaluator import Evaluator
from singleVis.utils import find_neighbor_preserving_rate
########################################################################################################################
#                                                     DVI PARAMETERS                                                   #
########################################################################################################################
"""This serve as an example of DeepVisualInsight implementation in pytorch."""
VIS_METHOD = "DVI" # DeepVisualInsight

########################################################################################################################
#                                                     LOAD PARAMETERS                                                  #
########################################################################################################################
parser = argparse.ArgumentParser(description='Process hyperparameters...')
parser.add_argument('--content_path', type=str)
args = parser.parse_args()

CONTENT_PATH = args.content_path
sys.path.append(CONTENT_PATH)
with open(os.path.join(CONTENT_PATH, "config.json"), "r") as f:
    config = json.load(f)
config = config[VIS_METHOD]

# record output information
# now = time.strftime("%Y-%m-%d-%H_%M_%S", time.localtime(time.time()))
# sys.stdout = open(os.path.join(CONTENT_PATH, now+".txt"), "w")

SETTING = config["SETTING"]
CLASSES = config["CLASSES"]
DATASET = config["DATASET"]
PREPROCESS = config["VISUALIZATION"]["PREPROCESS"]
GPU_ID = config["GPU"]
EPOCH_START = config["EPOCH_START"]
EPOCH_END = config["EPOCH_END"]
EPOCH_PERIOD = config["EPOCH_PERIOD"]

# Training parameter (subject model)
TRAINING_PARAMETER = config["TRAINING"]
NET = TRAINING_PARAMETER["NET"]
LEN = TRAINING_PARAMETER["train_num"]

# Training parameter (visualization model)
VISUALIZATION_PARAMETER = config["VISUALIZATION"]
LAMBDA1 = VISUALIZATION_PARAMETER["LAMBDA1"]
LAMBDA2 = VISUALIZATION_PARAMETER["LAMBDA2"]
B_N_EPOCHS = VISUALIZATION_PARAMETER["BOUNDARY"]["B_N_EPOCHS"]
L_BOUND = VISUALIZATION_PARAMETER["BOUNDARY"]["L_BOUND"]
ENCODER_DIMS = VISUALIZATION_PARAMETER["ENCODER_DIMS"]
DECODER_DIMS = VISUALIZATION_PARAMETER["DECODER_DIMS"]
S_N_EPOCHS = VISUALIZATION_PARAMETER["S_N_EPOCHS"]
N_NEIGHBORS = VISUALIZATION_PARAMETER["N_NEIGHBORS"]
PATIENT = VISUALIZATION_PARAMETER["PATIENT"]
MAX_EPOCH = VISUALIZATION_PARAMETER["MAX_EPOCH"]

VIS_MODEL_NAME = VISUALIZATION_PARAMETER["VIS_MODEL_NAME"]
EVALUATION_NAME = VISUALIZATION_PARAMETER["EVALUATION_NAME"]

# Define hyperparameters
DEVICE = torch.device("cuda:{}".format(GPU_ID) if torch.cuda.is_available() else "cpu")

import Model.model as subject_model
net = eval("subject_model.{}()".format(NET))

########################################################################################################################
#                                                    TRAINING SETTING                                                  #
########################################################################################################################
# Define data_provider
data_provider = NormalDataProvider(CONTENT_PATH, net, EPOCH_START, EPOCH_END, EPOCH_PERIOD, device=DEVICE, classes=CLASSES,verbose=1)
if PREPROCESS:
    data_provider._meta_data()
    if B_N_EPOCHS >0:
        data_provider._estimate_boundary(LEN//10, l_bound=L_BOUND)

# Define visualization models
model = VisModel(ENCODER_DIMS, DECODER_DIMS)

# Define Losses
negative_sample_rate = 5
min_dist = .1
_a, _b = find_ab_params(1.0, min_dist)
umap_loss_fn = UmapLoss(negative_sample_rate, DEVICE, _a, _b, repulsion_strength=1.0)
recon_loss_fn = ReconstructionLoss(beta=1.0)
single_loss_fn = SingleVisLoss(umap_loss_fn, recon_loss_fn, lambd=LAMBDA1)
# Define Projector
projector = DVIProjector(vis_model=model, content_path=CONTENT_PATH, vis_model_name=VIS_MODEL_NAME, device=DEVICE)

start_flag = 1
prev_model = VisModel(ENCODER_DIMS, DECODER_DIMS)

for iteration in range(EPOCH_START, EPOCH_END+EPOCH_PERIOD, EPOCH_PERIOD):
    # Define DVI Loss
    if start_flag:
        temporal_loss_fn = DummyTemporalLoss(DEVICE)
        criterion = DVILoss(umap_loss_fn, recon_loss_fn, temporal_loss_fn, lambd1=LAMBDA1, lambd2=0.0)
        start_flag = 0
    else:
        # TODO AL mode, redefine train_representation
        prev_data = data_provider.train_representation(iteration-EPOCH_PERIOD)
        curr_data = data_provider.train_representation(iteration)
        npr = find_neighbor_preserving_rate(prev_data, curr_data, N_NEIGHBORS)
        temporal_loss_fn = TemporalLoss(w_prev, DEVICE)
        criterion = DVILoss(umap_loss_fn, recon_loss_fn, temporal_loss_fn, lambd1=LAMBDA1, lambd2=LAMBDA2*npr)
    # Define training parameters
    optimizer = torch.optim.Adam(model.parameters(), lr=.01, weight_decay=1e-5)
    lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=4, gamma=.1)
    # Define Edge dataset
    t0 = time.time()
    spatial_cons = SingleEpochSpatialEdgeConstructor(data_provider, iteration, S_N_EPOCHS, B_N_EPOCHS, N_NEIGHBORS)
    edge_to, edge_from, probs, feature_vectors, attention = spatial_cons.construct()
    t1 = time.time()

    probs = probs / (probs.max()+1e-3)
    eliminate_zeros = probs>5e-2    #1e-3
    edge_to = edge_to[eliminate_zeros]
    edge_from = edge_from[eliminate_zeros]
    probs = probs[eliminate_zeros]

    dataset = DVIDataHandler(edge_to, edge_from, feature_vectors, attention)

    n_samples = int(np.sum(S_N_EPOCHS * probs) // 1)
    # chose sampler based on the number of dataset
    if len(edge_to) > pow(2,24):
        sampler = CustomWeightedRandomSampler(probs, n_samples, replacement=True)
    else:
        sampler = WeightedRandomSampler(probs, n_samples, replacement=True)
    edge_loader = DataLoader(dataset, batch_size=2000, sampler=sampler, num_workers=8, prefetch_factor=10)

    ########################################################################################################################
    #                                                       TRAIN                                                          #
    ########################################################################################################################

    trainer = DVITrainer(model, criterion, optimizer, lr_scheduler, edge_loader=edge_loader, DEVICE=DEVICE)

    t2=time.time()
    trainer.train(PATIENT, MAX_EPOCH)
    t3 = time.time()

    # save result
    save_dir = data_provider.model_path
    trainer.record_time(save_dir, "time_{}".format(VIS_MODEL_NAME), "complex_construction", str(iteration), t1-t0)
    trainer.record_time(save_dir, "time_{}".format(VIS_MODEL_NAME), "training", str(iteration), t3-t2)
    save_dir = os.path.join(data_provider.model_path, "Epoch_{}".format(iteration))
    trainer.save(save_dir=save_dir, file_name="{}".format(VIS_MODEL_NAME))

    print("Finish epoch {}...".format(iteration))

    prev_model.load_state_dict(model.state_dict())
    for param in prev_model.parameters():
        param.requires_grad = False
    w_prev = dict(prev_model.named_parameters())

########################################################################################################################
#                                                      VISUALIZATION                                                   #
########################################################################################################################

from singleVis.visualizer import visualizer

vis = visualizer(data_provider, projector, 200, "tab10")
save_dir = os.path.join(data_provider.content_path, "img")
if not os.path.exists(save_dir):
    os.mkdir(save_dir)
for i in range(EPOCH_START, EPOCH_END+1, EPOCH_PERIOD):
    vis.savefig(i, path=os.path.join(save_dir, "{}_{}_{}.png".format(DATASET, i, VIS_METHOD)))

########################################################################################################################
#                                                       EVALUATION                                                     #
########################################################################################################################
eval_epochs = range(EPOCH_START, EPOCH_END+1, EPOCH_PERIOD)
EVAL_EPOCH_DICT = {
    "mnist":[1,10,15],
    "fmnist":[1,25,50],
    "cifar10":[1,100,199]
}
eval_epochs = EVAL_EPOCH_DICT[DATASET]
evaluator = Evaluator(data_provider, projector)

for eval_epoch in eval_epochs:
    evaluator.save_epoch_eval(eval_epoch, 15, temporal_k=5, file_name="{}".format(EVALUATION_NAME))


import sys
import subprocess
import glob
import threading
import os
import shutil
import numpy as np
import logging
import queue
from collections import namedtuple

import ngspice_meas as ngmeas

'''
Current implementation execute one call to ngspice per simulation.
As SKY130 models are very large, there is quite a simulation overhead.

as future work to explore looping solutions using the control.
Must confirm the alter works ok with bins and no reset is needed.

------------------ --------------------------
.control
.include 'pipe.dat'

*** loop over the elements to be simulated
let row = 0
while row < NROWS
**** alters ****
   alter r1 r = pipe_dat[row][0]

**** analysis ****
   op
   print v(2)
   let r_value = @r1[r]
   print r_value

****
   let row = row + 1
end
quit
.endc

--------------------- pipe.dat ----------------------------
let NROWS=3
let pipe_dat=vector(6)
compose pipe_dat values
+ 4e3 2
+ 5e3 3
+ 6e3 4
reshape pipe_dat [3][2]

'''

# global definitions for filenames
MEAS_OUT = '_meas.txt'
VARS     = 'design_var.inc'
GLOBAL_PARAMS = 'global_params.inc'

Simulation = namedtuple("Simulation", ["corner_id", "netlist_file", "meas_file", "sim_results", "values" ])

# Runs each netlist in interactive mode and saves the standard output to a file
def run_simulation(cwd, netlist_file, meas_file, timeout=2):
    '''
      runs ngspice  in interactive mode and saves the
      standard output to a file to collect measures.
      Gives a timeout [in minutes].
      param: netlist: the testbench to be simulated
      param: corner: current working directory (defautl=".")
      param: timeout: the timeour in minutes (defautl=2)
      '''
    try:
        #with open(os.path.join(cwd, meas_file), 'w') as f:
            subprocess.run(['ngspice', "-b", "-o", meas_file, netlist_file ], cwd=cwd, stdout= subprocess.DEVNULL, timeout=timeout*60)
    except (subprocess.TimeoutExpired,BlockingIOError) as e:
        logging.warning(e)

def parse_sim_data(sim_results, line, vector_id, vector_idx, vector_size, parse_vector):
    '''
    parses the values from the outupt file

    updates vector size and parse_vector
    '''
    content = line.split()
    if len(content) >= 3 and content[1] == '=':
        sim_results[content[0].upper()] = float(content[2])
    elif content[0] == 'Index':
        parse_vector = True
    elif parse_vector:
        try:
            sim_results[vector_id][vector_idx] = [ float(x) for x in content[1:]]
            vector_idx = vector_idx + 1
            if vector_idx == vector_size: parse_vector = False
        except Exception as e:
            logging.warning(e + "was raised when parsing " + line)

    return vector_idx, vector_size, parse_vector

def parse_sim_results(cwd, meas_file):
    sim_results = {}
    vector_id = None
    parse_vector = False
    vector_idx = 0
    vector_size = None
    try:
        with open(os.path.join(cwd, meas_file), 'r') as file:
            for line in file:
                line = line.strip()
                if (not line) or (line[0] == '*' ) or (line[0] == '-'): continue

                try:
                    if "AC output" in line:
                        vector_id = "AC"
                        vector_size =  int(line.split()[-2])
                        vector_row_size =  int(line.split()[-1])
                        vector_idx = 0
                        sim_results[vector_id] = np.full((vector_size, vector_row_size), np.NaN)
                    elif "NOISE output" in line:
                        vector_id = "NOISE"
                        vector_size =  int(line.split()[-2])
                        vector_row_size =  int(line.split()[-1])
                        vector_idx = 0
                        sim_results[vector_id] = np.full((vector_size, vector_row_size), np.NaN)
                    else:
                        if "=" in line:
                            content = line.split()
                            if len(content) >= 3 and content[1] == '=': sim_results[content[0].upper()] = float(content[2])
                        elif 'Index' in line:
                            parse_vector = True
                        elif parse_vector:
                            content = line.split()
                            try:
                                sim_results[vector_id][vector_idx, :] = content[1:]
                                vector_idx = vector_idx + 1
                                if vector_idx == vector_size: parse_vector = False
                            except Exception as e:
                                logging.warning(str(e) + " was raised when parsing " + line)
                except ValueError as e:
                    logging.warning(str(e) + " when parsing " + line )

    except FileNotFoundError:
        logging.warning(os.path.join(cwd, meas_file) + " file not found!")

    return sim_results

def worker(q, cwd, param):
    '''
    worker thread function that just runs the simulatiosn and collect the sim_results
    '''
    prev_values = None
    while True:
        simulation = q.get()
        if simulation is None: break
        #updated design_var if needed
        if simulation.values is not prev_values:
            write_design_variables(cwd, param, simulation.values)
            prev_values = simulation.values
        #remove meas file

        run_simulation(cwd, simulation.netlist_file, simulation.meas_file)
        curr_meas = parse_sim_results(cwd, simulation.meas_file)
        with simulation.sim_results as meas:
            if simulation.corner_id in meas: meas[simulation.corner_id].update(curr_meas)
            else: meas[simulation.corner_id] = curr_meas
        q.task_done()

def load_corners(corners):
    corners_dic = {}
    corner_name = "UNDEF"
    try:
        with open(corners, 'r') as file:
            for line in file:
                line = line.strip()
                if (not line) or (line[0] == '*'): continue
                if '.ALTER' in line.upper():
                    corner_name = line.split()[1]
                    corners_dic[corner_name] = list()
                elif line[0] == '+':
                    corners_dic[corner_name][-1] += "\n"+line
                else:
                    corners_dic[corner_name].append(line)
    except FileNotFoundError:
        pass

    return corners_dic

def build_corner_netlist(cwd, corners_data, netlist):
     for n in netlist: create_corner_netlist(n, cwd, corners_data)

def create_corner_netlist(netlist,cwd, corners_data):
    with open(os.path.join(cwd,netlist), "r") as f:
        netlist_data = f.read()

    for corner, values in corners_data.items():
        corner_netlist = os.path.join(cwd, corner + "_" + netlist)

        with open(corner_netlist, 'w') as f:
            f.write(netlist_data.replace(".include 'nominal.corner'", "\n".join(values)))

def remove_tmp_files(netlist, cwd):
    for f in glob.glob(cwd+'*'+MEAS_OUT): os.remove(f)
    for f in glob.glob(cwd+'*'+VARS): os.remove(f)
    for f in glob.glob(cwd+'*'+GLOBAL_PARAMS): os.remove(f)

    for n in netlist:
        for f in glob.glob(cwd+'*_'+n): os.remove(f)

def write_design_variables(cwd, param, val):
    with open(os.path.join(cwd,VARS), 'w') as outFile:
      outFile.write('*  Design Var\n .param\n')
      outFile.write('\n'.join(('+{}={}'.format(p,v) for p, v in zip(param, val))))

class ThreadSafeDict(dict):
    def __init__(self, * p_arg, ** n_arg) :
        dict.__init__(self, * p_arg, ** n_arg)
        self._lock = threading.Lock()

    def __enter__(self) :
        self._lock.acquire()
        return self

    def __exit__(self, type, value, traceback) :
        self._lock.release()

class Ngspice():
    '''
    '''

    def __init__(self, cwd, netlist, param,  corners='corners.inc', num_threads=6):
        '''
    The netlist must be built including the nominal libraries and parameter
    definitions in a file named: 	nominal.corner

    Additional corners can be specified in a coners file using .alter syntax.
    Exmaple corners.inc:
    .ALTER SS
        .lib '../ptm130/ptm-130.lib' SS
        .param vdd = 1.5

    .ALTER FF
        .lib '../ptm130/ptm-130.lib' FF
        .param vdd = 1.5
    EOF

        params:
         cwd: current working directory string
         netlist: list of netlists <strings>
         corners: name os corner file
         num_treads: number of threads
        '''
        self.cwd = cwd
        self.netlist = netlist
        self.corners_data = load_corners(os.path.join(self.cwd, "cir", corners)) if corners is not None else {}
        self.num_threads=min(num_threads, os.cpu_count())
        self.param = param

        #list to hold the simulations to be executed per point
        # netlist * corners
        self.simulation_list = []
        for n in self.netlist:
            #place nominal 'TT' in queue
            self.simulation_list.append(Simulation('TT',n, n+MEAS_OUT, None, None))
            #if corners defined, build their simulation definitions
            for corner in self.corners_data.keys():
                self.simulation_list.append(Simulation(corner,corner + "_" + n, corner + "_" + n + MEAS_OUT, None, None))

        # create files for worker threads
        thread_source_path = os.path.join(self.cwd, "t")
        if os.path.exists(thread_source_path): shutil.rmtree(path=thread_source_path)
        shutil.copytree(src=os.path.join(self.cwd, "cir"), dst=thread_source_path)

        #create corner netlist in source dir for worker threads
        build_corner_netlist(thread_source_path,  self.corners_data, self.netlist)

        #copy folders fo reach worker thread and launch thread
        self.q = queue.Queue()

        for tid in range(self.num_threads):
            thread_path = os.path.join(self.cwd, "t"+str(tid))
            if os.path.exists(thread_path):  shutil.rmtree(thread_path)
            shutil.copytree(src=thread_source_path, dst=thread_path)
            t = threading.Thread(target=worker, args=(self.q, thread_path, self.param))
            t.daemon = True
            t.start()

    def __del__(self):
        '''
        Tells worker threads to stop, exits imediatly
        '''
        for _ in range(self.num_threads):
            self.q.put(None) #poison pill
        # not waiting on self.q.join() , workser dont update
        # task count on exit either

    def simulate (self, values):
        '''
        Runs ngspice.
        param: values: ndarray of shape (n, len(self.param)
        '''
        n = values.shape[0]

        #List of dictionariy that will hold sim_results of all corners
        sim_results = [ThreadSafeDict() for _ in range(n)]

        # fills jobs queue
        for i in range(n):
            for sim in self.simulation_list:
                self.q.put(Simulation(
                    sim.corner_id, sim.netlist_file, sim.meas_file,
                    sim_results[i], values[i,:]))

        self.q.join()

        # consider to create/update worker threads for this also
        for i in range(n):
            ngmeas.compute_additional_measures(sim_results[i])

        return sim_results

def simulate(cwd, netlist, param, val, corners='corners.inc', num_threads=6):
    '''
    Runs ngspice.

    The netlist must be built including the nominal libraries and parameter
    definitions in a file named: 	nominal.corner

    Additional corners can be specified in a coners file using .alter syntax.
    Exmaple corners.inc:
    .ALTER SS
        .lib '../ptm130/ptm-130.lib' SS
        .param vdd = 1.5

    .ALTER FF
        .lib '../ptm130/ptm-130.lib' FF
        .param vdd = 1.5
    EOF
    '''
    return Ngspice(cwd,netlist,param, corners,num_threads).simulate(val)

if __name__ == '__main__':

    for _ in range(100):
        run_simulation("./circuit_examples/ptm130_folded_cascode/cir", "tb_ac.cir", "tb_ac.cir_meas.txt.test", timeout=2)
        curr_meas = parse_sim_results("./circuit_examples/ptm130_folded_cascode/cir", "tb_ac.cir_meas.txt.test")


import time,requests,ctypes,keyboard,numpy,pyautogui,cv2,io
import PySimpleGUI as sg
from PIL import Image, ImageFile
from keyboard_listener import KeyboardListener
from mouse import Mouse
from mouse_listener import MouseListener
from screenshot import make_screenshots
from server import *

class Gui:
    def __new__(cls):
        if not hasattr(cls, 'instance'):
            cls.instance = super(Gui, cls).__new__(cls)
        return cls.instance

    def __init__(self):
        screensize = pyautogui.size()
        self.__layout = [
            [sg.Text('Main menu', justification='center', font='20', size=(100, 1))],
            [sg.Slider(range=(0,screensize.width),tooltip='Start x', default_value=0, key='-X-',
                       size=(100,15),pad=(10,0), orientation='horizontal', font=('Helvetica', 12))],
            [sg.Slider(range=(0,screensize.height),tooltip='Start y', default_value=0, key='-Y-',
                       size=(100,15),pad=(10,0), orientation='horizontal', font=('Helvetica', 12))],
            [sg.Slider(range=(1,screensize.width), pad=(10,5), tooltip='window record width', default_value=screensize.width, key='-WIDTH-',
                    size=(100,15), orientation='horizontal', font=('Helvetica', 12))],
            [sg.Slider(range=(1,screensize.height),tooltip='window record height', default_value=screensize.height, key='-HEIGHT-',
                       size=(100,15),pad=(10,5), orientation='horizontal', font=('Helvetica', 12))],
            [sg.Slider(range=(1,100),tooltip='quality image', default_value=50, key='-QUALITY-',
                       size=(100,15),pad=(10,5), orientation='horizontal', font=('Helvetica', 12))],
            [sg.Radio('I\'m share a screen (Server)', "RADIO1",enable_events=True,key='-SHARE-', default=True),
             sg.Radio('I\'m control a screen (Client)', "RADIO1",enable_events=True,key='-CONTROL-')],
            [sg.Text('IP:               ',key='local ip label'), sg.Input(tooltip='For server - local ip, for client - global server ip',key='ip')],
            [sg.Text('Port:             '), sg.Input(default_text='0',tooltip='Server port',key='port')],
            [sg.Button('Start', key='-START-', disabled=True, size=(100, 2))],
            [sg.Button('Apply', key='-APPLY-', size=(100, 2))],
            [sg.Button('Test connection',key='-TEST-CONNECT-',disabled=True, size=(100, 2))],
            [sg.Button('Exit',key='-EXIT-', size=(100, 2))]
        ]
        self.__window = sg.Window('Computer remote control', self.__layout, size=(500, 600), icon='icon.ico')

    def start(self):
        while True:
            event, values = self.__window.read(timeout=10)
            if event == sg.WIN_CLOSED or event == '-EXIT-':
                if Global.is_server_running:
                    requests.get(f'http://{Global.ip}:{Global.port}/shutdown')
                break

            elif event == '-SHARE-':
                Global.role = 'share'

            elif event == '-CONTROL-':
                Global.role = 'control'

            elif event == '-START-':
                self.__window.hide()
                if Global.role == 'share':
                    threading.Thread(target=make_screenshots,args=('screenshot.jpg',Global.quality,Global.region)).start()

                    while not keyboard.is_pressed('Esc'):
                        keyboard.play(Global.keyboard_events)
                        Global.keyboard_events.clear()
                        for event in Global.mouse_events:
                            if event['event_name'] == 'click' and event['pressed'] == True and event['button'] == 'left':
                                Mouse.press_left()
                            elif event['event_name'] == 'click' and event['pressed'] == True and event['button'] == 'right':
                                Mouse.press_right()
                            elif event['event_name'] == 'click' and event['pressed'] == False and event['button'] == 'left':
                                Mouse.release_left()
                            elif event['event_name'] == 'click' and event['pressed'] == False and event['button'] == 'right':
                                Mouse.release_right()
                            elif event['event_name'] == 'scroll':
                                Mouse.scroll(10*event['dx'],10*event['dy'])
                            elif event['event_name'] == 'move':
                                Mouse.move_to(event['x'],event['y'])
                        Global.mouse_events.clear()

                elif Global.role == 'control':
                    KeyboardListener().start_listen()
                    MouseListener().start_listen()
                    cv2.namedWindow('Window')
                    screensize = pyautogui.size()
                    x, y, w, h = cv2.getWindowImageRect('Window')
                    cv2.moveWindow('Window', screensize.width // 2 - w // 2, screensize.height // 2 - h // 2)
                    ImageFile.LOAD_TRUNCATED_IMAGES = True

                    while cv2.getWindowProperty('Window', 1) > 0:
                        keyboard_events, mouse_events = Global.keyboard_events, Global.mouse_events
                        Global.keyboard_events = []
                        Global.mouse_events = []
                        res = requests.get(f'http://{Global.ip}:{Global.port}/data',data={'mouse_events':json.dumps(mouse_events),'keyboard_events':json.dumps(keyboard_events)})

                        if res.content:
                            stream = io.BytesIO(res.content)
                            frame = Image.open(stream).convert("RGBA")
                            stream.close()
                            frame = numpy.array(frame)
                            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
                            cv2.imshow('Window', frame)
                            if not cv2.waitKey(1):
                                break

                self.__window.un_hide()

            elif event == '-APPLY-':
                Global.ip = values['ip']
                Global.port = int(values['port'])
                Global.region = (values['-X-'],values['-Y-'],values['-WIDTH-'],values['-HEIGHT-'])
                Global.quality = values['-QUALITY-']
                try:
                    if Global.role == 'share':
                        run_server(Global.ip,Global.port)
                except Exception as e:
                    ctypes.windll.user32.MessageBoxA(None, bytes(str(e),'utf-8'), b"Error", 0x30 | 0x0)
                self.__window['-TEST-CONNECT-'].update(disabled=False)
                self.__window['-APPLY-'].update(disabled=True)
                self.__window['-SHARE-'].update(disabled=True)
                self.__window['-CONTROL-'].update(disabled=True)
                self.__window['-X-'].update(disabled=True)
                self.__window['-Y-'].update(disabled=True)
                self.__window['-WIDTH-'].update(disabled=True)
                self.__window['-HEIGHT-'].update(disabled=True)
                self.__window['-START-'].update(disabled=False)

            elif event == '-TEST-CONNECT-':
                start = time.time()
                try:
                    requests.get(f'http://{Global.ip}:{Global.port}/ping')
                except Exception as e:
                    ctypes.windll.user32.MessageBoxA(None, bytes(str(e),'utf-8'), b"Error", 0x30 | 0x0)
                ping_ms = round((time.time() - start) * 1000, 1)
                ctypes.windll.user32.MessageBoxA(None, bytes(f"Ping: {ping_ms} ms",'utf-8'), b"Info", 0x40 | 0x0)

        self.__window.close()


"""Scripts for visualization"""

from collections import Counter
import io

from PIL import Image
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns

from scripts.utils.constants import CSV_ORG_OVERVIEW
from scripts.utils.io_utils import get_masks, get_orig_imgs
from scripts.utils.dw_mri import get_metrics_global_cyst_seg_dw_mri
from scripts.utils.global_cyst_classification import get_compactness

def plot_compactness(df, roc_auc_compactness, save_to=''):
    """Swarmplot of compactness, x = low / high quality, y = compactness. One point per sample.

    Args:
        df (pd.DataFrame): one row per organoid, with columns 'compactness' and 'Organoid quality'
        roc_auc_compactness (float): ROC for writing it in the title
        save_to (str, optional): path to save plot. Defaults to ''.
    """
    sns.set_style('white')
    plt.figure(figsize=(4, 3), facecolor='white')
    sns.swarmplot(data=df.sort_values('Organoid quality',
                  ascending=True), y='compactness', x='Organoid quality')
    plt.ylabel('Compactness')
    plt.title(f'ROC AUC: {roc_auc_compactness}')
    plt.grid()
    sns.despine()
    plt.tight_layout()

    if save_to != '':
        plt.savefig(save_to, dpi=300)

def get_df_cyst_sizes():
    """Get dataframe comprising cyst size for every organoid.

    Returns:
        x (pd.DataFrame): dataframe comprising cyst size for every organoid
    """
    # calculate cysticity
    cyst_masks = get_masks(kind='gt_cyst_loc_all45')
    org_ids, cyst_sizes = [], []
    for k, v in cyst_masks.items():
        c = Counter(v.flatten())[1]
        org_ids.append(k)
        cyst_sizes.append(c)
    x = pd.DataFrame.from_dict({'org_id': org_ids, 'cyst_size': cyst_sizes})
    return x

def plot_correlation_compactness_cysticity(save_to):
    """Scatterplot with x = cyst size y = compactness.

    Args:
        save_to (str, optional): path to save plot. Defaults to ''.
    """
    df = get_compactness()
    x = get_df_cyst_sizes()

    df = df.merge(x)

    # plot
    plt.figure(figsize=(4, 4), facecolor='white')
    sns.scatterplot(data=df, x='cyst_size', y='compactness')
    pearson = df['cyst_size'].corr(df['compactness'], method='pearson')
    plt.title(f'Correlation: {pearson:.2f} (Pearson)')
    plt.grid()
    sns.despine()
    plt.ylabel('Compactness')
    plt.xlabel('Cyst size in voxels')

    plt.tight_layout()
    if save_to != '':
        plt.savefig(save_to, dpi=300)

def plot_trace_lq_hq_mean_org_int(save_to=''):
    """Swarmplot of mean organoid intensity, x = low / high quality, y = mean organoid intensity. One point per sample.

    Args:
        save_to (str, optional): path to save plot. Defaults to ''.
    """
    params = {'mathtext.default': 'regular'}
    plt.rcParams.update(params)
    _, dfs_dwmri = get_metrics_global_cyst_seg_dw_mri()
    df_trace = dfs_dwmri[1]
    df_trace['mean_organoid_intensity_x1000'] = df_trace['mean_organoid_intensity'] * 1000
    plt.figure(figsize=(4, 3), facecolor='white')
    sns.swarmplot(data=df_trace.sort_values(
        'Organoid quality', ascending=True),
        y='mean_organoid_intensity_x1000', x='Organoid quality')
    plt.ylabel(r'Trace [$x10^{-3}$ $mm^2/s$]')
    plt.title('p < .001', fontsize=11)
    plt.xlabel('Organoid quality', fontsize=11)
    plt.grid()
    sns.despine()
    plt.tight_layout()
    if save_to != '':
        plt.savefig(save_to, dpi=700)

def get_org_boundaries(org_locs_planes, i):
    """Extracts organoid boundaries.

    Args:
        org_locs_planes (list): containing the to-be-visualized planes and plane boundaries.
                                Example: [(['88-117', '59', '32-67'], ['88-117', '60', '32-67']),
                                          (['40-69', '41', '40-75'], ['40-69', '45', '40-75']),
                                          (['5-55', '58', '15-90'], ['5-55', '61', '15-90']),
                                          (['17-67', '36', '15-85'], ['17-67', '53', '15-85'])]
        i (int): in range(4)

    Returns:
        list: list of 10 values, e.g. for example above and i = 0: [88, 117, 59, 32, 67, 88, 117, 60, 32, 67]
    """
    a, b = org_locs_planes[i][0][0].split('-')
    c = org_locs_planes[i][0][1]
    d, e = org_locs_planes[i][0][2].split('-')

    f, g = org_locs_planes[i][1][0].split('-')
    h = org_locs_planes[i][1][1]
    j, k = org_locs_planes[i][1][2].split('-')
    x = [a, b, c, d, e, f, g, h, j, k]
    x = [int(z) for z in x]
    return (*x,)

def plot_examples_lq_hq_organoids(
        org_ids, compactnesses, org_locs_planes, save_to=''):
    """Creates 2x2 plot with two examples per organoid and four organoids in total

    Args:
        org_ids (list): list of four organoid ids
        compactnesses (list): list of four compactnesses corresponding to the org_ids
        org_locs_planes (list): containing the to-be-visualized planes and plane boundaries.
                                Example: [(['88-117', '59', '32-67'], ['88-117', '60', '32-67']),
                                          (['40-69', '41', '40-75'], ['40-69', '45', '40-75']),
                                          (['5-55', '58', '15-90'], ['5-55', '61', '15-90']),
                                          (['17-67', '36', '15-85'], ['17-67', '53', '15-85'])]
        save_to (str, optional): path to save plot. Defaults to ''.
    """
    assert len(org_ids) == 4
    imgs = get_orig_imgs(kind='mri')
    extract_imgs = []
    img_bufs = [io.BytesIO() for i in range(4)]
    i = 0
    for org_id in org_ids:
        #     print('org_id', org_id)
        _, axs = plt.subplots(1, 2, figsize=(5, 5))
        a, b, c, d, e, f, g, h, j, k = get_org_boundaries(org_locs_planes, i)
        axs[0].imshow(np.rot90(imgs[org_id][a:b, c, d:e]*-1, 3), cmap='Greys')
        axs[1].imshow(np.rot90(imgs[org_id][f:g, h, j:k]*-1, 3), cmap='Greys')
        axs[0].axis('off')
        axs[1].axis('off')
        plt.tight_layout()
        plt.savefig(img_bufs[i])
        plt.close()
        i += 1

    extract_imgs = [Image.open(img_buf) for img_buf in img_bufs]
    subplot_prefixes = ['(a)', '(b)', '(c)', '(d)']
    _, axs = plt.subplots(2, 2, facecolor='white')
    axs = axs.T.ravel()
    for i in range(4):
        axs[i].imshow(extract_imgs[i])
        axs[i].axis('off')
        axs[i].set_title(
            f'{subplot_prefixes[i]} {org_ids[i]} (C={compactnesses[i]})', y=-
            0.01)

    axs[0].text(-10.8, 0.01, 'High-quality organoids', fontsize=12,
                verticalalignment='center', horizontalalignment='left',
                fontweight='bold')
    axs[2].text(-10.8, 0.01, 'Low-quality organoids', fontsize=12,
                verticalalignment='center', horizontalalignment='left',
                fontweight='bold')
    plt.tight_layout()
    if save_to != '':
        plt.savefig(save_to, dpi=300)

def plot_organoid_growth_over_time():
    """lineplot with x = days y = organoid volume
    """
    org_ids, org_sizes = [], []
    gt = get_masks(kind='gt_org_loc')
    for org_id, org_gt in gt.items():
        org_ids.append(org_id)
        c = Counter(org_gt.flatten())
        assert len(c) == 2
        assert 0 in c and 1 in c
        org_sizes.append(c[1])

    df = pd.DataFrame.from_dict({'org_id': org_ids, 'org_size': org_sizes})
    df = df.merge(pd.read_csv(CSV_ORG_OVERVIEW),
                  on='org_id').sort_values('org_id')
    df['org_nr'] = df['org_nr'].astype('str')

    sns.lineplot(data=df, x='day', y='org_size', hue='org_nr')
    plt.grid()
    plt.ylabel('Organoid volume in voxel')
    plt.xlabel('Day')
    plt.legend(title='Organoid', loc='center left', bbox_to_anchor=(1, 0.5))
    sns.despine(bottom=True, top=True, left=True, right=True)
    plt.tight_layout()
    return df

def get_green_binary_colors(x):
    """Converts 1-values of binary array to green RGB color.

    Args:
        x (ndarray): binary numpy array

    Returns:
        x (ndarray): binary numpy array with 1-values converted to green color.
    """
    x = x[:, :, np.newaxis]
    x = np.where(x == 0.0, [1.0, 1.0, 1.0, 0.0], x)
    x = np.where(x == 1.0, [60/255, 179/255, 113/255, 0.6], x)
    return x


from flask import Flask, render_template, request
from WDS import WDSWrapper
import json

app = Flask(__name__)

# Initialise WDS wrapper
with open('wds_credentials.json') as creds:
    wds_credentials = json.load(creds)
    wds = WDSWrapper(wds_credentials)

@app.route('/', methods=['GET'])
@app.route('/search', methods=['GET'])
def index():
    return render_template('search.html')

@app.route('/results', methods=['POST'])
def get_results():

    # Validate the query string
    # length? special characters?
    query = request.form.get('natural_language_query')

    # Run query
    print('Query: ' + request.form.get('natural_language_query'))
    results = wds.query(natural_language_query=query, max_docs=10)

    # Format results
    # get doc title, passage(s), page numbers?
    # format into appropriate dictionary
    # display_results = {}
    # for result in results['results']:

    # Render template ... with fields: title + passage for documents, original search query should be in search box (different colour or something)

    return render_template('results.html', results=results['results'])

if __name__=='__main__':
    app.run(debug=True)


import scrapy

class ScrapingSpider(scrapy.Spider):
    name = 'scraping'

    def start_requests(self):
        start_urls = [
                    'https://www.kaggle.com/aakashverma8900'
                ]
        for url in start_urls:
            yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):
        item = {}
        title = response.css('title::text').get()
        item['title'] = title
        yield item


#-*- coding:utf-8 -*-

import datetime
import StringIO

from django.db import models
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes import generic
from django.contrib.auth.models import User
from django.db.models import signals
from hibird.models import slugify

from utils import get_widget
from current_user import get_current_user, get_request
from jsonfield import JSONField, dumps as json_dumps, loads as json_loads

DEFAULT_LAYOUT_SLUG = getattr(settings, "DEFAULT_LAYOUT_SLUG", "base")
DEFAULT_LAYOUT_TEMPLATE = getattr(settings, "DEFAULT_LAYOUT_TEMPLATE", "base.html")

class Layout (models.Model):
	"""
		layout.containers = {
			'_parent_': {
				'header': [
					('module1', {}),
					('module2', {}),
				],
				'footer': [
					('module1', {}),
					('module2', {}),
				],
			},
			'article': [
				('entry', {}),
				('comment', {}),
			],
		}
	"""

	name = models.CharField(verbose_name="Layout Name", max_length=200)
	slug = models.CharField(verbose_name="Layout Slug", max_length=200, null=True, blank=True)
	entity_type = models.CharField(verbose_name="Entity Type", max_length=200, null=True, blank=True)

	owner = models.ForeignKey(User, null=True, related_name="layout_owned", editable=False)
	created_on = models.DateTimeField(auto_now_add=True, editable=False)
	updater = models.ForeignKey(User, null=True, related_name="layout_updated", editable=False)
	updated_on = models.DateTimeField(auto_now=True, editable=False)

	parent = models.ForeignKey('self', null=True, blank=True)
	template = models.CharField(max_length=200, null=True, blank=True)
	containers = models.TextField(null=True, blank=True)

	is_active = models.BooleanField(default=True, editable=False)

	@classmethod
	def get(cls, slug=None):
		if slug:
			slug = slug.replace('/','-').strip('-')
		try:
			return Layout.objects.filter(entity_type='').get(slug=slug)
		except:
			pass
		try:
			return Layout.objects.filter(entity_type__isnull=True).get(slug=slug)
		except:
			pass
		try:
			layout = Layout.objects.get(slug=DEFAULT_LAYOUT_SLUG)
			return layout
		except:
			layout, created = Layout.objects.get_or_create(name="Base", slug=DEFAULT_LAYOUT_SLUG)
			return layout

	def get_template(self):
		if self.template:
			return self.template
		return DEFAULT_LAYOUT_TEMPLATE

	def render_widget(self, entity, context, storage={}):
		(widget_name, widget_context) = context
		widget = get_widget(widget_name)(context=widget_context, storage=storage)
		request = get_request()
		return widget.render(request)

	def render_container(self, entity, container_name, storage={}):
		try:
			widgets_context = self.get_containers().get(container_name)
			if not widgets_context:
				return ""
			writer = StringIO.StringIO()
			try:
				for widget_context in widgets_context:
					writer.write(self.render_widget(entity, widget_context, storage=storage))
				return writer.getvalue()
			finally:
				writer.close()
		except:
			pass

	def get_containers(self):
		try:
			return self._flat_containers
		except:
			try:
				c = json_loads(self.containers or "")
			except ValueError:
				c = {}
			r = dict(c.get('_parent_', {}).items())
			for (k,v) in c.items():
				if k.startswith('_'): continue
				if v:
					r[k] = v
			self._flat_containers = r
			return r

	def save(self,*args,**kwargs):
		try:
			if not self.parent and self.slug != DEFAULT_LAYOUT_SLUG:
				self.parent = Layout.get()
			if self.parent and self.parent != self:
				try:
					c = json_loads(self.containers or "")
				except ValueError:
					c = {}
				r = dict(self.parent.get_containers().items())
				for (k,v) in c.items():
					if v:
						r[k] = None
						del r[k]
				if r:
					c['_parent_'] = r
				self.containers = json_dumps(c)
		except:
			pass
		super(Layout, self).save(*args, **kwargs)

		children = Layout.objects.filter(parent=self).exclude(pk=self.pk)
		for child in children:
			child.save()

	def __unicode__(self):
		if self.entity_type:
			return "%s / %s" % (self.entity_type, self.slug)
		else:
			return self.name

	class Slugification:
		prepopulated_fields = {'slug': ('name',),}


# -*- coding: utf-8 -*-
"""

"""
import json
import time
import http.client
import os.path
from collections import OrderedDict

import parameterlib as pl

import pyqtgraph as pg
from pyqtgraph.Qt import QtCore, QtGui
import pyqtgraph.parametertree.parameterTypes as pTypes
from pyqtgraph.parametertree import Parameter, ParameterTree, \
    ParameterItem, registerParameterType
import pyqtgraph.console
from pyqtgraph.dockarea import *

WINSIZE = [1200, 800]
MAXLENGTH = 60 * 60 * 24
PENCOLOR = ['r', 'g', 'b', 'c', 'm', 'y', 'w', 'k']

HOST = "10.42.0.116"
#HOST = "192.168.10.116"
#HOST = "192.168.10.170"
#HOST = "127.0.0.1"
PORT = 5000

HEAD_PATH, SCRIPT_NAME = os.path.split(__file__)

def rebuild_ordereddict(orddict):
    """Create a new dictionary with the meaningful values"""
    newdict = OrderedDict()
    for key in orddict:
        if orddict[key][0] != None:
            newdict[key] = orddict[key][0]
        else:
            newdict[key] = rebuild_ordereddict(orddict[key][1])
    return newdict

def search_entry_index(entry, builddict):
    """ """
    if type(builddict) == type([]):
        for item in builddict:
            if item['name'] == entry:
                return builddict.index(item)

    elif type(builddict) == type({}):
        for item in builddict['children']:
            if item['name'] == entry:
                return builddict['children'].index(item)

    else:
        return None

def set_teds_fields_from_dict(inputdict, teds, basetedsdic, baseregisterdict):
    """Uses the dictionary to set the TEDS fields"""
    rawteds = []
    if len(inputdict) != 0:
        rawteds = basetedsdic[:]

        for key in inputdict:
            if type(inputdict[key]) == type({}):
                rawteds.append(dict(baseregisterdict))
                rawteds[-1]['name'] = key

                for subkey in inputdict[key]:
                    index = search_entry_index(subkey, baseregisterdict)
                    if index != None:
                        rawteds[-1]['children'][index]['value'] = inputdict[key][subkey]

            else:
                index = search_entry_index(key, basetedsdic)
                if index != None:
                    rawteds[index]['value'] = inputdict[key]

    teds.clearChildren()
    teds.addChildren(rawteds)

def set_teds_list_from_dict(inputdict, tedslist):
    """ """
    tedslist.clearChildren()
    for key in inputdict:
        tedslist.addChild({'name': key, 'type': 'group', 'children': [
            {'name': inputdict[key], 'type': 'bool', 'value': False}
            ]})

def send_teds_to_server(teds, host, port, header):
    """ """
    tedsvalues = teds.getValues()
    tedsvalues = rebuild_ordereddict(tedsvalues)
    tedsstr = json.dumps(tedsvalues)
    http_post(pl.CONSOLE_TEDS_PAGE + '/' + tedsvalues[pl.TEDS_NAME_ID],
              host, port, header, tedsstr)

def update_teds_tedsdictlist(teds, tedsdictlist, address, host, port, header):
    """ """
    http_get_tedslist(address, host, port, header, tedsdictlist)
    set_new_teds(teds)

## If anything changes in the TEDS list, print a message
def teds_list_change(param, changes):

    global selected_teds, par_teds
    (param, change, data) = changes[0]

    if change == 'value':
        tedschanged = param
        tedschangedparent = tedschanged.parent().parent()
        selected_teds = tedschanged
        selectedtedsname = tedschanged.name()
        deselect_other_teds(tedschangedparent, selectedtedsname)
        teds = http_get(pl.CONSOLE_TEDS_PAGE + '/' +
                        tedschanged.parent().name(), HOST, PORT, pl.HEADERS)
        set_teds_fields_from_dict(teds, par_teds, pl.TEDS_BUILD_DICT,
                                  pl.REGISTER_BUILD_DICT)

def deselect_other_teds(tedslist, lastselected): # need to prevent uncheck teds
    """When select one TEDS, deselect the others"""
    with tedslist.treeChangeBlocker():
        for ch in tedslist.children():
            for gch in ch:
                if gch.name() is not lastselected:
                    gch.setValue(False)
        tedslist.treeStateChanges = [] # do not flush the changes to prevent inf loop

def set_new_teds(teds):
    """Clear all fields and create a new ID for the new TEDS"""
    for child in teds:
        if child.hasChildren():
            child.remove()
        else:
            child.setToDefault()
            teds.child(pl.TEDS_NAME_ID).setValue(pl.id_generator())

def plot_sensors(sensordata, plotwidget, plotcurves, sensorsdock):
    """ """
    rowcont = 0
    for transducer in sensordata['sensordata']:

        plotwidget[transducer] = pg.PlotWidget(
            title=transducer+': '+sensordata['sensordata'][transducer]['name'])
        plotwidget[transducer].showGrid(x = True, y = True, alpha = 0.5)
        plotwidget[transducer].addLegend()
        sensorsdock.addWidget(plotwidget[transducer], col=0, row=rowcont)

## Experimental
##        plotwidget[transducer] = sensorsdock.addPlot(
##            title=transducer+': '+sensordata['sensordata'][transducer]['name'],
##            col=0, row=rowcont)
##        plotwidget[transducer].showGrid(x = True, y = True, alpha = 0.5)
##        plotwidget[transducer].addLegend()

        plotcurves[transducer] = {}
        i = 0
        for register in sensordata['sensordata'][transducer]:
            if register != 'name':
                plotcurves[transducer][register] = plotwidget[transducer].plot(
                    sensordata['timedata'][0],
                    sensordata['sensordata'][transducer][register][0],
                    name=register + ' [' + sensordata['sensordata'][transducer][register][1] + ']',
                    pen=pg.mkPen(PENCOLOR[i]))
                i += 1

        rowcont += 1

def load_controllers_data(controllerparameter):
    """"""
    controllerdata = http_get(pl.CONSOLE_CONTROLLERS_DATA, HOST,
                              PORT, pl.HEADERS)

    controllerparameter.clearChildren()

    for transducerid in controllerdata:
        controllerparameter.addChild(
            {'name': transducerid, 'type': 'group', 'children': [
                {'name': 'name',
                 'type': 'str',
                 'value': controllerdata[transducerid]['name'],
                 'readonly': True}
                ]})

        for register in controllerdata[transducerid]:

            if register != 'name':
                controllerparameter.child(transducerid).addChild(
                    {'name': register,
                     'type': 'group',
                     'children': []})

                index = 0
                for param in controllerdata[transducerid][register]:

                    if pl.CONTROLLERS_PARAMETERS[index] == pl.TEDS_NAME_ACTUATORVALUE:
                        controllerparameter.child(transducerid).child(register).addChild(
                            {'name': pl.CONTROLLERS_PARAMETERS[index],
                             'type': 'str',
                             'value': controllerdata[transducerid][register][index],
                             'readonly': False})
                    else:
                        controllerparameter.child(transducerid).child(register).addChild(
                            {'name': pl.CONTROLLERS_PARAMETERS[index],
                             'type': 'str',
                             'value': controllerdata[transducerid][register][index],
                             'readonly': True})
                    index += 1

## HTTP Protocol
def http_post(address, host, port, headers, body):
    """ """
    conn = http.client.HTTPConnection(host=host, port=port)
    conn.request("POST", address, body, headers)
    response = conn.getresponse()
    print(response.status, response.reason)
    data = response.read()
    print(data)
    conn.close()
    return data

def http_get(address, host, port, headers):
    """ """
    conn = http.client.HTTPConnection(host=host, port=port)
    conn.request("GET", address, headers=headers)
    response = conn.getresponse()
    print(response.status, response.reason)
    data = response.read()
    print(data)
    convdata = json.loads(data)
    conn.close()
    return convdata

def http_get_tedslist(address, host, port, headers, tedslist):
     recievedtedslist = http_get(address, host, port, headers)
     set_teds_list_from_dict(recievedtedslist, tedslist)

# Buttons functions
def newteds():
    set_new_teds(par_teds)
    deselect_other_teds(par_tedslist, 'newteds')

def save():
    send_teds_to_server(par_teds, HOST, PORT, pl.HEADERS)
    update_teds_tedsdictlist(par_teds, par_tedslist, pl.CONSOLE_PAGE, HOST,
                             PORT, pl.HEADERS)
    set_new_teds(par_teds)

def remove():
    global selected_teds

    if selected_teds.name() != None:
        http_get(pl.REMOVE_TEDS_PAGE + '/' + selected_teds.parent().name(),
                 HOST, PORT, pl.HEADERS)
        update_teds_tedsdictlist(par_teds, par_tedslist, pl.CONSOLE_PAGE, HOST,
                                 PORT, pl.HEADERS)

def copy():
    par_teds.child(pl.TEDS_NAME_ID).setValue(pl.id_generator())
    deselect_other_teds(par_tedslist, 'newteds')

def addfunction():
    par_teds.addChild(pl.REGISTER_BUILD_DICT, autoIncrementName=True)

def update_plot():

    plot_curves  = {}
    plot_widget = {}
    sensors_data = http_get(pl.CONSOLE_PLOT_DATA, HOST, PORT, pl.HEADERS)
##    plot_sensors(sensors_data, plot_widget, plot_curves, plotsarea) # Experimental
    plot_sensors(sensors_data, plot_widget, plot_curves, sensors_dock)

def update_controllers():

    load_controllers_data(controllers)

def save_controllers():

    ctrldata = pl.build_dict_from_ordereddict(controllers.getValues())
    print(json.dumps(ctrldata, indent=1))

# Application

app = QtGui.QApplication([])

#variables initialization
selected_teds = None

## Create tree of Parameter objects
par_teds = Parameter.create(name='TEDS', type='group',
                            children=pl.TEDS_BUILD_DICT)
par_tedslist = Parameter.create(name='TEDS List', type='group', children={})

## Create ParameterTree tree
teds_tree = ParameterTree()
teds_tree.setParameters(par_teds, showTop=False)

tedslist_tree = ParameterTree()
tedslist_tree.setParameters(par_tedslist, showTop=False)

#par_teds.sigTreeStateChanged.connect(field_teds_change)
par_tedslist.sigTreeStateChanged.connect(teds_list_change)

## Create Window
win = QtGui.QMainWindow()
area = DockArea()
win.setCentralWidget(area)
win.setWindowTitle('Application')

d1 = Dock('TEDS Editor area', size=(WINSIZE[0]/4, WINSIZE[1]))
d2 = Dock('Availble TEDS', size=(WINSIZE[0]/4,WINSIZE[1]))
area.addDock(d1, 'left')
area.addDock(d2, 'right')

w1 = pg.LayoutWidget()
w1.addWidget(teds_tree)
d1.addWidget(w1)
w2 = pg.LayoutWidget()
w2.addWidget(tedslist_tree, row=0, col=0)
d2.addWidget(w2)

win.show()
win.resize(WINSIZE[0],WINSIZE[1])

http_get_tedslist(pl.CONSOLE_PAGE, HOST, PORT, pl.HEADERS, par_tedslist)
set_new_teds(par_teds)

## Buttons
saveBtn = QtGui.QPushButton('Save TEDS')
newtedsBtn = QtGui.QPushButton('New TEDS')
removeBtn = QtGui.QPushButton('Remove TEDS')
copyBtn = QtGui.QPushButton('Copy TEDS')
addfunctBtn = QtGui.QPushButton('Add Register Function')

w1.addWidget(saveBtn, row=1, col=0)
w1.addWidget(newtedsBtn, row=4, col=0)
w1.addWidget(removeBtn, row=6, col=0)
w1.addWidget(copyBtn, row=7, col=0)
w1.addWidget(addfunctBtn, row=8, col=0)

saveBtn.clicked.connect(save)
newtedsBtn.clicked.connect(newteds)
removeBtn.clicked.connect(remove)
copyBtn.clicked.connect(copy)
addfunctBtn.clicked.connect(addfunction)

#Create Sensor Area

sensors_dock = Dock('Sensor Area', size=(WINSIZE[0]/4, WINSIZE[1]))
area.addDock(sensors_dock, 'right')

##Experimental
##sensors_dock = Dock('Sensor Area', size=(WINSIZE[0]/4, WINSIZE[1]))
##area.addDock(sensors_dock, 'right')
##scrollarea = pg.QtGui.QScrollArea()
##sensors_dock.addWidget(scrollarea)
##plotsarea = pg.GraphicsLayoutWidget()
##plotsarea.setFixedHeight(WINSIZE[1])
##plotsarea.setFixedWidth(WINSIZE[0]/4)
##scrollarea.setWidget(plotsarea)

#Buttom area
dock_button = Dock('Button Area', size=(WINSIZE[0]/4, WINSIZE[1]/10))
area.addDock(dock_button, 'right')
area.moveDock(dock_button, 'bottom', sensors_dock)
#Update button
sensors_button_update = QtGui.QPushButton('Update Plots')
dock_button.addWidget(sensors_button_update, row=0, col=0)
sensors_button_update.clicked.connect(update_plot)

# Create Controllers Area
controllers_dock = Dock('Controllers Area', size=(WINSIZE[0]/4, WINSIZE[1]))
area.addDock(controllers_dock, 'right')

controllers = Parameter.create(name='Controllers', type='group',children={})
controllers_tree = ParameterTree()
controllers_tree.setParameters(controllers, showTop=False)

contr_widget = pg.LayoutWidget()
contr_widget.addWidget(controllers_tree)
controllers_dock.addWidget(contr_widget)

ctrlbtn = QtGui.QPushButton('Update Controllers')
contr_widget.addWidget(ctrlbtn, row=1, col=0)
ctrlbtn.clicked.connect(update_controllers)

ctrlsavebtn = QtGui.QPushButton('Save Controllers')
contr_widget.addWidget(ctrlsavebtn, row=2, col=0)
ctrlsavebtn.clicked.connect(save_controllers)

## Start Qt event loop unless running in interactive mode or using pyside.
if __name__ == '__main__':
    import sys
    if (sys.flags.interactive != 1) or not hasattr(QtCore, 'PYQT_VERSION'):
        QtGui.QApplication.instance().exec_()


import torch

def make_subtiles(x: torch.Tensor, h: int, w: int) -> torch.Tensor:
    """ Reshape a 3D or 4D tensor into a new 4D with the dimension 0 used to index the subtiles (and batch)
    Inputs:
     x: tensor to split into subtiles of size h, w. Size of x: (C,H,W) or (N,C,H,W)
     h: int indicating the height of the subtile
     w: int indicating the width of the subtile to use

    Output:
     Torch tensor of shape (N,C,H,W) with N = the total number of subtiles created.
    """
    if x.dim() == 3:
        x = x.unsqueeze(0)
    elif x.dim() != 4:
        raise ValueError(f'Input dimension must be either 3 or 4. {x.dim} dimensions provided')

    b, c, m, n = x.shape

    if m // h != m / h:
        raise ValueError(f'Required subtile size of {h} for tensor size of {m}, does not create an exact number of subtiles. Use padding or change subtile size.')
    elif n // w != n / w:
        raise ValueError(f'Required subtile size of {w} for tensor size of {n}, does not create an exact number of subtiles. Use padding or change subtile size.')

    return x.reshape(b, c, m // h, h, n // w, w).permute(2, 4, 0, 1, 3, 5).reshape(-1, c, h, w)

def unmake_subtiles(x: torch.Tensor, m: int, n: int) -> torch.Tensor:
    """ Reshape the subtiles created by make_subtiles into a tensor of the original size

    Inputs:
     x: torch tensor with 4D with structure (N,C,H,W)
     m: int of the origial height of the image (height to recreate)
     n: into f the original width of the image (width to recreate)
    """
    if x.dim() == 3:
        x = x.unsqueeze(0)
    elif x.dim() != 4:
        raise ValueError(f'Input dimension must be either 3 or 4. {x.dim} dimensions provided')

    c, h, w = x.shape[-3:]
    b = x.shape[0] // ((m // h) * (n // w))

    if m // h != m / h:
        raise ValueError(f'Required larger torch of size {m} while subtiles are of size {h} this does not work.')
    elif n // w != n / w:
        raise ValueError(f'Required larger torch of size {n} while subtiles are of size {w} this does not work.')

    return x.reshape(m // h, n // w, b, c, h, w).permute(2, 3, 0, 4, 1, 5).reshape(b, c, m, n)


from typing import List
from direction import Direction
from orientation import Orientation
from pos import Pos

class Brick:

    def __init__(self, pos: Pos):
        self.pos = pos

    def next_pos(self, direction: Direction) -> Pos:
        """
        Find and return next position in the given direction.
        This function does NOT check for the validity of the move.
        :param direction: Direction of the move (left, right, up, down)
        :return: Brick's next position/orientation in the given direction.
        """

        # nextpos based on current position params.
        nextpos = Pos(self.pos.x, self.pos.y, self.pos.orientation)

        # Update nextpos position object params based on current orientation and the direction of next move.
        # In a Pos object:
        # x value always points to the left block for horizontal lying brick.
        # y value always points to the upper block for vertical lying brick.
        if self.pos.orientation is Orientation.STANDING:
            if direction is Direction.UP:
                nextpos.y -= 2
                nextpos.orientation = Orientation.VERTICAL_LYING
            if direction is Direction.DOWN:
                nextpos.y += 1
                nextpos.orientation = Orientation.VERTICAL_LYING
            if direction is Direction.LEFT:
                nextpos.x -= 2
                nextpos.orientation = Orientation.HORIZONTAL_LYING
            if direction is Direction.RIGHT:
                nextpos.x += 1
                nextpos.orientation = Orientation.HORIZONTAL_LYING

        elif self.pos.orientation is Orientation.HORIZONTAL_LYING:
            if direction is Direction.UP:
                nextpos.y -= 1
            if direction is Direction.DOWN:
                nextpos.y += 1
            if direction is Direction.LEFT:
                nextpos.x -= 1
                nextpos.orientation = Orientation.STANDING
            if direction is Direction.RIGHT:
                nextpos.x += 2
                nextpos.orientation = Orientation.STANDING

        elif self.pos.orientation is Orientation.VERTICAL_LYING:
            if direction is Direction.UP:
                nextpos.y -= 1
                nextpos.orientation = Orientation.STANDING
            if direction is Direction.DOWN:
                nextpos.y += 2
                nextpos.orientation = Orientation.STANDING
            if direction is Direction.LEFT:
                nextpos.x -= 1
            if direction is Direction.RIGHT:
                nextpos.x += 1

        return nextpos

    def move(self, next_pos: Pos):
        """
        Move the brick to the given position.
        :param next_pos: Position object.
        """
        self.pos = next_pos

    def get_blocks_occupied(self) -> List[List[int]]:
        """
        Find the blocks occupied by the brick.
        :return: A list containing 1 or more elements,
         where each list element contains a block's x,y coordinates.
        """

        # occupies 1 block in standing position
        if self.pos.orientation is Orientation.STANDING:
            return [[self.pos.x, self.pos.y]]

        # occupies 2 blocks along y axis in vertical pos.
        if self.pos.orientation is Orientation.VERTICAL_LYING:
            return [[self.pos.x, self.pos.y], [self.pos.x, self.pos.y + 1]]

        # occupies 2 blocks along x axis in horizontal pos.
        if self.pos.orientation is Orientation.HORIZONTAL_LYING:
            return [[self.pos.x, self.pos.y], [self.pos.x + 1, self.pos.y]]


# Copyright 2020-2021 AstroLab Software
# Author: Marco Leoni, Julien Peloton
#
# Licensed under the MIT License;
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      https://opensource.org/licenses/mit-license.php
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import numpy as np

from scipy.optimize import least_squares
from sklearn.metrics import mean_squared_error

__all__ = ['delta_t', 'fsigmoid', 'errfunc_sigmoid', 'fit_sigmoid', 'compute_mse']

def delta_t(time_index: np.array) -> np.array:
    """ Re-index an index relatively to the first data point.

    Parameters
    ----------
    time_index : np.array

    Returns
    -------
    relative_time : np.array
        time relative to the first
        data point in the dataframe
    """

    relative_time = time_index - time_index[0]

    return relative_time

def fsigmoid(x: np.array, a: float, b: float, c: float) -> np.array:
    """Sigmoid function

    Parameters
    ---------
    x: np.array
    a: float
    b: float
    c: float

    Returns
    -------
    sigmoid: np.array
        fit with a sigmoid function
    """

    sigmoid = c / (1.0 + np.exp(-a * (x - b)))

    return np.array(sigmoid)

def errfunc_sigmoid(params: list, time: np.array, flux: np.array) -> float:
    """ Absolute difference between theoretical and measured flux.

    Parameters
    ----------
    params : list of float
        light curve parameters: (a, b, t0, tfall, trise)
    time : array_like
        exploratory variable (time of observation)
    flux : array_like
        response variable (measured flux)

    Returns
    -------
    diff : float
        absolute difference between theoretical and observed flux

    """
    return abs(flux - fsigmoid(time, *params))

def fit_sigmoid(time: np.array, flux: np.array) -> list:
    """ Find best-fit parameters using scipy.least_squares.

    Parameters
    ----------
    time : array_like
        exploratory variable (time of observation)
    flux : array_like
        response variable (measured flux)

    Returns
    -------
    result : list of float
        best fit parameter values
    """
    flux = np.asarray(flux)
    t0 = time[flux.argmax()] - time[0]
    if t0 > 0:
        dt = time[flux.argmax()] - time[flux.argmin()]
        slope = (flux.argmax() - flux.argmin()) / dt
    else:
        slope = 1.
    if flux[0] > 0:
        f0 = flux[0]
    else:
        f0 = 0

    aguess = slope
    cguess = np.max(flux)

    if f0 != 0 and cguess / f0 != 1.:
        bguess = np.log(cguess / f0 - 1.) / aguess
    else:
        bguess = 1.0

    guess = [aguess, bguess, cguess]
    result = least_squares(errfunc_sigmoid, guess, args=(time, flux))

    return result.x

def compute_mse(f_obs: np.array, f_exp: np.array) -> float:
    """ Compute mean squared error.

    Parameters
    ----------
    f_obs: np.array
        observed data points
    f_exp: np.array
        fitted (predicted) data points

    Returns
    -------
    test_mse: float
        mse between fitted and observed
    """

    test_mse = mean_squared_error(f_obs, f_exp)

    return test_mse

def main():
    return None

if __name__ == '__main__':
    main()


from django.shortcuts import get_object_or_404, redirect, render
from taggit.models import Tag
from django.urls import reverse
import json
from django.contrib.auth.models import User
from questions.models import Questions,Comments
from django.utils import timezone

# Create your views here.

def first_login(request):
    if request.user.is_authenticated and request.user.profile.first_login == True:
        if request.method == "POST":
            tags = json.loads(request.POST["tags"])
            print(tags)
            for tag in tags:
                request.user.profile.following.add(tag)
            request.user.profile.first_login = False
            request.user.profile.save()
            return redirect(reverse("home:home"))
        else:
            tags = Tag.objects.all()
            if len(tags)>=50:
                l = list([str(tag) for tag in tags[50:]])
                json_optional_tags = json.dumps(l)
                return render(request,'userProfile/interest.html',{'tags':tags[:50],'json_optional_tags':json_optional_tags})
            return render(request, 'userProfile/interest.html', {'tags': tags})
    else:
        return redirect(reverse("accounts:login"))

def profile(request,username):
    user = get_object_or_404(User,username = username)
    if request.user.username == username:
        if request.method == "POST":
            request.user.profile.bio=request.POST['new-bio-tt']
            request.user.profile.save()
            return redirect(reverse("profile:profile", args=[request.user]))
        else:
            bio = user.profile.bio
            questions = Questions.objects.filter(author = user)
            answers = Comments.objects.filter(cmnt_author = user)
            following = user.profile.following.names()
            content={
                "user":user,
                "bio":bio,
                "questions":questions,
                "answers":answers,
                "following":following,
            }
            return render(request,"userProfile/profile.html",content)
    else:
        bio = user.profile.bio
        questions = Questions.objects.filter(author=user)
        answers = Comments.objects.filter(cmnt_author=user)
        following = user.profile.following.names()
        content = {
            "user": user,
            "bio": bio,
            "questions": questions,
            "answers": answers,
            "following": following,
        }
        return render(request, "userProfile/profile_notuser.html", content)
        #show details only
    #if invalid username return 404


import numpy as np
import pandas as pd
import math
from numpy import mean
from numpy import std
import matplotlib.pyplot as plt
from sklearn.linear_model import LogisticRegression
from sklearn.impute import KNNImputer
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import RepeatedStratifiedKFold
from sklearn.pipeline import Pipeline
from matplotlib import pyplot
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, f1_score, confusion_matrix,  plot_confusion_matrix, classification_report, roc_auc_score, roc_curve

"""
#################################
###-----Data imputation----- ####
#################################
"""
#function to choose the number of k for knn imputation
def choose_k_for_knn_imputation(X, y):
    # evaluate each strategy on the dataset
    results = list()
    strategies = [str(i) for i in [1,3,5,7,9,15,18,21]]
    for s in strategies:
        # create the modeling pipeline
        pipeline = Pipeline(steps=[('scale', StandardScaler()),('i', KNNImputer(n_neighbors=int(s))), ('m', LogisticRegression())])
        # evaluate the model
        cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)
        scores = cross_val_score(pipeline, X, y, scoring='accuracy', cv=cv, n_jobs=-1)
        # store results
        results.append(scores)
        print('>%s %.3f (%.3f)' % (s, mean(scores), std(scores)))
    # plot model performance for comparison
    pyplot.boxplot(results, labels=strategies, showmeans=True)
    pyplot.show()

#imputation using knn
def imputation_with_knn(data, nb):
    #--define imputer
    imputer = KNNImputer(n_neighbors=nb)
    #--fit on the dataset
    imputer.fit(data)
    #--transform the dataset
    imputed_data = imputer.transform(data)
    #--print total missing
    print('Missing values in the imputed dataset: %d' % sum(np.isnan(imputed_data).flatten()))
    #--transform to dataframe
    imputed_data = pd.DataFrame(imputed_data, columns = data.columns)
    return imputed_data

"""
#################################
###-----Data splitting----- ####
#################################
"""
#--split into training and test set
def train_split_test(input_data, y_data, indata_labels):
    X_train, X_test, y_train, y_test = train_test_split(
           input_data, y_data, test_size=0.3,
           stratify=y_data, random_state=0
    )
    print("X_train", X_train.shape)
    print("y_train", y_train.shape)
    print("X_test", X_test.shape)
    print("y_test", y_test.shape)
    #transform the Target data into 'int'/binary
    y_train = y_train.astype("int")
    y_test = y_test.astype("int")
    #from numpy to dataframe
    X_train = pd.DataFrame(X_train, columns =indata_labels)
    X_test = pd.DataFrame(X_test, columns =indata_labels)

    return X_train, X_test, y_train, y_test

"""
#########################################
###-----Evaluation of ML models----- ####
#########################################
"""
#function to compute the evaluation metrics as: accuracy, precision, recall, etc.
def Print_confusion_matrix(cm, auc, heading): #cm for confusion matrix, heading is the title.
    print('\n', heading)
    true_negative  = cm[0,0]
    true_positive  = cm[1,1]
    false_negative = cm[1,0]
    false_positive = cm[0,1]
    total          = true_negative + true_positive + false_negative + false_positive
    accuracy       = (true_positive + true_negative)/total
    precision      = (true_positive)/(true_positive + false_positive)
    recall         = (true_positive)/(true_positive + false_negative)
    misclassification_rate = (false_positive + false_negative)/total
    F1             = (2*true_positive)/(2*true_positive + false_positive + false_negative)
    mcc            = (true_positive*true_negative - false_positive*false_negative) / math.sqrt((true_positive+false_positive)*(true_positive+false_negative)*(true_negative+false_positive)*(true_negative+false_negative))
    print('accuracy.................%7.4f' % accuracy)
    print('precision................%7.4f' % precision)
    print('recall...................%7.4f' % recall)
    print('F1.......................%7.4f' % F1)
    print('auc......................%7.4f' % auc)
    print('mcc......................%7.4f' % mcc)

#function to plot the ROC curve
def plotROC(tpr, fpr, roc_auc, dataset, label=''):
    """
    Plot ROC curve from tpr and fpr.
    """
    lw = 2
    plt.figure(figsize=(5, 5))
    plt.plot(fpr, tpr, color='darkorange',
             lw=lw, label='%s ROC curve (area = %0.2f)' % (label,roc_auc))
    plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Receiver operating characteristic of the %s set' % dataset)
    plt.legend(loc="lower right")
    plt.show()

#function to evaluate the model by plotting the confusion matrix, the classification report and ROC curve of the input dataset.
def evaluate_your_model_metrics(best_model, X_train, y_train, dataset_label):
    y_predicted_train = best_model.predict(X_train)
    probabilities_train = best_model.predict_proba(X_train)
    plot_confusion_matrix(best_model, X_train, y_train)
    plt.show()
    # plot the confusion matrix of the training set
    cm_train = confusion_matrix(y_train, y_predicted_train)
    auc_train = roc_auc_score(y_train, y_predicted_train)
    Print_confusion_matrix(cm_train, auc_train, '***Evaluation of the %s dataset***\n' %dataset_label)
    #plot the ROC curve
    fpr_train, tpr_train, th_train = roc_curve(y_train, probabilities_train[:, 1])
    plotROC(tpr_train, fpr_train, auc_train, '%s' %dataset_label, label='')
    #print the classification report
    c_report = classification_report(y_train, y_predicted_train)
    print('\nClassification report:\n', c_report)