import os
import sys
from functools import partial

from PyQt5.QtGui import *
from PyQt5.QtCore import *
from PyQt5.QtWidgets import *

from toolBar import ToolBar
from Canvas.canvas import Canvas

import cv2
import numpy as np

from grab_cut import Grab_cut
from choiceDiaGen import ChoiceDiaGen
from choiceDiaStyle import ChoiceDiaStyle
from zoomWidget import ZoomWidget
from birdDialog import BirdDialog

from generator import Generator
from styleChanger import StyleChanger

__appname__ = 'grab_cut'

class ResizesQWidget(QWidget):
    def sizeHint(self):
        return QSize(100, 150)

class struct(object):
    def __init__(self, **kwargs):
        self.__dict__.update(kwargs)

# 菜单栏和工具栏
class WindowMixin(object):

    # 根据名字和action列表创建一个菜单，比如File,[new,edit]
    def menu(self, title, actions=None):
        menu = self.menuBar().addMenu(title)
        if actions:
            addActions(menu, actions)
        return menu

    def toolbar(self, title, actions=None):
        toolbar = ToolBar(title)
        toolbar.setObjectName('{}ToolBar'.format(title))
        toolbar.setToolButtonStyle(Qt.ToolButtonTextUnderIcon)
        if actions:
            addActions(toolbar, actions)
        self.addToolBar(Qt.LeftToolBarArea, toolbar)  # 加到布局左侧
        return toolbar

# 创建一个新Action
def newAction(parent, text, slot=None, shortcut=None,
              tip=None, icon=None, checkable=False,
              enable=True):
    a = QAction(text, parent)
    if icon is not None:
        a.setIcon(QIcon(icon))
    if shortcut is not None:
        a.setShortcut(shortcut)
    if tip is not None:
        a.setToolTip(tip)
        a.setStatusTip(tip)
    if slot is not None:
        a.triggered.connect(slot)
    if checkable:
        a.setChecked(True)
    a.setEnabled(enable)
    return a

# 讲actions加入到父控件
def addActions(widget, actions):
    for action in actions:
        if action is None:
            widget.addSeparator()
        widget.addAction(action)  # weidget is toolBar or menu

# 主界面
class MainWindow(QMainWindow, WindowMixin):
    FIT_WINDOW, FIT_WIDTH, MANUAL_ZOOM = list(range(3))

    def __init__(self, defaultFilename=None):
        super().__init__()

        self.dirty = True  # 文件是否已保存
        self.mImgList = []  # 图片列表
        self.dirname = None  # 文件名
        self._beginner = True  #

        self.image_out_np = None  # 提取结果
        self.default_save_dir = None  # 默认保存路径
        self.filePath = None  # 当前载入的图片路径
        self.mattingFile = None

        # 垂直布局，
        listLayout = QVBoxLayout()
        listLayout.setContentsMargins(0, 0, 0, 0)

        # ---#显示图片的label pic
        matResultShow = ResizesQWidget()  # 返回是是Qwidget
        matResultShow.resize(150, 150)

        self.pic = QLabel(matResultShow)
        self.pic.resize(150, 150)
        self.setGeometry(50, 20, 150, 150)

        matResultShow.setLayout(listLayout)

        # 建一个dockwidget放图片label
        self.resultdock = QDockWidget('输出结果', self)
        self.resultdock.setObjectName('result')

        self.resultdock.setWidget(matResultShow)
        self.resultdock.resize(150, 150)
        # self.resultdock.setFeatures(QDockWidget.DockWidgetFloatable)

        # 建一个fileDoc放文件
        self.fileListWidget = QListWidget()  # 列表布局
        self.fileListWidget.itemDoubleClicked.connect(
            self.fileItemDoubleClicked)
        fileListLayout = QVBoxLayout()
        fileListLayout.setContentsMargins(0, 0, 0, 0)
        fileListLayout.addWidget(self.fileListWidget)
        fileListContainer = QWidget()
        fileListContainer.setLayout(fileListLayout)
        self.filedock = QDockWidget('导入文件列表', self)
        self.filedock.setObjectName('Files')
        self.filedock.setWidget(fileListContainer)

        self.zoomWidget = ZoomWidget()

        self.canvas = Canvas(parent=self)
        scroll = QScrollArea()
        scroll.setWidget(self.canvas)
        scroll.setWidgetResizable(True)
        self.scrollBars = {
            Qt.Vertical: scroll.verticalScrollBar(),
            Qt.Horizontal: scroll.horizontalScrollBar()
        }
        self.scrollArea = scroll
        self.canvas.scrollRequest.connect(self.scrollRequest)

        self.setCentralWidget(scroll)
        self.addDockWidget(Qt.RightDockWidgetArea, self.resultdock)
        self.addDockWidget(Qt.RightDockWidgetArea, self.filedock)
        # self.filedock.setFeatures(QDockWidget.DockWidgetFloatable)

        self.dockFeatures = QDockWidget.DockWidgetClosable | QDockWidget.DockWidgetFloatable
        self.resultdock.setFeatures(
            self.resultdock.features() ^ self.dockFeatures)

        # Actions
        action = partial(newAction, self)

        open_file = action('导入图片', self.openFile, 'Ctrl+O', '导入图片')
        open_dir = action('导入文件夹', self.openDir,
                          'Ctrl+D', '导入文件夹中的所有图片到列表')
        change_save_dir = action('&更改预设的保存路径', self.changeSavedirDialog)
        # open_next_img = action('&Next Image', self.openNextImg,
        #                        'Ctrl+N', 'Open next image')
        # open_pre_img = action('&Previous Image', self.openPreImg,
        #                        'Ctrl+M', 'Open previous image')
        save = action('保存结果', self.saveFile, 'Crl+S', '保存输出结果图')
        create = action('指定区域', self.createShape,
                        'w', '框选ROI')
        mark = action('标记微调', self.markDown, None, '左键白色，标记前景；右键黑色，标记后景')
        matting = action('迭代一次', self.grabcutMatting,
                         'e', '用当前标记迭代一次获取前景算法')

        # 用预训练模型生成图片
        generate = action('生成图片', self.generate, None, '输入文字，生成图片素材')

        # 用预训练模型进行风格迁移
        style = action('风格转换', self.styleChange, None, '选择一个风格，进行图像风格转换')
        # 字典，对应一个放缩比
        self.scalers = {
            self.FIT_WINDOW: self.scaleFitWindow,
            self.FIT_WIDTH: self.scaleFitWidth,
            # Set to one to scale to 100% when loading files.
            self.MANUAL_ZOOM: lambda: 1,
        }

        # store actions for further handling
        self.actions = struct(save=save, open_file=open_file,
                              open_dir=open_dir, change_save_dir=change_save_dir,
                              # open_next_img=open_next_img, open_pre_img=open_pre_img,
                              create=create, mark=mark, matting=matting, generate=generate, style=style)

        # Auto saving: enable auto saving if pressing next
        # self.autoSaving = QAction('Auto Saving', self)
        # self.autoSaving.setCheckable(True)
        # self.autoSaving.setChecked()

        # set toolbar
        self.tools = self.toolbar('Tools')
        self.actions.all = (open_file, open_dir,
                            change_save_dir, create,
                            # open_pre_img, open_next_img,
                            mark, matting, generate, style, save)
        addActions(self.tools, self.actions.all)

        # set status
        self.statusBar().showMessage('{} 已就绪.'.format(__appname__))

    def okToContinue(self):
        if self.dirty:
            reply = QMessageBox.question(self, "Attention",
                                         "you have unsaved changes, proceed anyway?",
                                         QMessageBox.Yes | QMessageBox.No | QMessageBox.Cancel)
            if reply == QMessageBox.Cancel:
                return False
            elif reply == QMessageBox.Yes:
                return self.fileSave
        return True

    def resetState(self):
        self.canvas.resetState()

    def errorMessage(self, title, message):
        return QMessageBox.critical(self, title,
                                    '
%s
%s' % (title, message))

    def beginner(self):
        return self._beginner

    def advanced(self):
        return not self.beginner()

    def openFile(self, _value=False):
        path = os.path.dirname(self.filePath) if self.filePath else '.'
        formats = ['*.%s' % fmt.data().decode("ascii").lower()
                   for fmt in QImageReader.supportedImageFormats()]
        filters = "Image (%s)" % ' '.join(formats)
        filename = QFileDialog.getOpenFileName(
            self, '%s - Choose Image or Label file' % __appname__, path, filters)
        if filename:
            if isinstance(filename, (tuple, list)):
                filename = filename[0]
            self.loadFile(filename)

    def openDir(self, dirpath=None):
        defaultOpenDirPath = dirpath if dirpath else '.'
        targetDirPath = QFileDialog.getExistingDirectory(self,
                                                         '%s - Open Directory' % __appname__, defaultOpenDirPath,
                                                         QFileDialog.ShowDirsOnly | QFileDialog.DontResolveSymlinks)
        self.importDirImages(targetDirPath)

    # 将导入图片显示在列表栏
    def importDirImages(self, dirpath):
        self.fileListWidget.clear()
        self.mImgList = self.scanAllImages(dirpath)
        # self.openNextImg()
        for imgPath in self.mImgList:
            item = QListWidgetItem(imgPath)
            self.fileListWidget.addItem(item)

    # 扫描路径下的所有文件，返回图片列表
    def scanAllImages(self, folderPath):
        extensions = ['.%s' % fmt.data().decode("ascii").lower()
                      for fmt in QImageReader.supportedImageFormats()]
        imageList = []

        for root, dirs, files in os.walk(folderPath):
            for file in files:
                if file.lower().endswith(tuple(extensions)):
                    relativePath = os.path.join(root, file)
                    path = os.path.abspath(relativePath)
                    imageList.append(path)
        imageList.sort(key=lambda x: x.lower())
        return imageList

    def fileItemDoubleClicked(self, item=None):
        currIndex = self.mImgList.index(item.text())  # 获取图片列表的index
        if currIndex < len(self.mImgList):
            filename = self.mImgList[currIndex]
            if filename:
                self.loadFile(filename)  # 载入图片列表

    # 读取图片到canvas
    def loadFile(self, filePath=None):
        self.resetState()  # 清理canvas
        self.canvas.setEnabled(False)

        # 高亮选中项
        if filePath and self.fileListWidget.count() > 0:
            index = self.mImgList.index(filePath)
            fileWidgetItem = self.fileListWidget.item(index)
            fileWidgetItem.setSelected(True)

        if filePath and os.path.exists(filePath):
            # load image
            self.ImageData = read(filePath, None)
        else:
            return

        image = QImage.fromData(self.ImageData)
        # 内存中没有图片
        if image.isNull():
            self.errorMessage(u'Error opening file',
                              u'
Make sure %s is a valid image file.' % filePath)
            self.status('Error reading %s' % filePath)
            return False
        self.status('Loaded %s' % os.path.basename(filePath))
        self.image = image  # Qimage格式
        self.filePath = filePath  # 当前载入的文件路径
        self.canvas.loadPixmap(QPixmap.fromImage(image))  # canvas中放置图片
        self.canvas.setEnabled(True)
        self.adjustScale(initial=True)
        self.paintCanvas()

    # 显示当前状态
    def status(self, message, delay=5000):
        self.statusBar().showMessage(message, delay)

    def adjustScale(self, initial=False):
        value = self.scalers[self.FIT_WINDOW if initial else self.zoomMode]()
        self.zoomWidget.setValue(int(100 * value))

    def scaleFitWindow(self):
        """Figure out the size of the pixmap in order to fit the main widget."""
        e = 2.0  # So that no scrollbars are generated.
        w1 = self.centralWidget().width() - e
        h1 = self.centralWidget().height() - e
        a1 = w1 / h1
        # Calculate a new scale value based on the pixmap's aspect ratio.
        w2 = self.canvas.pixmap.width() - 0.0
        h2 = self.canvas.pixmap.height() - 0.0
        a2 = w2 / h2
        return w1 / w2 if a2 >= a1 else h1 / h2

    def scaleFitWidth(self):
        # The epsilon does not seem to work too well here.
        w = self.centralWidget().width() - 2.0
        return w / self.canvas.pixmap.width()

    def paintCanvas(self):
        assert not self.image.isNull(), "cannot paint null image"
        self.canvas.scale = 0.01 * self.zoomWidget.value()
        self.canvas.adjustSize()
        self.canvas.update()

    def createShape(self):
        assert self.beginner()
        self.canvas.setEditing(False)
        self.actions.create.setEnabled(False)

    # 开始标记,mod换成editting
    def markDown(self):
        self.canvas.setEditing(True)

    def toggleDrawMode(self, edit=True):
        self.canvas.setEditing(edit)
        self.actions.createMode.setEnabled(edit)
        self.actions.editMode.setEnabled(not edit)

    # 生成图片
    def generate(self):
        # 种类选择对话框
        choiceDia = ChoiceDiaGen()
        choiceDia.show()
        choiceDia.hide()
        # 由Generator类控制生成对话框
        # 传入类型和属性列表
        gen = Generator(choiceDia.type,choiceDia.attrList)
        gen.generate()

        # 将生成的图片取出来显示在主页
        self.loadFile("StackGAN/resultImg/latest.png")

    def styleChange(self):
        # 风格选择对话框
        choiceDia = ChoiceDiaStyle()
        choiceDia.show()
        choiceDia.hide()
        print(choiceDia.type)
        changer = StyleChanger(choiceDia.type, self.filePath)
        changer.changeStyle()
        # self.loadFile("CycleGAN/targetImg/latest.png")
        result = cv2.imread("CycleGAN/targetImg/latest.png")
        # 转换为四通道
        result = self.addAchannel(result)
        self.showResultImg(result)
        self.image_out_np = result

    # 接收opencv读入的格式
    def addAchannel(self, x):
        b_channel, g_channel, r_channel = cv2.split(x)
        alpha_channel = np.ones(b_channel.shape, dtype=b_channel.dtype) * 255
        result_BGAR = cv2.merge((b_channel, g_channel, r_channel, alpha_channel))
        # result[np.all(result==[0,0,0,255],axis=2)]=[0,0,0,0]
        result_BGAR[np.all(result_BGAR == [0, 0, 0, 255], axis=2)] = [0, 0, 0, 0]
        return result_BGAR

    # 提取前景操作
    def grabcutMatting(self):

        if self.mattingFile is None:
            self.mattingFile = Grab_cut()

        def format_shape(s):
            return dict(line_color=s.line_color.getRgb(),
                        fill_color=s.fill_color.getRgb(),
                        points=[(p.x(), p.y()) for p in s.points],
                        backMark=self.canvas.getBackMark(),
                        whiteMark=self.canvas.getForMark())

        # 有四个点（矩形的话）+填充线颜色和边缘线颜色
        shape = format_shape(self.canvas.shapes[-1])
        self.image_out_np = self.mattingFile.image_matting(self.filePath,
                                                           shape, iteration=10)
        self.showResultImg(self.image_out_np)
        self.actions.save.setEnabled(True)

    # 接收opencv矩阵格式
    def showResultImg(self, image_np):
        # resize to pic
        # factor = min(self.pic.width() /
        #              image_np.shape[1], self.pic.height() / image_np.shape[0])
        # image_np = cv2.resize(image_np, None, fx=factor,
        #                       fy=factor, interpolation=cv2.INTER_AREA)
        # image_np = cv2.resize((self.pic.height(), self.pic.width()))
        image = QImage(image_np, image_np.shape[1],
                       image_np.shape[0], QImage.Format_ARGB32)
        matImg = QPixmap(image)
        self.pic.setFixedSize(matImg.size())
        self.pic.setPixmap(matImg)

    def saveFile(self):
        self._saveFile(self.saveFileDialog())

    def _saveFile(self, saved_path):
        print(saved_path)
        if saved_path:
            Grab_cut.resultSave(saved_path, self.image_out_np)
            self.setClean()
            self.statusBar().showMessage('Saved to  %s' % saved_path)
            self.statusBar().show()

    def saveFileDialog(self):
        caption = '%s - Choose File' % __appname__
        filters = 'File (*%s)' % 'png'
        if self.default_save_dir is not None and len(self.default_save_dir):
            openDialogPath = self.default_save_dir
        else:
            openDialogPath = self.currentPath()

        print(openDialogPath)
        dlg = QFileDialog(self, caption, openDialogPath, filters)
        dlg.setDefaultSuffix('png')
        dlg.setAcceptMode(QFileDialog.AcceptSave)
        filenameWithoutExtension = os.path.splitext(self.filePath)[0]
        dlg.selectFile(filenameWithoutExtension)
        dlg.setOption(QFileDialog.DontUseNativeDialog, False)
        if dlg.exec_():
            return dlg.selectedFiles()[0]
        return ''

    def currentPath(self):
        return os.path.dirname(self.filePath) if self.filePath else '.'

    def changeSavedirDialog(self, _value=False):
        if self.default_save_dir is not None:
            path = self.default_save_dir
        else:
            path = '.'

        dirpath = QFileDialog.getExistingDirectory(self,
                                                   '%s - Save annotations to the directory' % __appname__, path,
                                                   QFileDialog.ShowDirsOnly
                                                   | QFileDialog.DontResolveSymlinks)

        if dirpath is not None and len(dirpath) > 1:
            self.default_save_dir = dirpath

        self.statusBar().showMessage('%s . Annotation will be saved to %s' %
                                     ('Change saved folder', self.default_save_dir))
        self.statusBar().show()

    def setClean(self):
        self.dirty = False
        self.actions.save.setEnabled(False)

        self.actions.create.setEnabled(True)

    def openNextImg(self):
        pass

    def openPreImg(self):
        pass

    def scrollRequest(self, delta, orientation):
        units = - delta / (8 * 15)
        bar = self.scrollBars[orientation]
        bar.setValue(bar.value() + bar.singleStep() * units)

# 读取二进制图片 返回
def read(filename, default=None):
    try:
        with open(filename, 'rb') as f:
            return f.read()
    except Exception:
        return default

    def resetState(self):
        self.canvas.resetState()

if __name__ == "__main__":
    app = QApplication(sys.argv)
    ex = MainWindow()
    ex.show()

    sys.exit(app.exec_())


import time
from datetime import datetime
from bluepy.btle import BTLEDisconnectError
from miband import miband
from ibmcloudant.cloudant_v1 import CloudantV1
from ibm_cloud_sdk_core.authenticators import IAMAuthenticator
from ibmcloudant.cloudant_v1 import CloudantV1, Document
import os
from dotenv import load_dotenv

# All necessary imports

load_dotenv()

SERVICE_URL = os.getenv("SERVICE_URL")
API_KEY = os.getenv("API_KEY")
AUTH_KEY = os.getenv("AUTH_KEY")
MAC_ADDR = os.getenv("MAC_ADDR")

AUTH_KEY = bytes.fromhex(AUTH_KEY)
alternate = True

authenticator = IAMAuthenticator(API_KEY)

client = CloudantV1(authenticator=authenticator)

client.set_service_url(SERVICE_URL)

# All private keys loaded from .env file

def general_info():  # Prints general info about the band
    global band
    print("MiBand-4")
    print("Soft revision:", band.get_revision())
    print("Hardware revision:", band.get_hrdw_revision())
    print("Serial:", band.get_serial())
    print("Battery:", band.get_battery_info()["level"])
    print("Time:", band.get_current_time()["date"].isoformat())

# function to create connection and band object ;-;
def create_connection():
    success = False
    while not success:
        try:
            band = miband(MAC_ADDR, AUTH_KEY, debug=True)
            success = band.initialize()
            return band
        except BTLEDisconnectError:
            print("Connection to the MIBand failed. Trying out again in 3 seconds")
            time.sleep(3)
            continue
        except KeyboardInterrupt:
            print("\nExit.")
            exit()

band = create_connection()
general_info()

hr_list = {}
count = 0

def get_realtime():
    try:
        band.start_heart_rate_realtime(heart_measure_callback=heart_logger)
    except KeyboardInterrupt:
        print("\nExit.")

def heart_logger(data):  # data is the heart rate value
    data = abs(data)
    global count  # global variable to count the number of heart rate values
    print("Realtime heart BPM:", data)  # print the heart rate value
    hr_list[
        datetime.now().strftime("%d/%m/%y %H:%M:%S")
    ] = data  # add the heart rate value to the dictionary
    print(len(hr_list) // 2)
    if count % 3 == 0:  # Using every 10th heart rate value to create a new document
        time_ = str(datetime.now().strftime("%d/%m/%y %H:%M:%S"))
        data_entry: Document = Document(id=time_)

        # Add "add heart rate reading as value" field to the document
        data_entry.value = data

        # Save the document in the database
        create_document_response = client.post_document(
            db="jxtin", document=data_entry
        ).get_result()

        print(
            f"You have created the document:\n{data_entry}"
        )  # print the document that was created
        print("Logged the data")
    else:
        print("Didnt log the data")
    count += 1

get_realtime()


from rest_framework import serializers

from api.v1.auth.schemas import LanguageChoiceField, TimeZoneNameChoiceField
from users.models import User

class CurrentUserOutputSchema(serializers.ModelSerializer):
    language_code = LanguageChoiceField()
    time_zone = TimeZoneNameChoiceField()

    class Meta:
        model = User
        fields = (
            "id",
            "email",
            "full_name",
            "notification_token",
            "language_code",
            "time_zone",
            "date_joined",
            "is_staff",
            "is_superuser",
        )


import pytest
import responses

from repositories.app import APP

@pytest.fixture
def client():
    with APP.test_client() as client:
        APP.extensions["cache"].clear()
        yield client

@responses.activate
def test_get_repo(client):
    url = f"https://api.github.com/repos/owner/repo"
    response = {
        "full_name": "test/name",
        "description": "description",
        "clone_url": "clone url",
        "stargazers_count": 500,
        "created_at": "2020-01-17T22:24:45Z",
    }
    responses.add(responses.GET, url, json=response)

    r = client.get("/repositories/owner/repo")

    assert r.get_json() == {
        "fullName": "test/name",
        "description": "description",
        "cloneUrl": "clone url",
        "stars": 500,
        "createdAt": "2020-01-17T22:24:45+00:00",
    }
    assert r.status_code == 200
    assert r.is_json

@responses.activate
def test_404_error(client):
    url = f"https://api.github.com/repos/owner/repo"
    responses.add(responses.GET, url, status=404)

    r = client.get("/repositories/owner/repo")

    assert r.get_json() == {
        "status": 404,
        "error": "Not Found",
        "message": "requested repository does not exist",
    }
    assert r.status_code == 404

@responses.activate
def test_500_error(client):
    url = f"https://api.github.com/repos/owner/repo"
    responses.add(responses.GET, url, status=500)

    r = client.get("/repositories/owner/repo")

    assert r.get_json() == {
        "status": 500,
        "error": "Internal Server Error",
        "message": "the server encountered an unexpected internal server error",
    }
    assert r.status_code == 500


import matplotlib.pyplot as plt
import numpy as np
x=np.arange(-10,10,0.01)
y=1/(np.sin(x)+2)
z=1/(np.cos(x)+2)

plt.plot(x,y,x,z)                             #生成在一张图像上

fig2,(axs1,axs2)=plt.subplots(2,1)            #分配两个坐标轴并且按照(2,1)的形状
axs1.plot(x,y)
axs2.plot(x,z)                                #在两个轴上单独生成一次
plt.show()


from openpyxl import load_workbook, Workbook
from openpyxl.formatting.rule import ColorScaleRule
from openpyxl.styles import PatternFill, Font

def _cal_writer_final_report(barcode, ws_report, all_data, init_row, init_col, report_output):

    row_counter = init_row

    ws_report.cell(column=-1 + init_col, row=row_counter, value=barcode).font = Font(b=True, underline="single")
    row_counter += 1
    for plate_analysed in all_data["calculations"]:
        # Removing other calculations than avg and stdev
        if plate_analysed != "other_data":
            # Checks to see if the overview of avg and stv should be included
            if report_output[plate_analysed]["overview"]:
                # Writes the analysed method in, if the overview is set to true
                ws_report.cell(column=-1 + init_col, row=row_counter, value=plate_analysed).font = Font(b=True)
                # row_counter += 1
                for state in all_data["calculations"][plate_analysed]:
                    if report_output[plate_analysed][state]:
                        ws_report.cell(column=init_col, row=row_counter, value=state).font = Font(b=True)
                        for calc in all_data["calculations"][plate_analysed][state]:
                            # Writes avg and stdev including values
                            ws_report.cell(column=init_col + 1, row=row_counter, value=calc)
                            ws_report.cell(column=init_col + 2, row=row_counter,
                                           value=all_data["calculations"][plate_analysed][state][calc])
                            row_counter += 1
        else:
            if report_output["z_prime"]:
                ws_report.cell(column=init_col, row=row_counter,
                               value="z-Prime").font = Font(b=True)
                try:
                    ws_report.cell(column=init_col + 2, row=row_counter,
                                   value=all_data["calculations"][plate_analysed]["z_prime"])
                except KeyError:
                    ws_report.cell(column=init_col + 2, row=row_counter,
                                   value="Z-Prime is not calculated for the plates")
                row_counter += 1
            row_counter += 1
    return ws_report, row_counter

def _well_writer_final_report(ws, hits, final_report_setup, init_row):
    indent_col = 1
    row_counter = init_row

    for barcode in hits:
        # Writes headline for data inserts to see where the data is coming from
        ws.cell(column=indent_col, row=row_counter, value=barcode).font = Font(b=True, underline="single")
        row_counter += 1

        for method in hits[barcode]:
            if final_report_setup["methods"][method]:
                # writes method
                ws.cell(column=indent_col, row=row_counter, value=method).font = Font(b=True)
                row_counter += 1
                for split in hits[barcode][method]:
                    ws.cell(column=indent_col, row=row_counter, value=split).font = Font(b=True)
                    ws.cell(column=indent_col+1, row=row_counter,
                                   value=final_report_setup["pora_threshold"][split]["min"]).font = \
                        Font(underline="single")
                    ws.cell(column=indent_col+2, row=row_counter,
                                   value=final_report_setup["pora_threshold"][split]["max"]).font = \
                        Font(underline="single")
                    row_counter += 1
                    for well in hits[barcode][method][split]:
                        ws.cell(column=indent_col + 1, row=row_counter, value=well)
                        ws.cell(column=indent_col + 2, row=row_counter,
                                       value=hits[barcode][method][split][well])
                        row_counter += 1
        indent_col += 4
        row_counter = init_row

def _get_data(all_plate_data, final_report_setup):
    data_calc_dict = {}
    temp_hits = {}
    plate_counter = 0
    all_states = []
    all_methods = []

    for barcode in all_plate_data:
        plate_counter += 1
        temp_hits[barcode] = {}
        data_calc_dict[barcode] = {}
        for method in all_plate_data[barcode]["plates"]:
            if method != "other_data":
                if method not in all_methods:
                    all_methods.append(method)
            if final_report_setup["methods"][method]:
                temp_hits[barcode][method] = {"low": {}, "mid": {}, "high": {}}
                for well in all_plate_data[barcode]["plates"][method]["wells"]:
                    if well in all_plate_data[barcode]["plates"][method]["sample"]:
                        for split in final_report_setup["pora_threshold"]:
                            temp_well_value = all_plate_data[barcode]["plates"][method]["wells"][well]
                            if float(final_report_setup["pora_threshold"][split]["min"]) < float(temp_well_value) < \
                                    float(final_report_setup["pora_threshold"][split]["max"]):
                                temp_hits[barcode][method][split][well] = temp_well_value

        for method in all_plate_data[barcode]["calculations"]:
            data_calc_dict[barcode][method] = {}
            if method != "other_data":
                for state in all_plate_data[barcode]["calculations"][method]:
                    if state not in all_states:
                        all_states.append(state)

                    data_calc_dict[barcode][method][state] = {}
                    for calc in all_plate_data[barcode]["calculations"][method][state]:
                        data_calc_dict[barcode][method][state][calc] = \
                            all_plate_data[barcode]["calculations"][method][state][calc]

            else:
                for other_calc in all_plate_data[barcode]["calculations"][method]:
                    data_calc_dict[barcode][method][other_calc] = \
                        all_plate_data[barcode]["calculations"][method][other_calc]

    return temp_hits, data_calc_dict, plate_counter, all_states, all_methods

def _ws_creator(wb, name):

    return wb.create_sheet(f"{name}_Matrix")

def _matrix_writer(ws, data_calc_dict, state, plate_counter, all_methods):
    init_row = 2
    init_col = 2
    spacer = 4

    col_stdev = init_col + plate_counter + spacer
    col_counter = init_col + 1
    row_counter = init_row + 1
    col_stdev_counter = col_stdev + 1
    row_offset = init_row

    for method in all_methods:
        temp_avg_list = []
        temp_stdev_list = []
        mw_col = col_counter
        mw_row = row_counter
        mw_col_stdev = col_stdev_counter

        for barcodes in data_calc_dict:
            # Writes Plate names in row and clm for avg
            ws.cell(column=init_col - 1, row=row_counter, value=barcodes).font = Font(b=True)
            ws.cell(column=col_counter, row=row_offset - 1, value=barcodes).font = Font(b=True)

            # Writes Plate names in row and clm for stdev
            ws.cell(column=col_stdev - 1, row=row_counter, value=barcodes).font = Font(b=True)
            ws.cell(column=col_stdev_counter, row=row_offset - 1, value=barcodes).font = Font(b=True)

            for index_method, _ in enumerate(data_calc_dict[barcodes]):

                if index_method == 0:
                    # Writes method for avg
                    ws.cell(column=init_col, row=row_offset - 1, value=method).font = Font(b=True)
                    # Writes method for stdev
                    ws.cell(column=col_stdev, row=row_offset - 1, value=method).font = Font(b=True)
                    if method != "other_data":
                        for calc in data_calc_dict[barcodes][method][state]:
                            temp_value = data_calc_dict[barcodes][method][state][calc]
                            # gets avg values
                            if calc == "avg":
                                ws.cell(column=init_col, row=row_offset, value=calc).font = Font(b=True)
                                ws.cell(column=init_col, row=row_counter, value=temp_value)
                                ws.cell(column=col_counter, row=row_offset, value=temp_value)
                                temp_avg_list.append(temp_value)
                            elif calc == "stdev":
                                ws.cell(column=col_stdev, row=row_offset, value=calc).font = Font(b=True)
                                ws.cell(column=col_stdev, row=row_counter, value=temp_value)
                                ws.cell(column=col_stdev_counter, row=row_offset, value=temp_value)
                                temp_stdev_list.append(temp_value)
            # Sets offset for next loop, for writing headlines the right place
            col_counter += 1
            row_counter += 1
            col_stdev_counter += 1

        # calculate the % difference between avg for each plate
        _matrix_calculator(ws, mw_row, mw_col, temp_avg_list)
        # calculate the % difference between stdev for each plate
        _matrix_calculator(ws, mw_row, mw_col_stdev, temp_stdev_list)
        # makes sure that next loop is writen below the first method. One method per row, with avg and stdev for each.
        col_stdev = init_col + plate_counter + spacer
        col_counter = init_col + 1
        row_counter += spacer
        col_stdev_counter = col_stdev + 1
        row_offset += (plate_counter + spacer)

def _matrix_calculator(ws, row, col, temp_data_list):
    start_row = row
    start_col = col

    for index_x, _ in enumerate(temp_data_list):
        for index_y, _ in enumerate(temp_data_list):
            try:
                temp_value = (float(temp_data_list[index_x]) / float(temp_data_list[index_y])) * 100
            except ZeroDivisionError:
                temp_value = "Na"
            ws.cell(column=start_col + index_x, row=start_row + index_y, value=temp_value)

def _z_prime(ws, data_calc_dict):

    init_row = 2
    init_col = 2

    col_counter = init_col + 1
    row_counter = init_row + 1

    z_prime_list = []

    for barcodes in data_calc_dict:
        # Writes Plate names
        ws.cell(column=init_col-1, row=row_counter, value=barcodes).font = Font(b=True)
        ws.cell(column=col_counter, row=init_row-1, value=barcodes).font = Font(b=True)
        # Writes values for Z-Prime
        z_prime = data_calc_dict[barcodes]["other_data"]["z_prime"]
        ws.cell(column=init_col, row=row_counter, value=z_prime)
        ws.cell(column=col_counter, row=init_row, value=z_prime)
        col_counter += 1
        row_counter += 1
        z_prime_list.append(z_prime)

    col_counter = init_col + 1
    row_counter = init_row + 1

    for index_x, _ in enumerate(z_prime_list):
        for index_y, _ in enumerate(z_prime_list):
            temp_value = (z_prime_list[index_x] / z_prime_list[index_y]) * 100
            ws.cell(column=col_counter + index_x, row=row_counter + index_y, value=temp_value)

def bio_final_report_controller(analyse_method, all_plate_data, output_file, final_report_setup):
    wb = Workbook()
    ws_report = wb.active
    ws_report.title = "Full report"
    ws_well_info = wb.create_sheet("Well Info")
    ws_z_prime = wb.create_sheet("Z-Prime")
    # ws_minimum = wb.create_sheet("Minimum")
    # ws_maximum = wb.create_sheet("Maximum")

    init_row = 2
    init_col = 2
    row = init_row
    col = init_col
    # calc overview:

    for index, barcode in enumerate(all_plate_data):
        ws, row_counter = _cal_writer_final_report(barcode, ws_report, all_plate_data[barcode], row, col,
                                                   final_report_setup["calc"])
        # Writes 5 plates horizontal, before changing rows.
        col += 5

        if index % 5 == 0 and index > 0:
            row += row_counter
            col = init_col

    # gets data:
    temp_hits, data_calc_dict, plate_counter, all_states, all_methods = _get_data(all_plate_data, final_report_setup)

    # write well data
    _well_writer_final_report(ws_well_info, temp_hits, final_report_setup, init_row)

    # writes Matrix of data:
    # inside guard ! ! ! !
    print(all_states)
    for states in all_states:
        if final_report_setup["full_report_matrix"][states]:
            _matrix_writer(_ws_creator(wb, states), data_calc_dict, states, plate_counter, all_methods)

    # writes Z-prime
    if final_report_setup["full_report_matrix"]["z_prime"]:
        _z_prime(ws_z_prime, data_calc_dict)

    wb.save(output_file)


##### Native libraries

##### Python Libraries
import numpy as np
from IPython.core import debugger
breakpoint = debugger.set_trace
##### Local libraries
import Utils_Data
from Timer import Timer

##### NOTE: To download the full dataset (which will take about 30 hours on wifi maybe less on ethernet)
##### set the filename_urls to train.npy, set num_labels to 14951, set the
##### for loop iterations to data_urls_train.size
##### Path to datasets
path_urls = '../../data/image_retrieval/image_recognition/'
save_path = path_urls + 'images/'
filename_urls = 'train.npy' # Change this to train.npy to download the full dataset
##### Dataset format parameters
## Number of labels to use out of all the available ones
## For train.npy (max = 14951)
## For train_100.npy (max = 79)
## For train_1000.npy (max = 692)
## For train_10000.npy (max = 3487)
num_labels=50
## Percent of entries to place in train set
train_size=0.9
## Percent of entries to place in test set
test_size=0.1
# breakpoint()
##### Load dataset
dataset = np.load(path_urls+filename_urls)
##### Split dataset in train and test containing the specified number of classes
## The following function returns all entries sorted for both train and test sets.
(data_urls_train, labels_train, imgid_train, data_urls_test, labels_test, imgid_test) = Utils_Data.FormatDataset(dataset, num_labels=num_labels, train_size=train_size, test_size=test_size)

########## DOWNLOAD TRAINING SET ##############
#### UNCOMMENT THE FOLLOWING SNIPPET TO DOWNLOAD THE TRAIN SET
# n_images = data_urls_train.size
# ##### Downloads Train set
# for i in range(0,n_images):
# 	with Timer('Download Image Time'):
# 		print("Image {} out of {}".format(i, n_images))
# 		# image = Utils_Data.DownloadAndSaveImage(url=data_urls_train[i],out_dir=save_path,imgid=imgid_train[i])
# 		image = Utils_Data.DownloadResizeAndSave(url=data_urls_train[i],out_dir=save_path,imgid=imgid_train[i])

########## DOWNLOAD TEST SET ##############
#### UNCOMMENT THE FOLLOWING SNIPPET TO DOWNLOAD THE TEST SET
n_images = data_urls_test.size
##### Downloads Test set
for i in range(0,n_images):
	with Timer('Download Image Time'):
		print("Image {} out of {}".format(i, n_images))
		# image = Utils_Data.DownloadAndSaveImage(url=data_urls_train[i],out_dir=save_path,imgid=imgid_train[i])
		image = Utils_Data.DownloadResizeAndSave(url=data_urls_test[i],out_dir=save_path,imgid=imgid_test[i])


#!/usr/bin/python
# https://www.udemy.com/course/complete-python-developer-zero-to-mastery/

# 256. Building A Flask Server

# https://flask.palletsprojects.com/en/1.1.x/quickstart/

# https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types
# https://swapi.dev/   -  Star Wars API server
# http://www.mashup-template.com/templates.html   - Free HTML templates
# https://html5up.net/   - Free HTML templates
# https://robohash.org/   - Robot generating API

# We need to run:
# $ source ./venv/bin/activate
# $ export FLASK_APP=server.py
# $ export FLASK_ENV=development
# $ flask run

import os
import datetime
import csv
from flask import Flask, render_template, request, send_from_directory, redirect

app = Flask(__name__)

@app.route('/')
def my_home():
    print(render_template('index.html'))
    return render_template('index.html')

@app.route('/<string:page_name>')
def html_page(page_name):
    return render_template(page_name)

# https://flask.palletsprojects.com/en/1.1.x/quickstart/#accessing-request-data

@app.route('/submit_form', methods=['POST', 'GET'])
def submit_form():
    if request.method == 'POST':
        try:
            data=request.form.to_dict()
            write_to_csv(data)
            return redirect('/thankyou.html')
        except:
            return 'did not save to database'
    else:
        return 'something went wrong'

# https://flask.palletsprojects.com/en/1.1.x/patterns/favicon/

@app.route('/favicon.ico')
def favicon():
    return send_from_directory(os.path.join(app.root_path, 'static', 'assets'),
                               'favicon.ico', mimetype='image/vnd.microsoft.icon')

def write_to_file(data):
    """Write message to the database.txt"""
    with open('database.txt', mode='a') as database:
        date = str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
        email = data["email"]
        subject = data["subject"]
        message = data["message"]
        database.write(f'{date},   {email},   {subject},   {message}\n')

def write_to_csv(data):
    """Write message to the database.csv"""
    with open('database.csv', newline='', mode='a') as database:
        date = str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
        email = data["email"]
        subject = data["subject"]
        message = data["message"]
        message_writer = csv.writer(database, delimiter=',',
                 quotechar='"', quoting=csv.QUOTE_NONE)
        message_writer.writerow([date, email, subject, message])

Sec.19 Web Development with Python/portfolio/server.py


# Modules which need to be installed
import irc.bot
from dotenv import load_dotenv
load_dotenv()

# Setup / included imports
import os
import commands
import asyncio
prefix = os.getenv('COMMANDPREFIX')

# Make sure the Twitch credentials have been added to the .env file
if os.getenv('TWITCHUSERNAME') == "" or os.getenv('TWITCHTOKEN') == "":
    print("Please input your Twitch credentials in the .env file.")
    exit(0)

# Login to IRC as the streamer and listen for commands in Twitch Chat
class TwitchListener(irc.bot.SingleServerIRCBot):
    def __init__(self, username, token, channel):
        self.token = token
        self.channel = '#' + channel

        server = 'irc.chat.twitch.tv'
        port = 6667

        # Login to Twitch IRC
        print('Connecting to Twitch IRC: ' + server + ' on port ' + str(port))
        irc.bot.SingleServerIRCBot.__init__(self, [(server, port, token)], username, username)

    # Join this streamer's Twitch Chat channel
    def on_welcome(self, c, e):
        print('Joining ' + self.channel)
        c.join(self.channel)

    # Listen for messages, and if they start with the prefix, try to execute them as commands
    def on_pubmsg(self, c, e):
        if e.arguments[0][:1] == prefix:
            cmd = e.arguments[0].split(' ')[0][1:]
            commands.handleCommand(cmd)
        return

# Load the Twitch login values from the .env file and run the IRC 'bot' above
bot = TwitchListener(str(os.getenv('TWITCHUSERNAME')), str(os.getenv('TWITCHTOKEN')), str(os.getenv('TWITCHUSERNAME')))
bot.start()


import seaborn as sns
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import sklearn
df=pd.read_csv("insurance.csv")

tem=pd.get_dummies(df["region"])

df.drop("region",axis=1,inplace=True)
df=pd.concat([df,tem],axis=1)
print(df.head(10))

map={"yes":1,"no":0}
df["smoker"]=df["smoker"].map(map)
map1={"female":0,"male":1}
df["sex"]=df["sex"].map(map1)
print(df.head(10))
df.corr()
plt.figure(figsize=(20,20))
sns.heatmap(df.corr(),annot=True,cmap="coolwarm",linewidths=2)
plt.show()

x=df["smoker"]
y=df["expenses"]
plt.figure(figsize=(12,9))
plt.scatter(x,y)
plt.xlabel("Non Smoker Vs Smoker")
plt.ylabel("Charges")

Y=df["charges"]
X=df.drop("charges",axis=1)

from sklearn.model_selection import train_test_split
#Splitting the data into 85% for training and 15% for testing
x_train,x_test,y_train,y_test=train_test_split(X,Y,random_state=1,test_size=0.15)
from sklearn.linear_model import LinearRegression
#Training a multiple linear regression model
reg=LinearRegression().fit(x_train,y_train)
y_pred=reg.predict(x_test)

from sklearn.metrics import r2_score
#Checking the R squared error on test data
r2_score(y_test,y_pred)

# Storing independent features in a temporary variable
P_X=X

from sklearn.preprocessing import PolynomialFeatures
#Changing the data to a 3rd degree polynomial
pol=PolynomialFeatures(degree=3)
P_X=pol.fit_transform(X)
P_X
#Training the model similarly but with 3rd degree polynomial of X this time
x_train,x_test,y_train,y_test=train_test_split(P_X,Y,random_state=1,test_size=0.15)
reg=LinearRegression().fit(x_train,y_train)
y_pred=reg.predict(x_test)
r2_score(y_test,y_pred)

#Cross validating the score to check and avoid overfitting
from sklearn.model_selection import cross_val_score
c=cross_val_score(reg,P_X,Y,cv=4)
c

# Final Mean Accuracy
print("Mean accuracy after cross validation is:",c.mean()*100,end="%")


import shutil
import tempfile

from django.conf import settings
from django.contrib.auth import get_user_model
from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import Client, TestCase, override_settings
from django.urls import reverse

from ..forms import PostForm
from ..models import Post

User = get_user_model()
TEMP_MEDIA_ROOT = tempfile.mkdtemp(dir=settings.BASE_DIR)
small_gif = (
    b'\x47\x49\x46\x38\x39\x61\x02\x00'
    b'\x01\x00\x80\x00\x00\x00\x00\x00'
    b'\xFF\xFF\xFF\x21\xF9\x04\x00\x00'
    b'\x00\x00\x00\x2C\x00\x00\x00\x00'
    b'\x02\x00\x01\x00\x00\x02\x02\x0C'
    b'\x0A\x00\x3B'
)

@override_settings(MEDIA_ROOT=TEMP_MEDIA_ROOT)
class PostFormTests(TestCase):
    @classmethod
    def setUpTestData(cls):
        cls.user = User.objects.create_user(username='username')
        cls.uploaded = SimpleUploadedFile(
            name='small.gif',
            content=small_gif,
            content_type='image/gif'
        )
        cls.form = PostForm()

    @classmethod
    def tearDownClass(cls):
        super().tearDownClass()
        shutil.rmtree(TEMP_MEDIA_ROOT, ignore_errors=True)

    def setUp(self):
        self.authorized_client = Client()
        self.authorized_client.force_login(self.user)

    def test_create_post(self):
        """Валидная форма создает запись в Post."""
        # Подсчитаем количество записей в Post
        posts_count = Post.objects.count()
        form_data = {
            'text': 'Тестовый пост',
            'image': PostFormTests.uploaded,
        }
        self.uploaded.seek(0)
        # Отправляем POST-запрос
        response = self.authorized_client.post(
            reverse('posts:post_create'),
            data=form_data,
            follow=True
        )
        # Проверяем, сработал ли редирект
        self.assertRedirects(response, reverse(
            'posts:profile',
            kwargs={'username': 'username'})
        )
        # Проверяем, увеличилось ли число постов
        self.assertEqual(Post.objects.count(), posts_count + 1)
        # Проверяем, что создалась запись с заданным id
        self.assertTrue(
            Post.objects.filter(
                text='Тестовый пост',
                pk=1,
                image='posts/small.gif',
            ).exists()
        )

    def test_edit_post(self):
        Post.objects.create(
            text='Тестовый пост',
            author=self.user,
            pk=1,
            image=self.uploaded,
        )
        form_data = {
            'text': 'Тестовый пост изменился',
        }
        # Отправляем POST-запрос
        response = self.authorized_client.post(
            reverse('posts:post_edit', kwargs={'post_id': 1}),
            data=form_data,
            follow=True
        )
        post_changed = Post.objects.get(pk=1)
        # Проверяем, сработал ли редирект c тем же id
        self.assertRedirects(response, reverse(
            'posts:post_detail',
            kwargs={'post_id': 1})
        )
        self.assertEqual(post_changed.text, 'Тестовый пост изменился')

class CommentFormTests(TestCase):
    @classmethod
    def setUpTestData(cls):
        cls.user = User.objects.create_user(username='username')
        cls.guest_client = Client()
        cls.authorized_client = Client()
        cls.authorized_client.force_login(cls.user)

        Post.objects.create(
            text='Тестовый пост',
            author=cls.user,
            pk=1,
        )

    def test_add_comment(self):
        """Комментировать посты может только авторизованный пользователь."""
        form_data = {
            'text': 'Тестовый комментарий',
        }
        response1 = self.authorized_client.post(
            reverse('posts:add_comment', kwargs={'post_id': 1}),
            data=form_data,
            follow=True
        )
        response2 = self.guest_client.post(
            reverse('posts:add_comment', kwargs={'post_id': 1}),
            data=form_data,
            follow=True
        )
        # Проверяем, сработал ли редирект
        self.assertRedirects(response1, reverse(
            'posts:post_detail',
            kwargs={'post_id': 1})
        )
        self.assertRedirects(response2,
                             '/auth/login/?next=/posts/1/comment/'
                             )


from django.conf.urls import url
from testuser import views

app_name = 'test'

urlpatterns = [
    # url(r'^$',views.logout, name = 'logout'),
    url(r'^$',views.loginIndex, name = 'loginIndex'),
    url(r'^login/$',views.login, name = 'login'),
    # url(r'^signUp/$',views.signup, name = 'signup'),
    # url(r'^forgotPass/$',views.forgot, name = 'forgot'),
    # url(r'^login/check/$',views.loginCheck, name = 'logincheck'),
    # url(r'^signUp/check/$',views.signupCheck, name = 'signupcheck'),
]


#!/usr/bin/env python3
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
#       DESCRIPTION:
#
#       CALL SAMPLE:
#			~/data/solarity/sit-raspi/modbus/direct_marketing_interface.py --host_ip '192.168.0.34' --host_mac '00:90:E8:7B:76:9C' -v -t
#
#	REQUIRE
#
#       CALL PARAMETERS:
#               1)
#
#       @author: Philippe Gachoud
#       @creation: 20200408
#       @last modification:
#       @version: 1.0
#       @URL: $URL
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@

# INCLUDES
try:
	import sys
	import os, errno
	sys.path.append(os.path.join(os.path.dirname(__file__), '../lib')) #the way to import directories
	from sit_logger import SitLogger
	from pymodbus.constants import Endian
	from sit_modbus_device import SitModbusDevice #from file_name import ClassName
	from sit_modbus_register import SitModbusRegister
	from inverter_manager import InverterManager
	#import sitmodbus#, SitModbusRegister
	import logging # http://www.onlamp.com/pub/a/python/2005/06/02/logging.html
	from logging import handlers
	import argparse
	#sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'pysunspec'))
	from datetime import datetime, date, time, timedelta
except ImportError as l_err:
	print("ImportError: {0}".format(l_err))
	raise l_err

class DirectMarketerInterface(InverterManager):

# CONSTANTS

	DEFAULT_SLAVE_ADDRESS = 200

# CLASS ATTRIBUTES

# FUNCTIONS DEFINITION

	"""
		Initialize
	"""
	def __init__(self, a_slave_address=DEFAULT_SLAVE_ADDRESS):
		try:
			self.init_arg_parse()
			l_slave_address = self.DEFAULT_SLAVE_ADDRESS
			if self._args.slave_address:
				if self.valid_slave_address(self._args.slave_address):
					self._slave_address = int(self._args.slave_address)
			#self, a_slave_address=DEFAULT_SLAVE_ADDRESS, a_port=DEFAULT_MODBUS_PORT, an_ip_address=None
			super().__init__(l_slave_address, a_port=self.DEFAULT_MODBUS_PORT, an_ip_address=self._args.host_ip)
			self._logger = SitLogger().new_logger(__name__, self._args.host_mac)
			self._init_sit_modbus_registers()
			#self._logger.debug('init->' + self.out())
		except OSError as l_e:
			self._logger.warning("init-> OSError, probably rollingfileAppender" % (l_e))
			if e.errno != errno.ENOENT:
				raise l_e
		except Exception as l_e:
			print('Error in init: %s' % (l_e))
			raise l_e
			#exit(1)

	def _init_sit_modbus_registers(self):
		"""
			Initializes self._sit_modbus_registers
		"""
		#	P.44 of doc
		self.add_modbus_register('OutLimitPerc', 'Specified output limitation through direct marketer n% (0-10000)', 1, SitModbusRegister.REGISTER_TYPE_INT_16, SitModbusRegister.ACCESS_MODE_RW, 'uint16')
		self.add_modbus_register('OutLimitPercMan', 'Manual output limitation that has been set via Sunspec Modbus', 2, SitModbusRegister.REGISTER_TYPE_INT_16, SitModbusRegister.ACCESS_MODE_R, 'uint16')
		self.add_modbus_register('OutLimitPercIoBox', 'Output limitation through the electric utility company that has been set via the IO box.', 3, SitModbusRegister.REGISTER_TYPE_INT_16, SitModbusRegister.ACCESS_MODE_R, 'uint16')
		self.add_modbus_register('OutLimitMin', 'Minimum of all output limitations. The nominal PV system power is derated to this value.', 4, SitModbusRegister.REGISTER_TYPE_INT_16, SitModbusRegister.ACCESS_MODE_R, 'uint16')
#		self.add_modbus_register('Md', 'Model (Md): SMA Inverter Manager', 40021, SitModbusRegister.REGISTER_TYPE_STRING_16, SitModbusRegister.ACCESS_MODE_R, 'String16')
#		self.add_modbus_register('Opt', 'Options (Opt): Inverter Manager name', 40037, SitModbusRegister.REGISTER_TYPE_STRING_8, SitModbusRegister.ACCESS_MODE_R, 'String8')
#		self.add_modbus_register('Vr', 'Version (Vr): Version number of the installed firmware', 40045, SitModbusRegister.REGISTER_TYPE_STRING_8, SitModbusRegister.ACCESS_MODE_R, 'String8')
#		self.add_modbus_register('SN', 'Serial number (SN) of the device that uses the Modbus unit ID', 40053, SitModbusRegister.REGISTER_TYPE_STRING_16, SitModbusRegister.ACCESS_MODE_R, 'String16')
#		self.add_modbus_register('PPVphA', 'Voltage, line conductor L1 to N (PPVphA), in V-V_SF (40199): average value of all inverters', 40196, SitModbusRegister.REGISTER_TYPE_INT_16, SitModbusRegister.ACCESS_MODE_R, 'V', 40199)
#		self.add_modbus_register('AC_A', 'AC Current sum of all inverters', 40188, SitModbusRegister.REGISTER_TYPE_INT_16, SitModbusRegister.ACCESS_MODE_R, 'A', 40192)
#		self.add_modbus_register('W', 'Active power (W), in W-W_SF (40201): sum of all inverters', 40200, SitModbusRegister.REGISTER_TYPE_INT_16, SitModbusRegister.ACCESS_MODE_R, 'W', 40192)
#		self.add_modbus_register('WH', 'Total yield (WH), in Wh WH_SF (40212): sum of all inverters', 40210, SitModbusRegister.REGISTER_TYPE_INT_32, SitModbusRegister.ACCESS_MODE_R, 'WH', 40212)
#		self.add_modbus_register('TmpCab', 'Internal temperature, in °C Tmp_SF (40223): average value of all inverters', 40219, SitModbusRegister.REGISTER_TYPE_INT_16, SitModbusRegister.ACCESS_MODE_R, '°C', 40223)
#		self.add_modbus_register('ID', 'Model ID (ID): 120 = Sunspec nameplate model', 40238, SitModbusRegister.REGISTER_TYPE_INT_16, SitModbusRegister.ACCESS_MODE_R, 'uint16')
#		self.add_modbus_register('VArPct_Mod', 'Mode of the percentile reactive power limitation: 1 = in % of WMax', 40365, SitModbusRegister.REGISTER_TYPE_ENUM_16, SitModbusRegister.ACCESS_MODE_R, 'enum16')
#		self.add_modbus_register('VArPct_Ena', 'Control of the percentile reactive power limitation,(SMA: Qext): 1 = activated', 40365, SitModbusRegister.REGISTER_TYPE_ENUM_16, SitModbusRegister.ACCESS_MODE_RW, 'enum16')

	def init_arg_parse(self):
		"""
			Parsing arguments
		"""
		self._parser = argparse.ArgumentParser(description='Actions with Inverter Manager through TCP')
		self._parser.add_argument('-v', '--verbose', help='increase output verbosity', action="store_true")
		self._parser.add_argument('-t', '--test', help='Runs test method', action="store_true")
		self._parser.add_argument('-u', '--slave_address', help='Slave address of modbus device', nargs='?')

		#self._parser.add_argument('-u', '--base_url', help='NOT_IMPLEMENTED:Gives the base URL for requests actions', nargs='?', default=self.DEFAULT_BASE_URL)
		l_required_named = self._parser.add_argument_group('required named arguments')
		l_required_named.add_argument('-i', '--host_ip', help='Host IP', nargs='?', required=True)
		l_required_named.add_argument('-m', '--host_mac', help='Host MAC', nargs='?', required=True)
#		l_required_named.add_argument('-l', '--longitude', help='Longitude coordinate (beware timezone is set to Chile)', nargs='?', required=True)
#		l_required_named.add_argument('-a', '--lattitude', help='Lattitude coordinate (beware timezone is set to Chile)', nargs='?', required=True)
#		l_required_named.add_argument('-d', '--device_type', help='Device Type:' + ('|'.join(str(l) for l in self.DEVICE_TYPES_ARRAY)), nargs='?', required=True)
		l_args = self._parser.parse_args()
		self._args = l_args

# ACCESS

# IMPLEMENTATION

# EXECUTE ARGS

	"""
		Parsing arguments and calling corresponding functions
	"""
	def execute_corresponding_args(self):
		if self._args.verbose:
			self._logger.setLevel(logging.DEBUG)
		else:
			self._logger.setLevel(logging.DEBUG)
		if self._args.test:
			self.test()

		#if self._args.store_values:

	"""
		Test function
	"""
	def test(self):
		try:
			self.connect()
			self.read_all_sit_modbus_registers()
			print ("################# BEGIN #################")
#			self._logger.info("--> ************* device models *************: %s" % (l_d.models))	 #Lists properties to be loaded with l_d.<property>.read() and then access them
#			self._logger.info("-->inverter ************* l_d.inverter.points *************: %s" % (l_d.inverter.points))	#Gives the inverter available properties
#			self._logger.info("-->inverter ************* common *************: %s" % (l_d.common))
#			self._logger.info("-->inverter ************* common Serial Number *************: %s" % (l_d.common.SN))
			print ("################# END #################")
		except Exception as l_e:
			self._logger.exception("Exception occured: %s" % (l_e))
			print('Error: %s' % (l_e))
			self._logger.error('Error: %s' % (l_e))
			raise l_e
		finally:
			self.disconnect()

"""
Main method
"""
def main():
	#logging.basicConfig(level=logging.DEBUG, stream=sys.stdout, filemode="a+", format="%(asctime)-15s %(levelname)-8s %(message)s")
	logger = logging.getLogger(__name__)

	try:
		l_obj = DirectMarketerInterface()
		l_obj.execute_corresponding_args()
#		l_id.test()
		pass
	except KeyboardInterrupt:
		logger.exception("Keyboard interruption")
	except Exception:
		logger.exception("Exception occured")
	finally:
		logger.info("Main method end -- end of script")

if __name__ == '__main__':
    main()


from django.urls import path

from . import views

urlpatterns=[
    path('sub/',views.SubjectVW,name='sub'),
    path('trainer/',views.TrainerVW,name='trainer'),
    path('profile/',views.TranierDisplay,name='profile'),
    path('batchvw/',views.BatchVW,name='batchvw'),
    path('bdisplay/',views.BatchDisplay,name='bdisplay'),
    path('trainerupdate/<pk>/',views.TrainerUP,name='trainerupdate'),
    path('Home/',views.Home,name='Home'),
]


from pytesseract import Output
import pytesseract
import argparse
import imutils
import cv2

ap = argparse.ArgumentParser()
ap.add_argument("-i", "--image", required=True, help="path to input image")
ap.add_argument("-o", "--output", required=False, help="path to output image. override if not given.")
ap.add_argument("-a", "--angle", required=True, help="rotation angle", type=int)
args = vars(ap.parse_args())

original = cv2.imread(args["image"])

if original is None:
    exit("Thats not an image =(")

# rotate the image and save to disk
angle = args["angle"]
rotated = imutils.rotate_bound(original, angle=args["angle"])
output = args.get("output")
if output:
    text = f"Saving rotated image (by {angle} degrees) into: {output}"
else:
    output = args["image"]
    text = f"Overwriting rotated image (by {angle} degrees) into: {output}"
print(text)
cv2.imwrite(output, rotated)


"""Module containing the Fetcher Class to get financial
 data of given ticker using the yfinance package."""

from datetime import datetime, timedelta
import yfinance as yf
import pandas as pd

class Fetcher:
    """Class that fetches data about a given ticker

    This class does a few things:
    1. Checks for validity of arguments before instantiating
    2. Pulls data and checks for whether it contains > 1 day.

    Attributes:
        ticker: String containing the inputted ticker name
        start_date: Beginning day of retrieved financial data
        end_date: Final day of retrieved financial data
    """
    def __init__(self, args: dict) -> None:
        fetcher_args = self._check_args_validity(args)
        self._ticker = fetcher_args["ticker"]
        self._start_date = fetcher_args["start_date"]
        self._end_date = fetcher_args["end_date"]

    def _check_args_validity(self, args: dict) -> dict:
        """Checks for the validity of the CLI arguments

        This function checks for the validity of the input arguments
        before initializing the class. Otherwise, it throws an exception

        Args:
            args: dictionary containing the input CLI Arguments

        Returns:
            dictionary of parsed arguments to be used in yfinance
        """
        # Check for possible ticker errors
        ticker = args["ticker"]

        # Datetime automatically checks for datetime argument errors
        start_date = datetime.strptime(args["b"], "%Y%m%d")
        end_date = datetime.strptime(args["e"], "%Y%m%d") \
            if args["e"] is not None else datetime.now()

        # Compensate for yfinance bug, more specifically:
        # API Given Tracks until 1 day before end
        end_date += timedelta(days = 1)

        # Start date cannot be later than current date
        if start_date > datetime.now():
            raise ValueError("Start date cannot be after current time")

        # Start date cannot be later than the ending date
        if start_date > end_date:
            raise ValueError("End Date is earlier than Start Date")

        fetcher_args = {
            "ticker": ticker,
            "start_date": start_date,
            "end_date": end_date
        }
        return fetcher_args

    def fetch_data(self) -> pd.DataFrame:
        """Function that fetches data from yfinance.

        After checking, it checks for the data validity before proceeding.

        Returns:
            Dataframe with the columns representing financial data
            of the given ticker, arranged from earliest to latest date.
        """
        tracker = yf.Ticker(self._ticker)
        try:
            data: pd.DataFrame = tracker.history(
                start = self._start_date,
                end = self._end_date
            )[self._start_date : self._end_date]
            if len(data) == 0:
                raise Exception("No data available for given ticker.")
            if len(data) == 1:
                raise Exception("Only 1 data point seen. Check time period.")
            return data
        # Error can be caused as raw date is converted to seconds (Line 150):
        # https://github.com/ranaroussi/yfinance/blob/main/yfinance/base.py
        # Best solution is to try a date that's more recent, within 50 years
        except OverflowError as err:
            raise ValueError(
                "Start date too distant. Try a start date within 50 years."
                ) from err
        except BaseException as err:
            raise err


from django.contrib import admin, messages
from .models import Poll, Question, Choice

class ChoiceInline(admin.StackedInline):
    model = Choice
    extra = 0

class QuestionInline(admin.StackedInline):
    model = Question
    readonly_fields = ['question_type']
    extra = 0

class PollAdmin(admin.ModelAdmin):

    inlines = [QuestionInline]
    fieldsets = [
        (None, {'fields': ['name', 'slug', 'status', 'description']}),
        ('DATE INFO', {'fields': [('starting', 'finished')]})
    ]
    prepopulated_fields = {'slug': ('name',)}
    readonly_fields = [
        'starting',
        'finished',
    ]

    def save_model(self, request, obj, form, change):
        """Save Model override for access control of the poll"""
        import datetime

        def get_message(msg, type):
            messages.add_message(
                request,
                type,
                f'{msg}!'
            )

        if not obj.starting and obj.status == 'IN_PROGRESS':
            obj.starting = datetime.datetime.now()
            get_message('Poll has started!', messages.SUCCESS)
            obj.save()

        if obj.starting and not obj.finished and obj.status == 'FINISHED':
            obj.finished = datetime.datetime.now()
            get_message('Poll has finished!', messages.SUCCESS)
            obj.save()

        if not (obj.starting or obj.finished) and obj.status != 'WAITING':
            obj.status = 'WAITING'
            get_message('Woo Wee Woo Waa! Error!', messages.ERROR)
            obj.save()

        if not obj.id:
            obj.save()

class QuestionAdmin(admin.ModelAdmin):

    inlines = [ChoiceInline]

    # def save_model(self, request, obj, form, change):
    # When Admin choose type of the question is text, answer choices are removing
    #    choices = Choice.objects.filter(question=obj)
    #    if obj.question_type == '1' and choices:
    #       choices.delete()
    #   obj.save()

admin.site.register(Poll, PollAdmin)
admin.site.register(Question, QuestionAdmin)
admin.site.register(Choice)


from flask import Flask, render_template, request
from tensorflow.keras.layers import Dense, Embedding, Bidirectional, LSTM, Concatenate, Dropout
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras import Input, Model
import gensim
import numpy as np
import BahdanauAttention #모델.py 불러오기
from konlpy.tag import Mecab
import pickle
import tensorflow as tf
import re

lstm_model = BahdanauAttention.BahdanauAttention(64)

app = Flask(__name__)
app.config['JSON_AS_ASCII'] = False #한글 깨짐 현상
wv_model = gensim.models.Word2Vec.load('model/aihub_review_6.model')
mecab = Mecab(dicpath=r"C:\mecab\mecab-ko-dic") #mecab 윈도우에서 설정
tokenizer = pickle.load(open('model/tokenizer.pickle','rb'))

############ 모델 부분
max_len = 100
EMBEDDING_DIM = 100
sequence_input = Input(shape=(max_len,), dtype='int32')

VOCAB_SIZE = len(tokenizer.index_word) + 1
EMBEDDING_DIM = 100
embedding_matrix = np.zeros((VOCAB_SIZE, EMBEDDING_DIM))

# tokenizer에 있는 단어 사전을 순회하면서 word2vec의 100차원 vector를 가져옵니다
for word, idx in tokenizer.word_index.items():
    embedding_vector = wv_model[word] if word in wv_model else None
    if embedding_vector is not None:
        embedding_matrix[idx] = embedding_vector

embedded_sequences = Embedding(VOCAB_SIZE,
                               EMBEDDING_DIM,
                               input_length=max_len,
                               weights=[embedding_matrix],  # weight는 바로 위의 embedding_matrix 대입
                               trainable=False  # embedding layer에 대한 train은 꼭 false로 지정
                               )(sequence_input)

# embedded_sequences = Embedding(vocab_size, 128, input_length=max_len, mask_zero = True)(sequence_input)
lstm = Bidirectional(LSTM(64, dropout=0.5, return_sequences=True))(embedded_sequences)
lstm, forward_h, forward_c, backward_h, backward_c = Bidirectional(
    LSTM(64, dropout=0.5, return_sequences=True, return_state=True))(lstm)

state_h = Concatenate()([forward_h, backward_h])  # 은닉 상태
state_c = Concatenate()([forward_c, backward_c])  # 셀 상태
attention = lstm_model # 가중치 크기 정의
context_vector, attention_weights = attention(lstm, state_h)

dense1 = Dense(20, activation="relu")(context_vector)
dropout = Dropout(0.5)(dense1)
output = Dense(1, activation="sigmoid")(dropout)
model = Model(inputs=sequence_input, outputs=output)
model.load_weights('model/best_model.h5')
stopwords = ['도', '는', '다', '의', '가', '이', '은', '한', '에', '하', '고', '을', '를', '인', '듯', '과', '와', '네', '들', '듯', '지', '임', '게', '만', '게임', '겜', '되', '음', '면']

def sentiment_predict(new_sentence):
  new_sentence = re.sub(r'[^ㄱ-ㅎㅏ-ㅣ가-힣 ]','', new_sentence)
  new_sentence = mecab.morphs(new_sentence) # 토큰화
  new_sentence = [word for word in new_sentence] # 불용어 제거
  encoded = tokenizer.texts_to_sequences([new_sentence]) # 정수 인코딩
  pad_new = pad_sequences(encoded, maxlen = max_len,padding='post') # 패딩
  score = float(model.predict(pad_new)) # 예측
  return round(score, 2)
  # if(score > 0.5):
  #   print("{:.2f}% 확률로 욕설에 가깝습니다.".format(score * 100))
  # else:
  #   print("{:.2f}% 확률로 욕설이 아닙니다.".format((1 - score) * 100))

@app.route('/', methods=['GET','POST'])
def test():
    return render_template('user.html')

@app.route('/post', methods=['GET','POST'])
def post():
    original_test = request.form['test']
    score = sentiment_predict(original_test)

    return render_template('post.html',  score=score)

@app.route('/ajax_model', methods=['GET','POST'])
def ajax_model():
    original_test = request.json['send_data']
    score = sentiment_predict(original_test)
    return str(score*100)

if __name__ == '__main__':
    app.run()


### This file has been adopted from
### https://github.com/openlawlibrary/pygls/blob/master/examples/json-extension/server/server.py

import asyncio
from bisect import bisect

from cromwell_tools import api as cromwell_api
from cromwell_tools.cromwell_auth import CromwellAuth
from cromwell_tools.utilities import download

from functools import wraps

from pygls.features import (
    CODE_ACTION,
    DEFINITION,
    REFERENCES,
    TEXT_DOCUMENT_DID_OPEN,
    TEXT_DOCUMENT_DID_CHANGE,
    TEXT_DOCUMENT_DID_SAVE,
    TEXT_DOCUMENT_WILL_SAVE,
    WORKSPACE_DID_CHANGE_CONFIGURATION,
    WORKSPACE_DID_CHANGE_WATCHED_FILES,
)
from pygls.server import LanguageServer
from pygls.types import (
    CodeActionParams,
    ConfigurationItem,
    ConfigurationParams,
    Diagnostic,
    DiagnosticSeverity,
    DidChangeConfigurationParams,
    DidOpenTextDocumentParams,
    DidChangeTextDocumentParams,
    DidSaveTextDocumentParams,
    WillSaveTextDocumentParams,
    TextDocumentPositionParams,
    DidChangeWatchedFiles,
    FileChangeType,
    MessageType,
    Location,
    Position,
    Range,
)

from os import environ, name as platform, pathsep
from pathlib import Path
import re, sys
from requests import HTTPError
from threading import Timer
from time import sleep
from typing import Callable, Dict, Iterable, List, Set, Tuple, Union
from urllib.parse import urlparse

import WDL
from WDL import SourceNode, SourcePosition, Lint

PARSE_DELAY_SEC = 0.5 # delay parsing of WDL until no more keystrokes are sent

class Server(LanguageServer):
    SERVER_NAME = 'wdl'
    CONFIG_SECTION = SERVER_NAME

    CMD_RUN_WDL = SERVER_NAME + '.run'

    def __init__(self):
        super().__init__()
        self.wdl_paths: Dict[str, Set[str]] = dict()
        self.wdl_types: Dict[str, Dict[str, SourcePosition]] = dict()
        self.wdl_defs: Dict[str, Dict[SourcePosition, SourcePosition]] = dict()
        self.wdl_refs: Dict[str, Dict[SourcePosition, List[SourcePosition]]] = dict()
        self.wdl_symbols: Dict[str, List[SourcePosition]] = dict()
        self.aborting_workflows: Set[str] = set()

    def catch_error(self, log = False):
        def decorator(func: Callable):
            @wraps(func)
            def wrapper(*args, **kwargs):
                try:
                    return func(*args, **kwargs)
                except Exception as e:
                    if log:
                        self.show_message_log(str(e), MessageType.Error)
                    else:
                        self.show_message(str(e), MessageType.Error)
            return wrapper
        return decorator

server = Server()

def _get_client_config(ls: Server):
    config = ls.get_configuration(ConfigurationParams([
        ConfigurationItem(section=Server.CONFIG_SECTION)
    ])).result()
    return config[0]

# https://gist.github.com/walkermatt/2871026
def debounce(delay_sec: float, id_arg: Union[int, str]):
    """ Decorator that will postpone a functions
        execution until after wait seconds
        have elapsed since the last time it was invoked. """
    def decorator(func: Callable):
        @wraps(func)
        def debounced(*args, **kwargs):
            if not hasattr(debounced, 'timers'):
                debounced.timers: Dict[str, Timer] = dict()
            id = args[id_arg] if isinstance(id_arg, int) else kwargs[id_arg]
            if id in debounced.timers:
                debounced.timers[id].cancel()
            timer = Timer(delay_sec, lambda: func(*args, **kwargs))
            debounced.timers[id] = timer
            timer.start()
        return debounced
    return decorator

@debounce(PARSE_DELAY_SEC, 1)
def parse_wdl(ls: Server, uri: str):
    ls.show_message_log('Validating ' + uri, MessageType.Info)
    diagnostics, wdl = _parse_wdl(ls, uri)
    ls.publish_diagnostics(uri, diagnostics)
    ls.show_message_log(
        '{} {}'.format('Valid' if wdl else 'Invalid', uri),
        MessageType.Info if wdl else MessageType.Warning
    )

def _parse_wdl(ls: Server, uri: str):
    try:
        paths = _get_wdl_paths(ls, uri)
        doc = asyncio.run(
            WDL.load_async(uri, path=paths, read_source=_read_source(ls))
        )

        types = _get_types(doc.children, dict())
        ls.wdl_types[uri] = types
        ls.wdl_defs[uri], ls.wdl_refs[uri] = _get_links(doc.children, types, dict(), dict())
        ls.wdl_symbols[uri] = sorted(_get_symbols(doc.children, []))

        return list(_lint_wdl(ls, doc)), doc

    except WDL.Error.MultipleValidationErrors as errs:
        return [_diagnostic_err(e) for e in errs.exceptions], None

    except WDLError as e:
        return [_diagnostic_err(e)], None

    except Exception as e:
        ls.show_message_log(str(e), MessageType.Error)
        return [], None

def _read_source(ls: Server):
    async def read_source(uri: str, path, importer):
        uri = await WDL.resolve_file_import(uri, path, importer)
        if uri.startswith('/'):
            uri = 'file://' + uri
        source = ls.workspace.get_document(uri).source
        return WDL.ReadSourceResult(source_text=source, abspath=uri)
    return read_source

def _get_symbols(nodes: Iterable[SourceNode], symbols: List[SourcePosition]):
    for node in nodes:
        symbols.append(node.pos)
        _get_symbols(node.children, symbols)
    return symbols

# find SourcePosition as the minimum bounding box for cursor Position
def _find_symbol(ls: Server, uri: str, p: Position):
    if uri not in ls.wdl_symbols:
        return
    symbols = ls.wdl_symbols[uri]

    best_score = (sys.maxsize, sys.maxsize)
    best_sym: SourcePosition = None

    line = p.line + 1
    col = p.character + 1

    min_pos = SourcePosition(uri, uri, line, 0, line, 0)
    i = bisect(symbols, min_pos)

    while i < len(symbols):
        sym = symbols[i]
        if sym.line > line or (sym.line == line and sym.column > col):
            break
        elif sym.end_line > line or (sym.end_line == line and sym.end_column >= col):
            score = (sym.end_line - sym.line, sym.end_column - sym.column)
            if score <= best_score:
                best_score = score
                best_sym = sym
        i += 1
    return best_sym

def _get_types(nodes: Iterable[SourceNode], types: Dict[str, SourcePosition]):
    for node in nodes:
        if isinstance(node, WDL.StructTypeDef):
            types[node.type_id] = node.pos
        _get_types(node.children, types)
    return types

def _get_links(
    nodes: Iterable[SourceNode],
    types: Dict[str, SourcePosition],
    defs: Dict[SourcePosition, SourcePosition],
    refs: Dict[SourcePosition, List[SourcePosition]],
):
    for node in nodes:
        source: SourcePosition = None
        if isinstance(node, WDL.Call):
            source = node.callee.pos
        elif isinstance(node, WDL.Decl) and isinstance(node.type, WDL.Type.StructInstance):
            source = types[node.type.type_id]
        elif isinstance(node, WDL.Expr.Ident):
            ref = node.referee
            if isinstance(ref, WDL.Tree.Gather):
                source = ref.final_referee.pos
            else:
                source = ref.pos
        if source is not None:
            defs[node.pos] = source
            refs.setdefault(source, []).append(node.pos)
        _get_links(node.children, types, defs, refs)
    return defs, refs

SourceLinks = Union[SourcePosition, List[SourcePosition]]

def _find_links(ls: Server, uri: str, pos: Position, links: Dict[str, Dict[SourcePosition, SourceLinks]]):
    symbol = _find_symbol(ls, uri, pos)
    if (symbol is None) or (uri not in links):
        return
    symbols = links[uri]
    if symbol in symbols:
        return symbols[symbol]

def _find_def(ls: Server, uri: str, pos: Position):
    link = _find_links(ls, uri, pos, ls.wdl_defs)
    if link is not None:
        return Location(link.abspath, _get_range(link))

def _find_refs(ls: Server, uri: str, pos: Position):
    links = _find_links(ls, uri, pos, ls.wdl_refs)
    if links is not None:
        return [Location(link.abspath, _get_range(link)) for link in links]

def _lint_wdl(ls: Server, doc: WDL.Document):
    _check_linter_path()
    warnings = Lint.collect(Lint.lint(doc, descend_imports=False))
    _check_linter_available(ls)
    for pos, _, msg, _ in warnings:
        yield _diagnostic(msg, pos, DiagnosticSeverity.Warning)

def _check_linter_path():
    if getattr(_check_linter_path, 'skip', False):
        return

    LOCAL_BIN = '/usr/local/bin'
    PATH = environ['PATH'].split(pathsep)

    if platform == 'posix' and LOCAL_BIN not in PATH:
        environ['PATH'] = pathsep.join([LOCAL_BIN] + PATH)
    _check_linter_path.skip = True

def _check_linter_available(ls: Server):
    if getattr(_check_linter_available, 'skip', False):
         return

    if not Lint._shellcheck_available:
        ls.show_message('''
            WDL task command linter is not available on the system PATH.
            Please install ShellCheck and/or add it to the PATH:
            https://github.com/koalaman/shellcheck#installing
        ''', MessageType.Warning)
    _check_linter_available.skip = True

def _get_wdl_paths(ls: Server, wdl_uri: str, reuse_paths = True) -> List[str]:
    ws = ls.workspace
    if ws.folders:
        ws_uris = [f for f in ws.folders if wdl_uri.startswith(f)]
    elif ws.root_uri:
        ws_uris = [ws.root_uri]
    else:
        ws_uris = []
    wdl_paths: Set[str] = set()
    for ws_uri in ws_uris:
        if reuse_paths and (ws_uri in ls.wdl_paths):
            ws_paths = ls.wdl_paths[ws_uri]
        else:
            ws_paths: Set[str] = set()
            ws_root = Path(urlparse(ws_uri).path)
            for p in ws_root.rglob('*.wdl'):
                ws_paths.add(str(p.parent))
            ls.wdl_paths[ws_uri] = ws_paths
        wdl_paths.update(ws_paths)
    return list(wdl_paths)

WDLError = (WDL.Error.ImportError, WDL.Error.SyntaxError, WDL.Error.ValidationError)

def _diagnostic(msg: str, pos: SourcePosition = None, severity = DiagnosticSeverity.Error):
    return Diagnostic(_get_range(pos), msg, severity=severity)

def _get_range(p: SourcePosition = None):
    if p is None:
        return Range(
            Position(),
            Position(0, sys.maxsize),
        )
    else:
        return Range(
            Position(p.line - 1, p.column - 1),
            Position(p.end_line - 1, p.end_column - 1),
        )

def _diagnostic_err(e: WDLError):
    cause = ': {}'.format(e.__cause__) if e.__cause__ else ''
    msg = str(e) + cause
    return _diagnostic(msg, e.pos)

@server.thread()
@server.feature(TEXT_DOCUMENT_DID_OPEN)
@server.catch_error()
def did_open(ls: Server, params: DidOpenTextDocumentParams):
    parse_wdl(ls, params.textDocument.uri)

@server.thread()
@server.feature(TEXT_DOCUMENT_DID_CHANGE)
@server.catch_error()
def did_change(ls: Server, params: DidChangeTextDocumentParams):
    parse_wdl(ls, params.textDocument.uri)

@server.thread()
@server.feature(TEXT_DOCUMENT_DID_SAVE)
@server.catch_error()
def did_save(ls: Server, params: DidSaveTextDocumentParams):
    pass

@server.thread()
@server.feature(TEXT_DOCUMENT_WILL_SAVE)
@server.catch_error()
def will_save(ls: Server, params: WillSaveTextDocumentParams):
    pass

@server.feature(WORKSPACE_DID_CHANGE_CONFIGURATION)
def did_change_configuration(ls: Server, params: DidChangeConfigurationParams):
    pass

@server.thread()
@server.feature(WORKSPACE_DID_CHANGE_WATCHED_FILES)
@server.catch_error()
def did_change_watched_files(ls: Server, params: DidChangeWatchedFiles):
    for change in params.changes:
        if  change.type in [FileChangeType.Created, FileChangeType.Deleted] and \
            change.uri.endswith('.wdl'):
            _get_wdl_paths(ls, change.uri, reuse_paths=False)

@server.thread()
@server.feature(DEFINITION)
@server.catch_error()
def goto_definition(ls: Server, params: TextDocumentPositionParams):
    return _find_def(ls, params.textDocument.uri, params.position)

@server.thread()
@server.feature(REFERENCES)
@server.catch_error()
def find_references(ls: Server, params: TextDocumentPositionParams):
    return _find_refs(ls, params.textDocument.uri, params.position)

class RunWDLParams:
    def __init__(self, wdl_uri: str):
        self.wdl_uri = wdl_uri

@server.feature(CODE_ACTION)
@server.catch_error()
def code_action(ls: Server, params: CodeActionParams):
    return [{
        'title': 'Run WDL',
        'kind': Server.CMD_RUN_WDL,
        'command': {
            'command': Server.CMD_RUN_WDL,
            'arguments': [RunWDLParams(params.textDocument.uri)],
        },
    }]

@server.thread()
@server.command(Server.CMD_RUN_WDL)
@server.catch_error()
def run_wdl(ls: Server, params: Tuple[RunWDLParams]):
    wdl_uri = params[0].wdl_uri
    wdl_path = urlparse(wdl_uri).path

    _, wdl = _parse_wdl(ls, wdl_uri)
    if not wdl:
        return ls.show_message('Unable to submit: WDL contains error(s)', MessageType.Error)

    config = _get_client_config(ls)
    auth = CromwellAuth.from_no_authentication(config.cromwell.url)
    workflow = cromwell_api.submit(
        auth, wdl_path, raise_for_status=True,
    ).json()
    id = workflow['id']

    title = 'Workflow {} for {}'.format(id, wdl_path)
    _progress(ls, 'start', {
        'id': id,
        'title': title,
        'cancellable': True,
        'message': workflow['status'],
    })

    status: str = ''
    while True:
        if status != workflow['status']:
            status = workflow['status']
            if status == 'Succeeded':
                message_type = MessageType.Info
            elif status in ('Aborting', 'Aborted'):
                message_type = MessageType.Warning
            elif status == 'Failed':
                message_type = MessageType.Error
            else:
                _progress(ls, 'report', {
                    'id': id,
                    'message': status,
                })
                continue

            _progress(ls, 'done', {
                'id': id,
            })
            message = '{}: {}'.format(title, status)
            ls.show_message(message, message_type)

            diagnostics = _parse_failures(wdl, id, auth)
            return ls.publish_diagnostics(wdl_uri, diagnostics)

        sleep(config.cromwell.pollSec)

        if id in ls.aborting_workflows:
            workflow = cromwell_api.abort(
                id, auth, raise_for_status=True,
            ).json()
            ls.aborting_workflows.remove(id)
            continue

        try:
            workflow = cromwell_api.status(
                id, auth, raise_for_status=True,
            ).json()
        except HTTPError as e:
            ls.show_message_log(str(e), MessageType.Error)

def _progress(ls: Server, action: str, params):
    ls.send_notification('window/progress/' + action, params)

@server.feature('window/progress/cancel')
def abort_workflow(ls: Server, params):
    ls.aborting_workflows.add(params.id)

def _parse_failures(wdl: WDL.Document, id: str, auth: CromwellAuth):
    workflow = cromwell_api.metadata(
        id, auth,
        includeKey=['status', 'executionStatus', 'failures', 'stderr'],
        expandSubWorkflows=True,
        raise_for_status=True,
    ).json()
    if workflow['status'] != 'Failed':
        return

    calls = workflow['calls']
    if calls:
        diagnostics: List[Diagnostic] = []
        elements = wdl.workflow.elements

        for call, attempts in calls.items():
            for attempt in attempts:
                if attempt['executionStatus'] == 'Failed':
                    pos = _find_call(wdl.workflow.elements, wdl.workflow.name, call)
                    failures = _collect_failures(attempt['failures'], [])

                    stderr = _download(attempt['stderr'])
                    if stderr is not None:
                        failures.append(stderr)

                    msg = '\n\n'.join(failures)
                    diagnostics.append(_diagnostic(msg, pos))
        return diagnostics
    else:
        failures = _collect_failures(workflow['failures'], [])
        msg = '\n\n'.join(failures)
        return [_diagnostic(msg)]

class CausedBy:
    def __init__(self, causedBy: List['CausedBy'], message: str):
        self.causedBy = causedBy
        self.message = message

def _collect_failures(causedBy: List[CausedBy], failures: List[str]):
    for failure in causedBy:
        if failure['causedBy']:
            _collect_failures(failure['causedBy'], failures)
        failures.append(failure['message'])
    return failures

WorkflowElements = List[Union[WDL.Decl, WDL.Call, WDL.Scatter, WDL.Conditional]]

def _find_call(elements: WorkflowElements, wf_name: str, call_name: str):
    found: SourcePosition = None
    for el in elements:
        if found:
            break
        elif isinstance(el, WDL.Call) and '{}.{}'.format(wf_name, el.name) == call_name:
            found = el.pos
        elif isinstance(el, WDL.Conditional) or isinstance(el, WDL.Scatter):
            found = _find_call(el.elements, wf_name, call_name)
    return found

@server.catch_error(log=True)
def _download(url: str):
    return str(download(url), 'utf-8')


import torch
import torch.nn as nn
from torch.utils.data import Dataset

import h5py
import numpy as np
import utils.io as io
from datasets.hico_constants import HicoConstants
from datasets import metadata

import sys
import random

class HicoDataset(Dataset):
    '''
    Args:
        subset: ['train', 'val', 'train_val', 'test']
    '''
    data_sample_count = 0   # record how many times to process data sampling

    def __init__(self, data_const=HicoConstants(), subset='train', data_aug=False, sampler=None, test=False):
        super(HicoDataset, self).__init__()

        self.data_aug = data_aug
        self.data_const = data_const
        self.test = test
        self.subset_ids = self._load_subset_ids(subset, sampler)
        self.sub_app_data = self._load_subset_app_data(subset)
        self.sub_spatial_data = self._load_subset_spatial_data(subset)
        self.word2vec = h5py.File(self.data_const.word2vec, 'r')
        self.sub_pose_feat = self._load_subset_pose_data(subset)

    def _load_subset_ids(self, subset, sampler):
        global_ids = io.load_json_object(self.data_const.split_ids_json)
        bad_det_ids = io.load_json_object(self.data_const.bad_faster_rcnn_det_ids)
        # skip bad instance detection image with 0-1 det
        # !NOTE: How to reduce the number of bad instance detection images
        subset_ids = [id for id in global_ids[subset] if id not in bad_det_ids['0']+bad_det_ids["1"]]
        if sampler:
            # import ipdb; ipdb.set_trace()
            ''' when changing the model, use sub-dataset to quickly show if there is something wrong '''
            subset_ids = random.sample(subset_ids, int(len(subset_ids)*sampler))
        return subset_ids

    def _load_subset_app_data(self, subset):
        print(f'Using {self.data_const.feat_type} feature...')
        if subset == 'train' or subset == 'val' or subset == 'train_val':
            return h5py.File(self.data_const.hico_trainval_data, 'r')
        elif subset == 'test':
            return h5py.File(self.data_const.hico_test_data, 'r')
        else:
            print('Please double check the name of subset!!!')
            sys.exit(1)

    def _load_subset_spatial_data(self, subset):
        if subset == 'train' or subset == 'val' or subset == 'train_val':
            return h5py.File(self.data_const.trainval_spatial_feat, 'r')
        elif subset == 'test':
            return h5py.File(self.data_const.test_spatial_feat, 'r')
        else:
            print('Please double check the name of subset!!!')
            sys.exit(1)

    def _load_subset_pose_data(self, subset):
        if subset == 'train' or subset == 'val' or subset == 'train_val':
            return h5py.File(self.data_const.trainval_keypoints_feat, 'r')
        elif subset == 'test':
            return h5py.File(self.data_const.test_keypoints_feat, 'r')
        else:
            print('Please double check the name of subset!!!')
            sys.exit(1)

    def _get_obj_one_hot(self,node_ids):
        num_cand = len(node_ids)
        obj_one_hot = np.zeros([num_cand,80])
        for i, node_id in enumerate(node_ids):
            obj_idx = int(node_id)-1
            obj_one_hot[i,obj_idx] = 1.0
        return obj_one_hot

    def _get_word2vec(self,node_ids):
        word2vec = np.empty((0,300))
        for node_id in node_ids:
            vec = self.word2vec[metadata.coco_classes[node_id]]
            word2vec = np.vstack((word2vec, vec))
        return word2vec

    def _get_interactive_label(self, edge_label):
        interactive_label = np.zeros(edge_label.shape[0])
        interactive_label = interactive_label[:, None]
        valid_idxs = list(set(np.where(edge_label==1)[0]))
        if len(valid_idxs) > 0:
            # import ipdb; ipdb.set_trace()
            interactive_label[valid_idxs,:] = 1
        return interactive_label

    @staticmethod
    def displaycount():
        print("total times to process data sampling:", HicoDataset.data_sample_count)

    # def get_verb_one_hot(self,hoi_ids):
    #     num_cand = len(hoi_ids)
    #     verb_one_hot = np.zeros([num_cand,len(self.verb_to_id)])
    #     for i, hoi_id in enumerate(hoi_ids):
    #         verb_id = self.verb_to_id[self.hoi_dict[hoi_id]['verb']]
    #         verb_idx = int(verb_id)-1
    #         verb_one_hot[i,verb_idx] = 1.0
    #     return verb_one_hot

    def __len__(self):
        return len(self.subset_ids)

    def __getitem__(self, idx):
        global_id = self.subset_ids[idx]

        data = {}
        single_app_data = self.sub_app_data[global_id]
        single_spatial_data = self.sub_spatial_data[global_id]
        single_pose_data = self.sub_pose_feat[str(global_id)]
        data['roi_labels'] = single_app_data['classes'][:]
        data['node_num'] = single_app_data['node_num'].value
        data['edge_labels'] = single_app_data['edge_labels'][:]
        data['features'] = single_app_data['feature'][:]
        data['spatial_feat'] = single_spatial_data[:]
        data['word2vec'] = self._get_word2vec(data['roi_labels'])
        # data['pose_feat'] = single_pose_data[:]
        data['pose_to_human'] = single_pose_data['pose_to_human'][:]
        data['pose_to_obj_offset'] = single_pose_data['pose_to_obj_offset'][:]
        if self.test:
            data['global_id'] = global_id
            data['img_name'] = global_id + '.jpg'
            data['det_boxes'] = single_app_data['boxes'][:]
            data['roi_scores'] = single_app_data['scores'][:]
        # import ipdb; ipdb.set_trace()
        if self.data_aug:
            thresh = random.random()
            if thresh > 0.5:
                data = self._data_sampler(data)
        return data

    # for inference
    def sample_date(self, global_id):
        data = {}
        single_app_data = self.sub_app_data[global_id]
        single_spatial_data = self.sub_spatial_data[global_id]
        single_pose_data = self.sub_pose_feat[str(global_id)]
        data['global_id'] = global_id
        data['img_name'] = global_id + '.jpg'
        data['det_boxes'] = single_app_data['boxes'][:]
        data['roi_labels'] = single_app_data['classes'][:]
        data['roi_scores'] = single_app_data['scores'][:]
        data['node_num'] = single_app_data['node_num'].value
        # data['node_labels'] = single_app_data['node_labels'][:]
        data['edge_labels'] = single_app_data['edge_labels'][:]
        data['features'] = single_app_data['feature'][:]
        data['spatial_feat'] = single_spatial_data[:]
        data['word2vec'] = self._get_word2vec(data['roi_labels'])
        data['pose_to_human'] = single_pose_data['pose_to_human'][:]
        data['pose_to_obj_offset'] = single_pose_data['pose_to_obj_offset'][:]
        data['keypoints'] = single_app_data['keypoints'][:]

        return data
# for DatasetLoader
def collate_fn(batch):
    '''
        Default collate_fn(): https://github.com/pytorch/pytorch/blob/1d53d0756668ce641e4f109200d9c65b003d05fa/torch/utils/data/_utils/collate.py#L43
    '''
    batch_data = {}
    batch_data['global_id'] = []
    batch_data['img_name'] = []
    batch_data['det_boxes'] = []
    batch_data['roi_labels'] = []
    batch_data['roi_scores'] = []
    batch_data['node_num'] = []
    batch_data['edge_labels'] = []
    batch_data['features'] = []
    batch_data['spatial_feat'] = []
    batch_data['word2vec'] = []
    # batch_data['pose_feat'] = []
    batch_data['pose_to_human'] = []
    batch_data['pose_to_obj_offset'] = []
    batch_data['keypoints'] = []
    for data in batch:
        batch_data['roi_labels'].append(data['roi_labels'])
        batch_data['node_num'].append(data['node_num'])
        batch_data['edge_labels'].append(data['edge_labels'])
        batch_data['features'].append(data['features'])
        batch_data['spatial_feat'].append(data['spatial_feat'])
        batch_data['word2vec'].append(data['word2vec'])
        # batch_data["pose_feat"].append(data["pose_feat"])
        batch_data["pose_to_human"].append(data["pose_to_human"])
        batch_data["pose_to_obj_offset"].append(data["pose_to_obj_offset"])
        if 'global_id' in data.keys():
            batch_data['global_id'].append(data['global_id'])
            batch_data['img_name'].append(data['img_name'])
            batch_data['det_boxes'].append(data['det_boxes'])
            batch_data['roi_scores'].append(data['roi_scores'])
        if 'keypoints' in data.keys():
            batch_data['keypoints'].append(data['keypoints'])

    # import ipdb; ipdb.set_trace()
    batch_data['edge_labels'] = torch.FloatTensor(np.concatenate(batch_data['edge_labels'], axis=0))
    batch_data['features'] = torch.FloatTensor(np.concatenate(batch_data['features'], axis=0))
    batch_data['spatial_feat'] = torch.FloatTensor(np.concatenate(batch_data['spatial_feat'], axis=0))
    batch_data['word2vec'] = torch.FloatTensor(np.concatenate(batch_data['word2vec'], axis=0))
    # batch_data['pose_feat'] = torch.FloatTensor(np.concatenate(batch_data['pose_feat'], axis=0))
    batch_data['pose_to_human'] = torch.FloatTensor(np.concatenate(batch_data['pose_to_human'], axis=0))
    batch_data['pose_to_obj_offset'] = torch.FloatTensor(np.concatenate(batch_data['pose_to_obj_offset'], axis=0))

    return batch_data