# views.py
import os
from flask import Flask, request, render_template, flash, redirect, url_for, get_flashed_messages, session, abort
from .forms import LoginForm, RegistrationForm, ShoppingListForm, additemsForm
from . import app
from app.modals import User

@app.route('/', methods= ['GET', 'POST'])
def index():
    if request.method == 'POST':
        if 'user_id' in session:
            return redirect(url_for ('viewitems'))
    else:
        return redirect(url_for ('login'))

@app.route('/signin', methods=['GET', 'POST'])
def login():
    form = LoginForm()
    if request.method == 'POST':
        if form.validate_on_submit():
            if 'user_id' in session:
                return redirect(url_for('viewitems'))
            return redirect(url_for('showsignup'))

        else:
            if form.validate() == False:
                flash('All fields are required.')
                return render_template('signin.html', form = form)

    return render_template('signin.html',
                           title='Sign In',
                           form = form)

@app.route('/signup', methods= ['GET', 'POST'])
def showsignup():

        form = RegistrationForm(request.form)

        if request.method == 'POST':

            if form.validate_on_submit():
                    #return error if email is already registered
                user_dict = User.users.items()
                existing_user = {k:v for k,v in user_dict if form.Email.data in v['email']}
                if existing_user:
                    flash('email exists please log in instead')
                    return render_template('signin.html', form = LoginForm())
                else:
                    #if email doesnot exist register user
                    new_user = User(form.Username.data, form.Email.data, form.Password.data, form.Password2.data)
                    new_user.create_user()
                  #assign a user_id
                #create a session
                    for key, value in user_dict:     # gets id, eg 2
                        if form.Email.data in value['email']:
                            session['user_id'] = key
                            flash('You have successfully signed up')
                            return redirect(url_for('login', form = LoginForm()))

            elif form.errors:
                if form.Password.data != form.Password2.data:
                    flash({"message": "Your passwords don't match!"})
                    return render_template('signup.html', form = form)

        elif request.method == 'GET':
            return render_template('signup.html',
                           title='Sign up',
                           form=form)

@app.route('/create_shoppinglist', methods= ['GET', 'POST'])
def createshoppinglist():
    form = ShoppingListForm()
    if request.method == 'POST':
        if form.validate() == False:
            flash('All fileds requirred')
            return render_template('create_list.html', form = form)
        else:
            flash('You have successfully created a list')
            return redirect('/additem')

    elif request.method == 'GET':
        return render_template('create_shoppinglist.html',
                           title='items',
                           form=form)

@app.route('/additem', methods= ['GET', 'POST'])
def additem():
    if request.method == "GET":
        form = additemsForm(request.form)
        return render_template('additem.html', form = form)
        flash("You have added an item to your shopping list")
    elif request.method == 'POST':
        return redirect('/viewitems')

@app.route('/viewitems', methods= ['GET', 'POST'])
def viewitems():
    form = additemsForm(request.form)
    if request.method == "POST":
        itemname = request.form['itemname']
        Quantity = request.form['Quantity']
        Price = request.form['Price']
        return render_template('viewitems1.html', itemname = itemname, Quantity = Quantity, Price = Price)
    elif request.method == "GET":
        return redirect('/additem')


"""
Crawls through the 'source' source and looks for labels starting with 'rancon'.

If such a label is found on a service, then it will register the service in the
'backend'. If the backend supports tag all services will be tagged 'rancon'.
depending on the backend the registration behavior can be influenced by tags
set on the source (e.g. rancon.name, ...).

Every rancon.* tag will be available as variable "%NAME%" in the backend.
Please look through the documentation to make more sense of this, it is easy
but just a little bit complex because of the flexibility.
"""

import asyncio
import sys
import time

from rancon import settings
from rancon import tools
from .version import __version__

import uvloop
import prometheus_client.exposition
import prometheus_client.core
from prometheus_client import Counter, Gauge, Histogram, Summary
import sanic
from sanic import Sanic
from sanic.response import HTTPResponse, text

asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
app = Sanic(__name__)

@app.route("/")
async def index(request):
    """ returns a hello string """
    return text("rancon v{} :)".format(__version__))

@app.route("/metrics")
async def metrics(request):
    """ returns the latest metrics """
    names = request.args.get("name", None)
    registry = prometheus_client.core.REGISTRY
    if names:
        registry = registry.restricted_registry(names)
    tmp = prometheus_client.exposition.generate_latest(registry)
    return HTTPResponse(body_bytes=tmp)

@app.route("/health")
async def health(request):
    """ returns the current health state of the system """
    duration = time.time() - LAST_CALL_ROUTE_SERVICES

    if duration > settings.args.hangup_detection:
        msg = "system hang-up detected, it's been {} seconds since start of "\
              "route_services".format(int(duration))
        raise sanic.exceptions.ServerError(msg)
    return text("OK")

LAST_CALL_ROUTE_SERVICES = time.time()

# NB: documentation does not fit the implementation here. the parameters here should be
# name, labelnames, labelvalues but documented are name, description, so I'm not sure
# what to use here. Also, it's not clear what labelnames and labelvalues should be if they
# were used and how they could be retrieved.
ROUTE_TIME = Summary('rancon_route_services_seconds',
                     'Number of seconds route_services takes')

METRIC_SERVICES_FOUND = Gauge('rancon_discovered_services',
                              'Number of discovered services')

METRIC_SUCCESSFUL_REGS = Counter('rancon_successful_registrations',
                                 'Number of services registered')

METRIC_FAILED_REGS = Counter('rancon_failed_registrations',
                             'Number of failed service registrations')

METRIC_RAISED_REGS = Counter('rancon_registration_exceptions',
                             'Number of exception during service registrations')

METRIC_SUCCESSFUL_DEREGS = Counter('rancon_successful_deregistrations',
                                   'Number of services deregistered, UNUSED')

METRIC_FAILED_DEREGS = Counter('rancon_failed_deregistrations',
                               'Number of failed service deregistrations, '
                               'UNUSED')

METRIC_VERSION = Gauge('rancon_version',
                         'Rancon version number',
                         ('version',)).labels(__version__).set(1)

@ROUTE_TIME.time()
def route_services(schedule_next=5, loop=None):
    """ checks for services to register and then register them with consul """
    if loop is not None:
        loop.call_later(schedule_next, route_services, schedule_next, loop)
    global LAST_CALL_ROUTE_SERVICES
    LAST_CALL_ROUTE_SERVICES = time.time()

    log = tools.getLogger(__name__)
    backend = settings.backend
    source = settings.source

    services_to_route = source.get_services()
    registered_services = []

    METRIC_SERVICES_FOUND.set(len(services_to_route))
    for service in services_to_route:
            try:
                # I think that will re-raise or pass the exception thorug,
                # so we also need to catch it manually
                with METRIC_RAISED_REGS.count_exceptions():
                    success, routed_service = backend.register(service)
                registered_services.append(routed_service)
            except Exception as e:
                success = False
                log.error("EXCEPTION CAUGHT WHILE REGSITERING '{}': {}"
                          .format(str(service), str(e)))
            if success:
                METRIC_SUCCESSFUL_REGS.inc()
            else:
                METRIC_FAILED_REGS.inc()

    if len(registered_services) == 0:
        log.debug("No services registered (of {} services found)"
                  .format(len(services_to_route)))
    backend.cleanup(registered_services)
    log.debug("Run completed @ {}".format(time.ctime()))

def start(sys_argv):
    # prepare
    settings.parse_params(sys_argv)
    # not before here :)
    log = tools.getLogger(__name__)
    # run
    log.error("Start @ {}".format(time.ctime()))

    if not settings.args.continuous:
        route_services()
        sys.exit(0)

    loop = asyncio.get_event_loop()
    loop.call_soon(route_services, settings.args.wait, loop)
    app.run(host="0.0.0.0", port=8000, loop=loop)
    log.info("Exiting.")

def console_entrypoint():
    start(sys.argv[1:])


import argparse
import os
import pandas as pd
import json

from pandas.core.frame import DataFrame

from fol.foq_v2 import (concate_n_chains, copy_query,
                        negation_sink,
                        binary_formula_iterator,
                        concate_iu_chains,
                        decompose_D,
                        parse_formula,
                        projection_sink, to_D,
                        union_bubble,
                        DeMorgan_replacement,
                        to_d,
                        transformation)

parser = argparse.ArgumentParser()
parser.add_argument("--benchmark_name", type=str, default="benchmark")
parser.add_argument("--input_formula_file", type=str, default="outputs/generated_formula_anchor_node=3.csv")
parser.add_argument("--knowledge_graph", action="append")

def convert_query(df,
                  old_form_name='DNF+MultiIUD',
                  new_form_name='DNF+MultiIUd',
                  convert_functional=decompose_D):
    def convertor(f):
        query_instance = parse_formula(f)
        query_instance = convert_functional(query_instance)
        return query_instance.formula

    df[new_form_name] = df[old_form_name].map(convertor)
    return df

def convert_grounded_query(df,
                           old_form_name='DNF+MultiIUD',
                           new_form_name='DNF+MultiIUd',
                           old_form_formula=None,
                           convert_functional=None):
    assert old_form_formula is not None
    assert convert_functional is not None

    def grounded_convertor(f):
        query_instance = parse_formula(old_form_formula)
        query_instance.additive_ground(json.loads(f))
        query_instance = convert_functional(query_instance)
        return query_instance.dumps

    df[new_form_name] = df[old_form_name].map(grounded_convertor)
    return df

if __name__ == "__main__":
    args = parser.parse_args()
    target_folder = f"data/{args.benchmark_name}"
    formula_file = args.input_formula_file
    df = pd.read_csv(formula_file)
    df = convert_query(df)
    df.to_csv(formula_file, index=False)

    for kg in args.knowledge_graph:
        folder = os.path.join(target_folder, kg)
        for i, row in df.iterrows():
            print(row.formula_id)
            data_file = f"data-{row.formula_id}.csv"
            data_df = pd.read_csv(os.path.join(folder, data_file))
            converted_data_df = convert_grounded_query(
                data_df,
                old_form_formula=row['DNF+MultiIUD'],
                convert_functional=decompose_D)
            converted_data_df.to_csv(os.path.join(folder, data_file), index=False)


# pybutton
# create at 2015/5/28
# autor: qianqians
from tools import argv_instance, tuple_rbg
from pyelement import pyelement

class pybutton(pyelement):
	def __init__(self, text, cname, layout, praframe):
		# when normal
		self.normaltext = text

		self.type = "button"

		super(pybutton, self).__init__(cname, layout, praframe)

	def sub(self, id = None):
		js = " var table_" + self.id + " = document.createElement(\"input\");\n"
		js += " table_" + self.id + ".type = \"button\";\n"
		js += " table_" + self.id + ".value=\"" + self.normaltext + "\";\n"

		js += super(pybutton, self).sub()

		if id:
			js += " " + id + ".appendChild(table_" + self.id + ");\n"
		else:
			js += " table_pop.appendChild(table_" + self.id + ");\n"

		return js

	def flush(self):
		# if img is not none, use img for button,
		# if img is none, use text for button,
		# handle onclick in js and send a requst to service
		# codegen css in page
		shtml = ""

		if self.html is not None:
			shtml = self.html
		else:
			shtml += "
<button " + "id=\"" + self.id + "\"" + " type=\"button\" "

			for event, onevent in self.uievent.iteritems():
				shtml += event + "=\"" + self.id + event + "(this)\" "

			shtml += ">" + self.normaltext + "</button>
"

		js = ""
		if self.js is not None:
			js = self.js
		else:
			for event, onevent in self.uievent.iteritems():
				js += "function " + self.id + event + "(id){"
				js += onevent
				js += "}\n"
		return shtml, js

	def client_set(self, text, textcolor, backcolor):
		js = "id.value=\"" + text + "\";"
		js += "id.style.color=" + tuple_rbg(textcolor) + ";"
		js += "id.style.backgroundColor=" + tuple_rbg(backcolor) + ";"
		return js

	def server_set(self, text, textcolor, backcolor):
		js = "document.getElementById(\"" + self.id + "\").value=\"" + text + "\";\n"
		js += "document.getElementById(\"" + self.id + "\").style.color=" + tuple_rbg(textcolor) + ";\n"
		js += "document.getElementById(\"" + self.id + "\").style.backgroundColor=" + tuple_rbg(backcolor) + ";\n"
		return js


import jieba
from flask import Flask,render_template,request
from peewee import fn
from wordcloud import WordCloud
from src.database import User,Weibo
from src.save import save_user,save_weibo
from src.spider import getuser, getresponse,getweibo
from flask_paginate import get_page_parameter, Pagination
app = Flask(__name__)

@app.route('/',methods=['POST','GET'])
def user():
    # 获取用户数量
    y_sum = User.select().count()

    # 对地点进行分组聚合,{‘地点’：数量}
    locas = Weibo.select(Weibo.location, fn.Count(Weibo.location).alias('count')).group_by(Weibo.location)
    loca_dicts = {}
    for temp in locas:
        loca_dicts[f'{temp.location}'] = temp.count
    loca_y = loca_dicts.keys()
    loca_x = loca_dicts.values()
    l_y = [item for item in loca_y]
    l_x = [item for item in loca_x]

    # 粉丝数排名
    followers = User.select()
    follower_dict = {}
    for f in followers:
        follower_dict[f.name] = f.number_of_followers
    f_name_datas = [f for f in dict(sorted(follower_dict.items(), key=lambda x: x[1], reverse=True)).keys()]
    f_follow_datas = [f for f in dict(sorted(follower_dict.items(), key=lambda x: x[1], reverse=True)).values()]

    # 微博排名
    weibocounts = Weibo.select(Weibo.user_id, fn.Count(Weibo.user_id).alias('count')).group_by(Weibo.user_id)
    weiborank_dict = {}
    for w in weibocounts:
        id = w.user_id
        user = User.select().where(User.id == id).first().name
        weiborank_dict[user] = w.count
    weiborank = [
        [w for w in dict(sorted(weiborank_dict.items(), key=lambda x:x[1], reverse=True)).keys()],
        [w for w in dict(sorted(weiborank_dict.items(), key=lambda x:x[1], reverse=True)).values()]
    ]

    #词云
    # 读取文件
    pd_data = Weibo.select()
    datas = []
    for pd in pd_data:
        datas.append(pd.content)

    # 切割分词
    result = ' '.join(jieba.lcut_for_search(''.join(datas)))
    # 设置停用词
    stop_words = ['@', '#', '，', '“', '：', '/', '_', '我', '微博', '的', '视频', '了', '是', '你', '们', '在']
    ciyun_words = ''

    for word in result:
        if word not in stop_words:
            ciyun_words += word

    # 设置参数，创建WordCloud对象
    wc = WordCloud(
        font_path='msyh.ttc',  # 中文
        background_color='white',  # 设置背景颜色为白色
        stopwords=stop_words,  # 设置禁用词，在生成的词云中不会出现set集合中的词
        height=500,
        width=700,
    )
    # 根据文本数据生成词云
    wc.generate(ciyun_words)
    # 保存词云文件
    wc.to_file('./static/img/word.png')

    # 传递的数据
    datas = {
        'count': Weibo.select().count(),
        'follow': User.select().where(User.number_of_followers),
        'loca_': [l_x, list(l_y)],
        'followrank': [f_name_datas,f_follow_datas],
        'weiborank': weiborank,
        'y_sum': y_sum,
    }

    if request.method == 'POST':
        # 获取在页面输入的ID，进行操作
        uid = request.form.get('uid')
        # if uid != '':
        if User.select().where(User.id == uid):
            print('你已经监控过改用户！')
        else:
            response = getresponse(uid)
            try:
                user_data = getuser(response, uid)
                save_user(user_data)
            except Exception as e:
                print('预计uid有问题！',e)

    return render_template('dashboard.html',datas=datas)

@app.route('/tables',methods=['POST','GET'])
def tables():
    user_count = User.select()
    # 更新操作
    if request.method == 'POST':
        flag = request.form['button']
        if flag == 'true':
            try:
                for u in user_count:
                    response = getresponse(u.id)
                    try:
                        datalist = getweibo(response, u.id)
                        save_weibo(datalist, u.id)
                    except Exception as e:
                        print('my error', e)
            except Exception as e:
                print('更新有问题', e)
        else:
            print('no')
    # 分页
    page = request.args.get(get_page_parameter(), type=int, default=1)
    per_page = 5
    pagination = Pagination(page=page, per_page=per_page, total=len(user_count), css_framework='bootstrap4')
    start = (page - 1) * per_page
    end = start + per_page
    userdata = user_count[start:end]
    return render_template('tables.html',userdata=userdata, pagination=pagination)

if __name__ == '__main__':
    app.run(debug=True)


import unittest
from hamcrest import assert_that, instance_of, equal_to, raises, calling
from app.exiftool.s3.object import Object
from app.exiftool.s3.object_iterator import ObjectIterator
from tests.config import Config as TestConfig

class TestS3ObjectIterator(unittest.TestCase):
    def setUp(self) -> None:
        self.config = TestConfig()
        self.config.patcher.environment()
        self.addCleanup(self.config.patcher.stop())

    def test_next_success(self) -> None:
        with self.config.use_cassette('s3_object_iterator_succeeded_pdf.yml'):
            object_iterator = ObjectIterator(
                Object('test-bucket', 'test/test.pdf')
            )

            assert_that(
                object_iterator.__next__(),
                instance_of(bytes)
            )
            assert_that(object_iterator.first_byte_pos, equal_to(0))
            assert_that(object_iterator.last_byte_pos, equal_to(5300))

    def test_next_failure(self) -> None:
        with self.config.use_cassette('s3_object_iterator_succeeded_pdf.yml'):
            object_iterator = ObjectIterator(
                Object('test-bucket', 'test/test.pdf')
            )
            object_iterator.__next__()

            assert_that(
                calling(object_iterator.__next__),
                raises(StopIteration)
            )  # type: ignore


import torch
from colorama import Fore as F

def train_step(model, loss_fn, acc_fn, optimizer, dataloader, epochs):
    """
    Trains a model for a binary classification task, calculating both loss and accuracy
    Args:
        model: the model that will be trained
        loss_fn: loss function, should be BCEWithLogitsLoss
        acc_fn: accuracy function (ideally from torchmetrics)
        optimizer: optimizer from torch.optim
        dataloader: dataloader for the data the model will be trained on
        epochs: the number of times the model will run through the entire dataloader

    Returns:
        Each epoch the print out of the current epoch number, loss value, accuracy value (all coloured)
    """
    model.train()
    train_loss, train_acc = 0, 0
    for epoch in range(epochs):
        for X, y in dataloader:
            y = y.unsqueeze(dim=1)
            logits = model(X)
            pred = (torch.sigmoid(logits) > 0.5) # Convert logits to probabilites using sigmoid function, then to labels by setting a 0.5 treshold
            loss = loss_fn(logits.type(torch.float32), y.type(torch.float32))
            acc = acc_fn(pred, y) * 100
            train_loss += loss.item()
            train_acc += acc
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

        train_loss /= len(dataloader)
        train_acc /= len(dataloader)
        print(f'Epoch: {F.BLUE}{epoch}{F.RESET} | Loss: {F.RED}{train_loss:.2f}{F.RESET} | Accuracy: {F.GREEN}{train_acc:.2f}{F.RESET}')

def test_step(model, loss_fn, acc_fn, dataloader):
    """
    Tests a model on a binary classification task, calculating both loss and accuracy
    Args:
        model: the model that will be tested
        loss_fn: loss function, should be BCEWithLogitsLoss
        acc_fn: accuracy function (ideally from torchmetrics)
        dataloader: dataloader for the data the model will be tested on

    Returns:
        At the end of testing; the measured loss and accuracy value (all coloured)
    """
    model.eval()
    test_loss, test_acc = 0, 0
    with torch.inference_mode():
        for X, y in dataloader:
            y = y.unsqueeze(dim=1)
            logits = model(X)
            pred = (torch.sigmoid(logits) > 0.5).float() # Convert logits to probabilites using sigmoid function, then to labels by setting a 0.5 treshold
            loss = loss_fn(logits.type(torch.float32), y.type(torch.float32))
            acc = acc_fn(pred, y) * 100
            test_loss += loss.item()
            test_acc += acc

        test_loss /= len(dataloader)
        test_acc /= len(dataloader)
        print(f'{F.CYAN}MÝR TESTING{F.RESET}\nLoss: {F.RED}{test_loss:.2f}{F.RESET} | Accuracy: {F.GREEN}{test_acc:.2f}{F.RESET}')


import pytest
from tast5_1 import Task5
import logging

@pytest.mark.parametrize('filename, content',
                        [
                            ('1.txt', 'a\n'),
                            ('2.txt', 'b\n'),
                            ('3.txt', 'c\n'),
                            ('4.txt', 'd\n'),
                            ('5.txt', 'e\n')
                        ]
                        )
def test_data(filename, content):
    db = Task5()
    logging.info(f'Testing file {filename}')
    db.connect(filename)
    data = db.get_data()
    assert data == content


import oneflow as flow

from .recurrent import rnn

def _FullyConnected(input_blob, weight_blob, bias_blob):
    output_blob = flow.matmul(input_blob, weight_blob)
    if bias_blob:
        output_blob = flow.nn.bias_add(output_blob, bias_blob)
    return output_blob

class SimpleRNNCell:

    def __init__(self, units,
                 activation=flow.math.tanh,
                 use_bias=True,
                 kernel_initializer=flow.glorot_uniform_initializer(),
                 recurrent_initializer=flow.glorot_normal_initializer(),  # should be orthogonal_initializer
                 bias_initializer=flow.zeros_initializer(),
                 kernel_regularizer=None,
                 recurrent_regularizer=None,
                 bias_regularizer=None,
                 dropout=0.,
                 recurrent_dropout=0.,
                 dtype=flow.float32,
                 trainable=True,
                 **kwargs):
        self.units = units
        self.activation = activation
        self.use_bias = use_bias
        self.kernel_initializer = kernel_initializer
        self.recurrent_initializer = recurrent_initializer
        self.bias_initializer = bias_initializer
        self.kernel_regularizer = kernel_regularizer
        self.recurrent_regularizer = recurrent_regularizer
        self.bias_regularizer = bias_regularizer
        self.dropout = min(1., max(0., dropout))
        self.recurrent_dropout = min(1., max(0., recurrent_dropout))
        self.trainable = trainable
        self.dtype = dtype
        self.layer_index = kwargs['layer_index'] if 'layer_index' in kwargs else ''
        self.direction = kwargs['direction'] if 'layer_index' in kwargs else 'forward'

    def _build(self, inputs):
        input_size = inputs.shape[-1]
        units = self.units
        dtype = self.dtype
        trainable = self.trainable
        with flow.scope.namespace('layer' + str(self.layer_index)):
            with flow.scope.namespace(self.direction):
                self.kernel_blob = flow.get_variable(
                    name='input' + '-kernel',
                    shape=[input_size, units],
                    dtype=dtype,
                    trainable=trainable,
                    regularizer=self.kernel_regularizer,
                    initializer=self.kernel_initializer
                )

                self.recurrent_kernel_blob = flow.get_variable(
                    name='input' + '-recurrent-kernel',
                    shape=[units, units],
                    dtype=dtype,
                    trainable=trainable,
                    regularizer=self.recurrent_regularizer,
                    initializer=self.recurrent_initializer
                )

                self.bias_blob = flow.get_variable(
                    name='input' + '-bias',
                    shape=[units],
                    dtype=dtype,
                    trainable=trainable,
                    regularizer=self.bias_regularizer,
                    initializer=self.bias_initializer
                ) if self.use_bias else None

    def __call__(self, inputs, states):
        self._build(inputs)

        hx = states[0]

        if 0 < self.dropout < 1.:
            inputs = flow.nn.dropout(inputs, rate=self.dropout)

        if 0 < self.recurrent_dropout < 1.:
            hx = flow.nn.dropout(hx, rate=self.recurrent_dropout)

        hy = _FullyConnected(inputs, self.kernel_blob, self.bias_blob)
        output = hy + _FullyConnected(hx, self.recurrent_kernel_blob, None)
        output = self.activation(output)

        return output, [output]

def simple_rnn(inputs,
               units,
               activation=flow.math.tanh,
               use_bias=True,
               kernel_initializer=flow.glorot_uniform_initializer(),
               recurrent_initializer=flow.glorot_normal_initializer(),  # should be orthogonal_initializer
               bias_initializer=flow.zeros_initializer(),
               unit_forget_bias=True,
               kernel_regularizer=None,
               recurrent_regularizer=None,
               bias_regularizer=None,
               dropout=0.,
               recurrent_dropout=0.,
               return_sequences=False,
               initial_state=None,
               **kwargs):
    return rnn(inputs,
               SimpleRNNCell(units,
                             activation=activation,
                             use_bias=use_bias,
                             kernel_initializer=kernel_initializer,
                             recurrent_initializer=recurrent_initializer,
                             bias_initializer=bias_initializer,
                             unit_forget_bias=unit_forget_bias,
                             kernel_regularizer=kernel_regularizer,
                             recurrent_regularizer=recurrent_regularizer,
                             bias_regularizer=bias_regularizer,
                             dropout=dropout,
                             recurrent_dropout=recurrent_dropout,
                             ),
               return_sequences=return_sequences, initial_state=initial_state, kwargs=kwargs)


import urllib.request
import http.cookiejar
import urllib.parse
from bs4 import BeautifulSoup
import os
from People import People
import queue
import io
import gzip
import time
def config_init():
    """initial configuration"""
    cookie = http.cookiejar.CookieJar()
    cookie_support = urllib.request.HTTPCookieProcessor(cookie)
    proxy_handle = urllib.request.ProxyHandler({'http':'http://10.56.192.29:8080'})
    opener = urllib.request.build_opener(proxy_handle,cookie_support) # add proxy and cookie
    urllib.request.install_opener(opener)

def login(email,psw):
    """ login to zhihu.com and return the main page bytes"""
    postdata = urllib.parse.urlencode({
        'email':email,
        'password':psw
    })
    postdata = postdata.encode('UTF-8')
    headers = {
            'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:26.0) Gecko/20100101 Firefox/26.0',
            'Referer':'http://www.zhihu.com'
            }
    req = urllib.request.Request(
            headers = headers,
            url = 'http://www.zhihu.com/login',
            data = postdata
        )
    main_page = urllib.request.urlopen(req)
    return main_page

def FetchPage(url,data):
    """ Fetch page, data type is dict"""
    postdata = None
    if not (data==None) :
        postdata = urllib.parse.urlencode(data)
        postdata = postdata.encode('UTF-8')
    headers = {
            'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:26.0) Gecko/20100101 Firefox/26.0',
            'Referer':'http://www.zhihu.com'
            }
    if not (postdata==None):
        req = urllib.request.Request(
                headers = headers,
                url = url,
                data = postdata
            )
    else:
        req = urllib.request.Request(
                headers = headers,
                url = url,
            )
    try:
        page = urllib.request.urlopen(req)
    except:
        print('fetch page error')
    ## using gzip to fetch page
    if page.code == 200:
        predata = page.read().decode('utf-8')
        pdata = io.StringIO(predata)
        gzipper = gzip.GzipFile(fileobj = pdata)
        try:
            pagedata = gzipper.read()
            print('gzip')
        except:
            # if the server don't support gzip download directly
            pagedata = predata
    else:
        return None
    page.close()
    return pagedata

if __name__ == '__main__':
    start = time.time()
    email='zengzhibin054@gmail.com'
    psw = '********'
    domain = 'http://www.zhihu.com'
    config_init()  #initial configuratin
    main_page = login(email,psw) #login zhihu.com
    main_soup = BeautifulSoup(main_page)

    peopleQueue = queue.Queue()
    urlQueue = queue.Queue()

    ## profile page
    profile_tag = main_soup.find('div',class_='top-nav-profile')
    profile_link = domain+profile_tag.a['href']
    profile_name = profile_tag.a.span.getText()
    me = People(profile_name,profile_link)
    peopleQueue.put(me)

    count = 0
    ## followees page
    while peopleQueue.qsize() > 0:
        item = peopleQueue.get()
        print('link -> '+item.getUrl())
        print('name -> '+item.getName())
        followees_pages = FetchPage(item.getUrl()+r'/followees',None)
        followees_soup = BeautifulSoup(followees_pages)
        followees_links = followees_soup.find_all('div',
            class_='zm-profile-card zm-profile-section-item zg-clear no-hovercard')
        for followee in followees_links:
            link = domain+followee.a['href']
            name = followee.a['title']
            person = People(name,link)
            peopleQueue.put(person)
        count = count + 1
        if count%100==0:
            print(count)
        if count >100:
            break
    elapsed = time.time() - start
    print ("Elapsed Time: %s" % (elapsed))
    print('threadNum:'+str(1))
    f = open('zhihu.txt','a')
    s = 'count:'+str(count)+','+'Thread:'+str(1)+','+'elapsed:'+str(elapsed)
    f.write(s+'\n')
    f.close()


import sys
sys.path.append("../SimpleNN")
"""SingleNN potential."""
from fp_calculator import set_sym, calculate_fp
from NN import MultiLayerNet

import torch
from torch.autograd import grad
from Batch import batch_pad
import time
import numpy as np

import pickle
from ase.data import chemical_symbols, atomic_numbers
from ase.calculators.calculator import (Calculator, all_changes,
                                        PropertyNotImplementedError)
import os

class SingleNN(Calculator):

    implemented_properties = ['energy', 'energies', 'forces']

    def __init__(self, model_path,cal_list = None,**kwargs):
        Calculator.__init__(self, **kwargs)
        self.model_path = model_path
        self.cal_list = cal_list

    def initialize(self, atoms):
        self.numbers = atoms.get_atomic_numbers()
        self.energies = np.empty(len(atoms))
        self.forces = np.empty((len(atoms), 3))

    def calculate(self, atoms=None, properties=['energy'],system_changes=all_changes):
        cal_list = self.cal_list
        if os.path.exists(self.model_path+'/best_model'):
            model = torch.load(self.model_path+'/best_model')
            ensemble_training = False
        else:
            ensemble_training = True
            models = []
            ensemble = 0
            end = False
            while end is False:
                if os.path.exists(self.model_path+f'/best_model-{ensemble}'):
                    models += [torch.load(self.model_path+f'/best_model-{ensemble}')]
                    ensemble += 1
                else:
                    end = True

        sym_params = pickle.load(open(self.model_path+"/sym_params.sav", "rb" ))
        [Gs, cutoff, g2_etas, g2_Rses, g4_etas, g4_zetas, g4_lambdas, elements, weights, element_energy]=sym_params
        params_set = set_sym(elements, Gs, cutoff,
            g2_etas=g2_etas, g2_Rses=g2_Rses,
            g4_etas=g4_etas, g4_zetas = g4_zetas,
            g4_lambdas= g4_lambdas, weights=weights)
        if cal_list is None:
            N_atoms = len(atoms)
        else:
            N_atoms = len(cal_list)
        nelem = len(elements)
        N_sym = params_set[elements[0]]['num']
        data = calculate_fp(atoms, elements, params_set, cal_list = cal_list)
        fps = data['x']
        dfpdXs = data['dx']

        fp = torch.zeros((N_atoms,N_sym))
        dfpdX = torch.zeros((N_atoms, N_sym, N_atoms, 3))
        elements_num = torch.tensor([atomic_numbers[ele] for ele in elements])
        atom_idx = data['atom_idx'] - 1

        a_num = elements_num[atom_idx]
        atom_numbers = a_num.repeat_interleave(nelem).view(len(a_num),nelem)
        # change to float for pytorch to be able to run without error
        if cal_list is not None:
            e_mask = (atom_numbers == elements_num).float()[cal_list]
            atom_idx = atom_idx[cal_list]
        else:
            e_mask = (atom_numbers == elements_num).float()
        fp_track = [0]*nelem
        if element_energy is not None:
            element_energy = torch.sum(element_energy * e_mask)

        for i,idx in enumerate(atom_idx):
            ele = elements[idx]
            fp[i,:] = torch.tensor(fps[ele][fp_track[idx],:]).float()
            if cal_list is None:
                dfpdX[i,:,:,:] = torch.tensor(dfpdXs[ele][fp_track[idx],:,:,:]).float()
            fp_track[idx] += 1
        fp.requires_grad = True

        if ensemble_training:
            scaling = models[0].scaling
        else:
            scaling = model.scaling

        gmin = scaling['gmin']
        gmax = scaling['gmax']
        emin = scaling['emin']
        emax = scaling['emax']
        eps = 1e-5
        sfp = (fp - gmin) / (gmax - gmin+eps)

        if ensemble_training:
            all_energy = []
            all_forces = []
            for model in models:
                Atomic_Es = model(sfp)
                E_predict = torch.sum(torch.sum(Atomic_Es * e_mask,
                    dim = 1)*(emax-emin)+emin,dim=0)
                dEdfp = grad(E_predict,
                    fp,
                    grad_outputs=torch.ones_like(E_predict),
                    create_graph = True,
                    retain_graph = True)[0].view(1,fp.shape[0]*fp.shape[1])
                dfpdX = dfpdX.view(fp.shape[0]*fp.shape[1],fp.shape[0]*3)
                F_predict = -torch.mm(dEdfp,dfpdX).view(fp.shape[0],3)
                forces = F_predict.data.numpy()
                if element_energy is not None:
                    energy = (E_predict + element_energy).data.numpy()
                else:
                    energy = E_predict.data.numpy()
                all_energy += [energy]
                all_forces += [forces]
            all_energy = np.array(all_energy)
            all_forces = np.array(all_forces)
            ensemble_energy = np.mean(all_energy)
            energy_std = np.std(all_energy)
            ensemble_forces = np.mean(all_forces, axis=0)
            forces_std = np.std(all_forces, axis=0)
            self.energy = ensemble_energy
            self.forces = ensemble_forces
            self.results['energy'] = self.energy
            self.results['free_energy'] = self.energy
            self.results['forces'] = self.forces
            self.results['energy_std'] = energy_std
            self.results['forces_std'] = forces_std
        else:
            Atomic_Es = model(sfp)
            E_predict = torch.sum(torch.sum(Atomic_Es * e_mask,
                dim = 1)*(emax-emin)+emin,dim=0)
            dEdfp = grad(E_predict,
                fp,
                grad_outputs=torch.ones_like(E_predict),
                create_graph = True,
                retain_graph = True)[0].view(1,fp.shape[0]*fp.shape[1])
            dfpdX = dfpdX.view(fp.shape[0]*fp.shape[1],fp.shape[0]*3)
            F_predict = -torch.mm(dEdfp,dfpdX).view(fp.shape[0],3)
            self.forces = F_predict.data.numpy()
            if element_energy is not None:
                self.energy = (E_predict + element_energy).data.numpy()
            else:
                self.energy = E_predict.data.numpy()
            self.results['energy'] = self.energy
            self.results['free_energy'] = self.energy
            self.results['forces'] = self.forces

class SingleNNTrainer(object):
    def __init__(self, model_path, scaling, N_sym, n_nodes, activations, nelem, optim_params, device=torch.device('cpu')):

        self.model = MultiLayerNet(N_sym, n_nodes, activations, nelem, scaling=scaling)
        self.model_path = model_path
        self.opt_method = optim_params['opt_method']
        if self.opt_method == 'lbfgs':
            self.history_size = optim_params['history_size']
            self.lr = optim_params['lr']
            self.max_iter = optim_params['max_iter']
            self.line_search_fn = optim_params['line_search_fn']
            self.optimizer = torch.optim.LBFGS(self.model.parameters(), lr=self.lr,
                max_iter=self.max_iter, history_size=self.history_size,
                line_search_fn=self.line_search_fn)
        else:
            print('Optimization method not implemented!')
            raise
        self.device = device
        # self.convergence = hyperparams['convergence']
        # self.n_nodes = n_nodes
        # self.activations = hyperparams['activations']

    def train(self, train_dict, val_dict, E_coeff, F_coeff, epoch, val_interval, n_val_stop, convergence, is_force, logfile):
        device = self.device
        opt_method = self.opt_method
        optimizer = self.optimizer
        model = self.model.to(device)
        model_path = self.model_path

        t0 = time.time()
        SSE = torch.nn.MSELoss(reduction='sum')
        SAE = torch.nn.L1Loss(reduction='sum')
        scaling = self.model.scaling
        gmin = scaling['gmin'].to(device)
        gmax = scaling['gmax'].to(device)
        emin = scaling['emin'].to(device)
        emax = scaling['emax'].to(device)
        n_val = 0
        E_cov = convergence['E_cov']
        F_cov = convergence['F_cov']
        t_ids = np.array(list(train_dict.keys()))
        batch_info = batch_pad(train_dict,t_ids)
        b_fp = batch_info['b_fp'].to(device)

        if is_force:
            b_dfpdX = batch_info['b_dfpdX'].view(b_fp.shape[0],
                                                 b_fp.shape[1]*b_fp.shape[2],
                                                 b_fp.shape[1]*3,
                                                 ).to(device)
        b_e_mask = batch_info['b_e_mask'].to(device)
        b_fp.requires_grad = True
        eps = 1e-5
        sb_fp = ((b_fp - gmin) / (gmax - gmin + eps))
        N_atoms = batch_info['N_atoms'].view(-1).to(device)
        b_e = batch_info['b_e'].view(-1).to(device)
        b_f = batch_info['b_f'].to(device)

        sb_e = ((b_e - emin) / (emax - emin))
        sb_f = (b_f / (emax - emin))
        t1 = time.time()
        logfile.write(f'Batching takes {t1-t0}.\n')

        v_ids = np.array(list(val_dict.keys()))
        v_batch_info = batch_pad(val_dict,v_ids)
        v_b_fp = v_batch_info['b_fp'].to(device)
        if is_force:
            v_b_dfpdX = v_batch_info['b_dfpdX'].view(v_b_fp.shape[0],
                                                     v_b_fp.shape[1]*v_b_fp.shape[2],
                                                     v_b_fp.shape[1]*3,
                                                     ).to(device)
        v_b_e_mask = v_batch_info['b_e_mask'].to(device)
        v_b_fp.requires_grad = True
        v_sb_fp = ((v_b_fp - gmin) / (gmax - gmin + eps))
        v_N_atoms = v_batch_info['N_atoms'].view(-1).to(device)
        v_b_e = v_batch_info['b_e'].view(-1).to(device)

        v_b_f = v_batch_info['b_f'].to(device)

        v_sb_e = ((v_b_e - emin) / (emax - emin))
        v_sb_f = (v_b_f / (emax - emin))

        if opt_method == 'lbfgs':
            for i in range(epoch):
                def closure():
                    global E_MAE, F_MAE
                    optimizer.zero_grad()
                    Atomic_Es = model(sb_fp)
                    E_predict = torch.sum(Atomic_Es * b_e_mask, dim = [1,2])
                    if is_force:
                        F_predict = self.get_forces(E_predict, b_fp, b_dfpdX)
                        metrics = self.get_metrics(sb_e, sb_f, N_atoms, t_ids,
                                               E_predict, F_predict, SSE, SAE, scaling, b_e_mask)
                        [E_loss, F_loss, E_MAE, F_MAE, E_RMSE, F_RMSE] = metrics
                        loss = E_coeff * E_loss + F_coeff * F_loss
                    else:
                        metrics =  self.get_metrics(sb_e, None, N_atoms, t_ids,
                                               E_predict, None, SSE, SAE, scaling, b_e_mask)
                        [E_loss, F_loss, E_MAE, F_MAE, E_RMSE, F_RMSE] = metrics
                        loss = E_coeff * E_loss

                    loss.backward(retain_graph=True)
                    return loss

                optimizer.step(closure)

                if i % val_interval == 0:
                    n_val += 1
                    Atomic_Es = model(sb_fp)
                    E_predict = torch.sum(Atomic_Es * b_e_mask, dim = [1,2])
                    if is_force:
                        F_predict = self.get_forces(E_predict, b_fp, b_dfpdX)
                        metrics =  self.get_metrics(sb_e, sb_f, N_atoms, t_ids,
                                               E_predict, F_predict, SSE, SAE, scaling, b_e_mask)
                        [E_loss, F_loss, E_MAE, F_MAE, E_RMSE, F_RMSE] = metrics
                        loss = E_coeff * E_loss + F_coeff * F_loss
                    else:
                        metrics = self.get_metrics(sb_e, None, N_atoms, t_ids,
                                               E_predict, None, SSE, SAE, scaling, b_e_mask)
                        [E_loss, F_loss, E_MAE, F_MAE, E_RMSE, F_RMSE] = metrics
                        loss = E_coeff * E_loss
                    logfile.write(f'{i}, E_RMSE/atom = {E_RMSE}, F_RMSE = {F_RMSE}, loss={loss}\n')
                    logfile.write(f'{i}, E_MAE/atom = {E_MAE}, F_MAE = {F_MAE}\n')

                    v_Atomic_Es = model(v_sb_fp)
                    v_E_predict = torch.sum(v_Atomic_Es * v_b_e_mask, dim = [1,2])
                    if is_force:
                        v_F_predict = self.get_forces(v_E_predict, v_b_fp, v_b_dfpdX)
                        v_metrics =  self.get_metrics(v_sb_e, v_sb_f, v_N_atoms, v_ids,
                                               v_E_predict, v_F_predict, SSE, SAE, scaling, v_b_e_mask)
                        [v_E_loss, v_F_loss, v_E_MAE, v_F_MAE, v_E_RMSE, v_F_RMSE] = v_metrics
                        v_loss = E_coeff * v_E_loss + F_coeff * v_F_loss
                    else:
                        v_metrics =  self.get_metrics(v_sb_e, None, v_N_atoms, v_ids,
                                                 v_E_predict, None, SSE, SAE, scaling, v_b_e_mask)
                        [v_E_loss, v_F_loss, v_E_MAE, v_F_MAE, v_E_RMSE, v_F_RMSE] = v_metrics
                        v_loss = E_coeff * v_E_loss

                    try:
                        if v_loss < best_v_loss:
                            best_loss = loss
                            best_E_MAE = E_MAE
                            best_F_MAE = F_MAE
                            best_v_loss = v_loss
                            best_v_E_MAE = v_E_MAE
                            best_v_F_MAE = v_F_MAE
                            torch.save(model,model_path)
                            n_val = 1
                    except NameError:
                        best_loss = loss
                        best_E_MAE = E_MAE
                        best_F_MAE = F_MAE
                        best_v_loss = v_loss
                        best_v_E_MAE = v_E_MAE
                        best_v_F_MAE = v_F_MAE
                        torch.save(model,model_path)
                        n_val = 1

                    logfile.write(f'val, E_RMSE/atom = {v_E_RMSE}, F_RMSE = {v_F_RMSE}\n')
                    logfile.write(f'val, E_MAE/atom = {v_E_MAE}, F_MAE = {v_F_MAE}\n')
                    logfile.flush()
                    if n_val > n_val_stop:
                        break

        t2 = time.time()
        logfile.write(f'Training takes {t2-t0}\n')
        logfile.close()
        return [best_loss, best_E_MAE, best_F_MAE, best_v_loss, best_v_E_MAE, best_v_F_MAE]

    def get_forces(self, E_predict, b_fp, b_dfpdX):
        b_dEdfp = grad(E_predict,
                        b_fp,
                        grad_outputs=torch.ones_like(E_predict),
                        create_graph = True,
                        retain_graph = True)[0].view(b_fp.shape[0],1,b_fp.shape[1]*b_fp.shape[2])
        F_predict = - torch.bmm(b_dEdfp,b_dfpdX).view(b_fp.shape[0],b_fp.shape[1],3)
        return F_predict

    def get_metrics(self, sb_e, sb_f, N_atoms, ids, E_predict, F_predict, SSE, SAE, scaling, b_e_mask):

        gmin = scaling['gmin']
        gmax = scaling['gmax']
        emin = scaling['emin']
        emax = scaling['emax']

        E_loss = SSE(sb_e, E_predict / N_atoms) / len(ids)
        E_MAE = SAE(sb_e, E_predict / N_atoms) / len(ids) * (emax - emin)
        E_RMSE = torch.sqrt(E_loss) * (emax - emin)
        if sb_f is None:
            F_loss = 0
            F_MAE = 0
            F_RMSE = 0
        else:
            F_loss = SSE(sb_f, F_predict) / (3 * torch.sum(N_atoms))
            F_MAE = SAE(sb_f, F_predict) / (3 * torch.sum(N_atoms)) * (emax - emin)
            F_RMSE = torch.sqrt(F_loss) * (emax - emin)
            F_max = torch.max(torch.abs(sb_f-F_predict))*(emax-emin)
            print('F_max = ',F_max.data.numpy(), 'eV/A')
        return [E_loss, F_loss, E_MAE, F_MAE, E_RMSE, F_RMSE]

    def evaluate(self, data_dict, E_coeff, F_coeff, is_force):
        device = self.device
        for key in data_dict.keys():
            data_dict[key] = data_dict[key].to(device)

        model_path = self.model_path
        SSE = torch.nn.MSELoss(reduction='sum')
        SAE = torch.nn.L1Loss(reduction='sum')
        model = torch.load(model_path)
        scaling = model.scaling
        gmin = scaling['gmin']
        gmax = scaling['gmax']
        emin = scaling['emin']
        emax = scaling['emax']

        ids = np.array(list(data_dict.keys()))
        batch_info = batch_pad(data_dict,ids)
        b_fp = batch_info['b_fp']

        if is_force:
            b_dfpdX = batch_info['b_dfpdX'].view(b_fp.shape[0],
                                                 b_fp.shape[1]*b_fp.shape[2],
                                                 b_fp.shape[1]*3)
        b_e_mask = batch_info['b_e_mask']
        b_fp.requires_grad = True
        sb_fp = (b_fp - gmin) / (gmax - gmin)
        N_atoms = batch_info['N_atoms'].view(-1)
        b_e = batch_info['b_e'].view(-1)
        b_f = batch_info['b_f']

        sb_e = (b_e - emin) / (emax - emin)
        sb_f = b_f / (emax - emin)

        Atomic_Es = model(sb_fp)
        E_predict = torch.sum(Atomic_Es * b_e_mask, dim = [1,2])
        if is_force:
            F_predict = get_forces(E_predict, b_fp, b_dfpdX)
            metrics =  get_metrics(sb_e, sb_f, N_atoms, ids,
                                   E_predict, F_predict, SSE, SAE, scaling, b_e_mask)
            [E_loss, F_loss, E_MAE, F_MAE, E_RMSE, F_RMSE] = metrics
            loss = E_coeff * E_loss + F_coeff * F_loss
        else:
            metrics =  get_metrics(sb_e, None, N_atoms, ids,
                                   E_predict, None, SSE, SAE, scaling, b_e_mask)
            [E_loss, F_loss, E_MAE, F_MAE, E_RMSE, F_RMSE] = metrics
            loss = E_coeff * E_loss
        return [loss, E_MAE, F_MAE]


import asyncio

from django.core.cache import cache

from apps.integration.models import KotakNeoApi as KotakNeoApiModel
from apps.integration.utils import divide_and_list, get_option_ltp
from apps.integration.utils.broker.dummy import DummyApi
from apps.integration.utils.broker.kotak_neo import KotakNeoApi
from apps.trade.models import DeployedOptionStrategy as DeployedOptionStrategyModel

class StrategyOrder(object):
    def __init__(self, opt_strategy: DeployedOptionStrategyModel):
        self.opt_strategy = opt_strategy
        self.symbol = opt_strategy.instrument.symbol
        self.websocket_ids = self.opt_strategy.websocket_ids.split(",")
        self.strategy_id = self.opt_strategy.pk
        self.str_strategy_id = str(self.opt_strategy.pk)
        self.options = self.opt_strategy.options.copy()
        self.parameters = [
            p.parameters
            for p in self.opt_strategy.parameters.filter(is_active=True).order_by(
                "name"
            )
        ]
        self.parameters_len = len(self.parameters) or 1
        self.entry_type = self.opt_strategy.strategy_type
        self.exit_type = "BUY" if self.entry_type == "SELL" else "SELL"
        self.strategy_tradingsymbol_cache = f"TRADINGSYMBOLS_{self.opt_strategy.pk}"
        self.broker = self.opt_strategy.broker
        self.slippage = float(self.opt_strategy.slippage)
        user_params = (
            cache.get("DEPLOYED_STRATEGIES", {})
            .get(self.str_strategy_id, {})
            .get("user_params", [])
        )
        if user_params and cache.get(self.strategy_tradingsymbol_cache):
            self.user_params = user_params
            self.entered = True
        else:
            self.entered = False
            self.user_params = []
            for user in self.opt_strategy.users.filter(is_active=True).order_by(
                "order_seq"
            ):
                self.user_params.append(
                    {
                        "broker_api": user.broker_api,
                        "user": user.broker_api.user,
                        "broker": user.broker_api.broker,
                        "quantity_multiple": [
                            item * self.opt_strategy.lot_size
                            for item in divide_and_list(self.parameters_len, user.lots)
                        ],
                    }
                )

    async def get_kotak_neo_parameters(self, user):
        row = KotakNeoApiModel.objects.get(broker_api__user=user)

        return {
            "username": user.username,
            "headers": {
                "Authorization": f"Bearer {row.access_token}",
                "Sid": row.sid,
                "Auth": row.auth,
                "neo-fin-key": row.neo_fin_key,
                "Content-Type": "application/x-www-form-urlencoded",
            },
            "query_params": {"sId": row.hs_server_id},
        }

    async def quantity_wise_strategy_map(self, user_params):
        quantity_map_list = []

        for idx in range(self.parameters_len):
            quantity_map_list.append(
                {
                    user_param["user"].username: user_param["quantity_multiple"][idx]
                    for user_param in user_params
                }
            )

        return quantity_map_list

    async def place_dummy_order(self, buy_pending, sell_pending, user_params):
        quantity_map_list = await self.quantity_wise_strategy_map(user_params)
        users = [user_param["user"] for user_param in user_params]

        if not user_params:
            return

        dummyapi = DummyApi(users=users)

        if buy_pending:
            await asyncio.gather(
                *[
                    dummyapi.option_place_and_chase_order(
                        symbol=self.symbol,
                        tradingsymbol=row["tradingsymbol"],
                        quantity_map=quantity_map_list[row.get("idx", 0)],
                        transaction_type="BUY",
                        price=get_option_ltp(
                            symbol=self.symbol,
                            tradingsymbol=row["tradingsymbol"],
                            websocket_id=row["websocket_id"],
                        ),
                        tag_prefix=self.str_strategy_id,
                        websocket_id=row["websocket_id"],
                    )
                    for row in buy_pending
                ]
            )

        if sell_pending:
            await asyncio.gather(
                *[
                    dummyapi.option_place_and_chase_order(
                        symbol=self.symbol,
                        tradingsymbol=row["tradingsymbol"],
                        quantity_map=quantity_map_list[row.get("idx", 0)],
                        transaction_type="SELL",
                        price=get_option_ltp(
                            symbol=self.symbol,
                            tradingsymbol=row["tradingsymbol"],
                            websocket_id=row["websocket_id"],
                        ),
                        tag_prefix=self.str_strategy_id,
                        websocket_id=row["websocket_id"],
                    )
                    for row in sell_pending
                ]
            )

    async def place_kotak_neo_order(self, buy_pending, sell_pending, user_params):
        quantity_map_list = await self.quantity_wise_strategy_map(user_params)
        users = [
            await self.get_kotak_neo_parameters(user_param["user"])
            for user_param in user_params
        ]
        if not users:
            return

        knapi = KotakNeoApi(users=users)

        if buy_pending:
            await asyncio.gather(
                *[
                    knapi.option_place_and_chase_order(
                        symbol=self.symbol,
                        tradingsymbol=row["tradingsymbol"],
                        quantity_map=quantity_map_list[row.get("idx", 0)],
                        transaction_type="BUY",
                        expected_price=get_option_ltp(
                            symbol=self.symbol,
                            tradingsymbol=row["tradingsymbol"],
                            websocket_id=row["websocket_id"],
                        )
                        + self.slippage,
                        tag_prefix=self.str_strategy_id,
                        websocket_id=row["websocket_id"],
                    )
                    for row in buy_pending
                ]
            )

        if sell_pending:
            await asyncio.gather(
                *[
                    knapi.option_place_and_chase_order(
                        symbol=self.symbol,
                        tradingsymbol=row["tradingsymbol"],
                        quantity_map=quantity_map_list[row.get("idx", 0)],
                        transaction_type="SELL",
                        expected_price=get_option_ltp(
                            symbol=self.symbol,
                            tradingsymbol=row["tradingsymbol"],
                            websocket_id=row["websocket_id"],
                        )
                        - self.slippage,
                        tag_prefix=self.str_strategy_id,
                        websocket_id=row["websocket_id"],
                    )
                    for row in sell_pending
                ]
            )

    async def place_order(
        self,
        sell_pending: list,
        buy_pending: list,
        user_params: list | None = None,
    ):
        if buy_pending or sell_pending:
            if not user_params:
                user_params = self.user_params

            if self.broker == "kotak_neo":
                kotak_neo_users_params = [
                    user_param
                    for user_param in user_params
                    if user_param["broker"] == "kotak_neo"
                ]

                dummy_users_params = [
                    user_param
                    for user_param in user_params
                    if user_param["broker"] == "dummy"
                ]

                await asyncio.gather(
                    self.place_dummy_order(
                        buy_pending,
                        sell_pending,
                        user_params=dummy_users_params,
                    ),
                    self.place_kotak_neo_order(
                        buy_pending,
                        sell_pending,
                        user_params=kotak_neo_users_params,
                    ),
                )


import numpy as np
import matplotlib.pyplot as plt
from matplotlib.patches import Rectangle

simlen = 10000

pX = pY = np.ones(6)
pX = pY = pX/6
pZ = np.array([ 1/36, 2/36, 3/36, 4/36, 5/36, 6/36, 5/36, 4/36, 3/36, 2/36, 1/36 ])
possible = np.arange(1,7)
possible_Z = np.arange(2, 13)
X= np.random.choice(possible, simlen, p=pX)
Y= np.random.choice(possible, simlen, p=pY)
cdfY = np.cumsum(pY)
Z = X + Y

simX, simpX = np.unique(X , return_counts= True)
simZ, simpZ = np.unique(Z , return_counts= True)
simY, simpY = np.unique(Y , return_counts= True)

simpX = simpX/simlen
simpY = simpY/simlen
simpZ = simpZ/simlen

simp_cdfY = np.cumsum(simpY)
# for Question1,
p1theo = cdfY[5] - cdfY[3]
p1sim = simp_cdfY[5] - simp_cdfY[3]

print("Theoretical value of probability of getting sum > 9 given X>5 is: " + str(p1theo))
print("Actual value of probability of getting sum > 9 given X>5 is: " + str(p1sim))

print("----------")
#for Question2,
p2theo = (pX[2]*pY[4] + pX[1]*pY[5])/cdfY[3]
p2sim = (simpX[2]*simpY[4] + simpX[1]*simpY[5])/simp_cdfY[3]

print("Theoretical value of probability of getting sum = 8 given Y < 4 is: " + str(p2theo))
print("Actual value of probability of getting sum = 8 given Y < 4 is: " + str(p2sim))

#plotting

fig,axes = plt.subplots(2,figsize = (12,20))

possible1 = possible.reshape(1,6)
eq9X = np.repeat(possible1,3, axis = 1)

pos = np.array([[1,2,3]])
eq9Y = np.repeat(pos , 6 , axis = 0)
#plot-1
axes[0].set_title("Plot for verifying current problem")
axes.flat[0].set(xlabel = 'Questions' , ylabel = 'Answers')
axes[0].stem([1, 2] , [p1theo,p2theo] , label = 'Actual', markerfmt = 'ro' , basefmt = 'g-' ,linefmt = 'b')
axes[0].set_xticks([1,2])
axes[0].plot([1, 2] , [p1sim, p2sim], 'yo',label = 'Theoretical')
axes[0].legend()

validX = np.repeat(possible,6).reshape(6,6)
validY = np.repeat(possible1,6 ,axis = 0)
print(validX)
print(validY)
#plot-2
axes[1].set_title("X+Y = 8 | Y < 4")
axes.flat[1].set(xlabel = 'X' , ylabel = 'Y')
axes[1].set_xlim(-0.2 , 7)
axes[1].set_ylim(-0.2 , 7)
axes[1].add_patch(Rectangle((0.6, 0.6), 5.8,5.8 , edgecolor = '#000000',facecolor = '#00000020' ,fill =True , label = 'Valid region for die roll results'))
axes[1].add_patch(Rectangle((0.8, 0.8), 5.4,2.4 , edgecolor = 'blue',facecolor = '#0000f020' ,fill =True ,label = 'Y<4'))
axes[1].scatter(validX,validY ,c = 'black' ,label = 'Integral points (X,Y)')
axes[1].scatter(eq9X,eq9Y , label = 'Integral points where Y<4')

axes[1].plot(possible[1:],8-possible[1:],'yo-' , label ='X+Y = 8')
axes[1].legend()

plt.show()


#!/usr/bin/env python3
import sqlalchemy as sa
import sqlalchemy.exc
import sys
from contextlib import contextmanager
from pathlib import Path
import click
import csv

sys.path.append('tm_navigator')
from tm_navigator.models import *

engine = sa.create_engine('postgresql+psycopg2://postgres@localhost/tm_navigator')
sa.orm.configure_mappers()
Session = sa.orm.sessionmaker(bind=engine)

@contextmanager
def session_scope():
    """Provide a transactional scope around a series of operations."""
    session = Session()
    try:
        yield session
        session.commit()
    except:
        session.rollback()
        raise
    finally:
        session.close()

from sqlalchemy_utils import aggregates

aggregates.local_condition = lambda prop, objects: sa.literal(True)

class ListSession(list):
    def execute(self, query):
        return self._session.execute(query)

def update_aggregates(session, *classes_modified):
    ls = ListSession([c() for c in classes_modified])
    ls._session = session
    aggregates.manager.construct_aggregate_queries(ls, None)

def copy_from_csv(session, model, csv_file):
    with session.connection().connection.cursor() as cursor, csv_file.open() as csv_f:
        fieldnames = csv_f.readline()
        cursor.copy_expert('copy {0} ({1}) from stdin with csv'.format(model.__tablename__, fieldnames), csv_f)

    update_aggregates(session, model)

def check_files(directory, expected_names, extension='.csv', cli=True):
    file_names = {p.name for p in directory.iterdir() if p.is_file()}
    expected_files = {t + extension for t in expected_names}

    if expected_files & file_names:
        s = 'Found files {}.'.format(
            ', '.join('"{}"'.format(f) for f in sorted(expected_files & file_names)))
        if cli:
            click.secho(s, fg='green')
        else:
            print(s)
    if expected_files - file_names:
        s = 'Not found files {}.'.format(
            ', '.join('"{}"'.format(f) for f in sorted(expected_files - file_names)))
        if cli:
            click.secho(s, fg='red')
            click.echo('Will try to continue with the files present.')
        else:
            print(s)
            print('Will try to continue with the files present.')

def delete_data_for(session, models, cli=True):
    def delete_data(iter_func=lambda: None):
        for table in reversed(Base.metadata.sorted_tables):
            matching_models = [m for m in models if m.__table__ == table]
            if not matching_models:
                continue
            model = matching_models[0]
            session.query(model).delete()
            iter_func()

    if cli:
        with click.progressbar(label='Deleting data', length=len(models)) as pbar:
            delete_data(lambda: pbar.update(1))
    else:
        delete_data()

def load_data_for(session, models, directory, cli=True):
    def load_data(iter_func=lambda: None):
        for table in Base.metadata.sorted_tables:
            matching_models = [m for m in models if m.__table__ == table]
            if not matching_models:
                continue
            model = matching_models[0]

            file = directory / '{}.csv'.format(model.__tablename__)
            copy_from_csv(session, model, file)
            iter_func()

    if cli:
        with click.progressbar(label='Loading data', length=len(models)) as pbar:
            load_data(lambda: pbar.update(1))

    else:
        load_data()

@click.group()
def cli():
    pass

dir_type = click.Path(exists=True, file_okay=False, resolve_path=True)

@cli.command()
def describe():
    with session_scope() as session:
        SchemaMixin.activate_public_schema(session)
        dses = session.query(DatasetMeta).order_by(DatasetMeta.id).all()
        for ds in dses:
            ds.activate_schemas()
            click.secho('- Dataset #{id}: {title}, {ntm} models'.format(id=ds.id,
                                                                        title=ds.title or 'untitled',
                                                                        ntm=len(ds.topic_models)),
                        fg='blue')
            try:
                click.echo('  Documents: {cnt}'.format(cnt=session.query(Document).count()))
                click.echo('  Terms: ' +
                           ', '.join('{m.count} {m.name} with {nocc} occurrences'.format(m=m, nocc=nocc)
                                     for m, nocc in session.query(Modality, sa.func.count())
                                     .join(Modality.terms).join(Term.documents)
                                     .group_by(Modality).order_by(Modality.name)))
            except sa.exc.ProgrammingError:
                click.echo('  Error - can\'t find data')
                session.rollback()
            click.echo()

            for tm in ds.topic_models:
                tm.activate_schemas()
                click.secho('  - Topic Model #{id}: {title}'.format(id=tm.id, title=tm.title or 'untitled'),
                            fg='blue')
                try:
                    click.echo('    Topics: ' +
                               ', '.join('{cnt} {t} at lvl {lvl}'.format(lvl=lvl, t='background' if bck else '', cnt=cnt)
                                         for lvl, bck, cnt in session.query(Topic.level, Topic.is_background, sa.func.count())
                                         .group_by(Topic.level, Topic.is_background).order_by(Topic.level, Topic.is_background)))
                except sa.exc.ProgrammingError:
                    click.echo('    Error - can\'t find data')
                    session.rollback()
                click.echo()

def add_dataset_():
    with session_scope() as session:
        SchemaMixin.activate_public_schema(session)

        ds = DatasetMeta()
        session.add(ds)
        session.flush()

        ds.activate_schemas()
        Base.metadata.create_all(engine,
                                 tables=map(lambda c: c.__table__, models_dataset))
        return ds.id

@cli.command()
def add_dataset():
    dataset_id = add_dataset_()
    click.echo('Added Dataset #{id}'.format(id=dataset_id))

def load_dataset_(dataset_id, title, directory, cli=False):
    directory = Path(directory)
    target_models = models_dataset

    check_files(directory, [m.__tablename__ for m in target_models], cli=cli)
    if cli:
        click.confirm('Proceeding will overwrite the corresponding data in the database. Continue?',
                      abort=True, default=True)

    models = [m
              for m in target_models
              if (directory / '{}.csv'.format(m.__tablename__)).is_file()]

    with session_scope() as session:
        SchemaMixin.activate_public_schema(session)
        ds = session.query(DatasetMeta).filter_by(id=dataset_id).one()
        ds.activate_schemas()

        if title is not None:
            ds.title = title
        delete_data_for(session, models, cli)
        load_data_for(session, models, directory, cli)

@cli.command()
@click.option('-d', '--dataset-id', type=int, required=True)
@click.option('-t', '--title', type=str)
@click.option('-dir', '--directory', type=dir_type, required=True)
def load_dataset(dataset_id, title, directory):
    load_dataset_(dataset_id, title, directory, cli=True)

def add_topicmodel_(dataset_id):
    with session_scope() as session:
        SchemaMixin.activate_public_schema(session)

        ds = session.query(DatasetMeta).filter_by(id=dataset_id).one()
        tm = TopicModelMeta(dataset=ds)
        session.add(tm)
        session.flush()
        tm.domains.append(TopicModelDomain(domain='{}.'.format(tm.id)))

        tm.activate_schemas()
        Base.metadata.create_all(engine,
                                 tables=map(lambda c: c.__table__, models_topic + models_assessment))

        return tm.id

@cli.command()
@click.option('-d', '--dataset-id', type=int, required=True)
def add_topicmodel(dataset_id):
    topicmodel_id = add_topicmodel_(dataset_id)
    click.echo('Added Topic Model #{id} for Dataset #{did}'.format(id=topicmodel_id, did=dataset_id))

def load_topicmodel_(topicmodel_id, title, directory, cli=False):
    directory = Path(directory)
    target_models = models_topic

    check_files(directory, [m.__tablename__ for m in target_models], cli=cli)
    if cli:
        click.confirm('Proceeding will overwrite the corresponding data in the database. Continue?',
                      abort=True, default=True)

    models = [m
              for m in target_models
              if (directory / '{}.csv'.format(m.__tablename__)).is_file()]

    with session_scope() as session:
        SchemaMixin.activate_public_schema(session)
        tm = session.query(TopicModelMeta).filter_by(id=topicmodel_id).one()
        tm.activate_schemas()

        if title is not None:
            tm.title = title
        delete_data_for(session, models, cli)
        load_data_for(session, models, directory, cli)

@cli.command()
@click.option('-m', '--topicmodel-id', type=int, required=True)
@click.option('-t', '--title', type=str)
@click.option('-dir', '--directory', type=dir_type, required=True)
def load_topicmodel(topicmodel_id, title, directory):
    load_topicmodel_(topicmodel_id, title, directory, cli=True)

def dump_assessments_(topicmodel_id, directory):
    directory = Path(directory)
    with session_scope() as session:
        SchemaMixin.activate_public_schema(session)
        tm = session.query(TopicModelMeta).filter_by(id=topicmodel_id).one()
        tm.activate_schemas()

        assessments = session.query(ATopic).all()
        topic_count = session.query(Topic).count() - 1

        grades = {}
        for assessment in assessments:
            grades[assessment.topic_id] = assessment.value
        with open(directory.joinpath('topic_assessments.csv'), 'w', newline='') as csvfile:
            writer = csv.writer(csvfile)
            writer.writerow(['topic_id', 'value'])
            for idx, grade in grades.items():
                writer.writerow([idx, grade])

@cli.command()
@click.option('-m', '--topicmodel-id', type=int, required=True)
@click.option('-dir', '--directory', type=dir_type, required=True)
def dump_assessments(topicmodel_id, directory):
    dump_assessments_(topicmodel_id, directory)

if __name__ == '__main__':
    cli()


from numpy import linspace, sin

from enable.api import ColorTrait, marker_trait
from chaco.api import ArrayPlotData, Plot
from enable.api import ComponentEditor
from traits.api import HasTraits, Instance, Int
from traitsui.api import Group, Item, View

class ScatterPlotTraits(HasTraits):

    plot = Instance(Plot)
    color = ColorTrait("blue")
    marker = marker_trait
    marker_size = Int(4)

    traits_view = View(
        Group(
            Item("color", label="Color", style="custom"),
            Item("marker", label="Marker"),
            Item("marker_size", label="Size"),
            Item("plot", editor=ComponentEditor(), show_label=False),
            orientation="vertical",
        ),
        width=800,
        height=600,
        resizable=True,
        title="Chaco Plot",
    )

    def __init__(self):
        # Create the data and the PlotData object
        x = linspace(-14, 14, 100)
        y = sin(x) * x ** 3
        plotdata = ArrayPlotData(x=x, y=y)
        # Create a Plot and associate it with the PlotData
        plot = Plot(plotdata)
        # Create a line plot in the Plot
        self.renderer = plot.plot(("x", "y"), type="scatter", color="blue")[0]
        self.plot = plot

    def _color_changed(self):
        self.renderer.color = self.color

    def _marker_changed(self):
        self.renderer.marker = self.marker

    def _marker_size_changed(self):
        self.renderer.marker_size = self.marker_size

# ===============================================================================
# demo object that is used by the demo.py application.
# ===============================================================================
demo = ScatterPlotTraits()

if __name__ == "__main__":
    demo.configure_traits()


import tflearn as tl
import numpy as np
import os,glob,cv2
import sys,argparse
from read_image import read_valid_image
import tensorflow as tf
from tflearn.layers.conv import conv_2d, max_pool_2d
from tflearn.layers.core import input_data , dropout, fully_connected
# from tflearn.layers.estimator import  regression

image_data,image_label,num_class,label_name = read_valid_image()
# First, pass the path of the image
dir_path = os.path.dirname(os.path.realpath(__file__))
image_path=sys.argv[1]
filename = dir_path +'/' +image_path
image_size=32
num_channels=1
images = []
# Reading the image using OpenCV
image = cv2.imread(filename,0)

# Resizing the image to our desired size and preprocessing will be done exactly as done during training
image = cv2.resize(image, (image_size, image_size), cv2.INTER_LINEAR)
image = image[:, :, np.newaxis]
images.append(image)
images = np.array(images, dtype=np.uint8)
images = images.astype('float32')
x = np.multiply(images, 1.0/255.0)
# y = np.zeros([1,2])
#The input to the network is of shape [None image_size image_size num_channels]. Hence we reshape.
# x_batch = images.reshape(1, image_size,image_size,num_channels)
convnet = input_data(shape=[None,32,32,1],name='input')

convnet = conv_2d(convnet,32,2,activation='relu')
convnet = max_pool_2d(convnet,2)

convnet = conv_2d(convnet,64,2,activation='relu')
convnet = max_pool_2d(convnet,2)

convnet = conv_2d(convnet,32,2,activation='relu')
convnet = max_pool_2d(convnet,2)

convnet = conv_2d(convnet,64,2,activation='relu')
convnet = max_pool_2d(convnet,2)

convnet = conv_2d(convnet,32,2,activation='relu')
convnet = max_pool_2d(convnet,2)

convnet = conv_2d(convnet,64,2,activation='relu')
convnet = max_pool_2d(convnet,2)

convnet = fully_connected(convnet, 1024,activation='relu')

convnet = fully_connected(convnet,2,activation='softmax',name="output")
model = tl.DNN(convnet)
model.load('tflearn')
final = model.predict(x)
for i in range(0,len(label_name)):
    print(label_name[i],final[0][i])

# print(final)


import os
import random

from urllib.parse import quote
from typing import List, Optional
from dataclasses import dataclass, field
from enum import Enum

from jinja2 import Template

SEAT_NUMBER_MAX_LENGTH = 3

# ascii characters are prohibited :D
ru_alphabet_lower = 'абвгдеёжзиклмопрстуфхцчшщэюя'
names = ['Сергій', 'Іван', 'Микола', 'Софія', 'Дмитро', 'Ганна', 'Надія', 'Віктор', 'Василь']
surnames = ['Іванов', 'Морозов', 'Третяк', 'Проценко', 'Сорокін', 'Дубілет', 'Зеленський']

class WagonType(Enum):
    K = 'К'  # Купе
    P = 'П'  # Плацкарт
    L = 'Л'  # Люкс

    def __str__(self):
        return self.value

@dataclass
class Seat:
    number: str
    first_name: str
    last_name: str

    def __str__(self) -> str:
        return f'Seat <number: {self.number}>'

    def __post_init__(self):
        if 3 < len(self.number) < 3:
            self._normalize_seat_number()

    def _normalize_seat_number(self):
        self.number = self.number.zfill(SEAT_NUMBER_MAX_LENGTH)

@dataclass
class TrainOrder:
    from_station_id: int
    to_station_id: int
    train_code: str
    date: str
    wagon_number: int
    wagon_class: str
    wagon_type: WagonType
    wagon_railway: int
    seats: List[Seat] = field(default_factory=list)

    def __str__(self) -> str:
        return f'Ticker order <From: {self.from_station_id}, To: {self.to_station_id}, ' \
               f'Wagon: {self.wagon_number}, ' \
               f'Date: {self.date}, Seats: {self.seats}>'

    @staticmethod
    def roll_random_name(name_list: List[str]) -> str:
        return random.choice(name_list)

    def add_seat(self, seat_number: str,
                 first_name: Optional[str] = None,
                 last_name: Optional[str] = None):
        if not first_name:
            first_name = self.roll_random_name(names)
        if not last_name:
            last_name = self.roll_random_name(surnames)
        seat = Seat(seat_number, first_name, last_name)
        self.seats.append(seat)
        return self

    def serialize(self) -> str:
        with open(os.path.join(os.path.abspath(os.path.dirname(__file__)),
                               'order_template.jinja2')) as fd:
            tpl = Template(fd.read())
            tpl_str = tpl.render(**vars(self))
            tpl_str = (
                quote(tpl_str.replace('\n', ''))
                    .replace('%3D', '=')
                    .replace('%26', '&')
                    .replace('%20', '')
            )
            return tpl_str


from ete3 import Tree , TreeStyle , NodeStyle , faces , AttrFace
import csv

#t = Tree("(A:1,(B:1,(E:1,D:1):0.5):0.5);" )
#t.render("mytree.png", w=183, units="mm")

t = Tree("(0);" )

all_orgs = []
with open('testlineage.csv') as csvfile:
    reader = csv.DictReader(csvfile)
    for org in reader:
        all_orgs.append((org['id'],org['ancestor_id'],org['recorded_at'],org['on_lod']))
with open('testunpruned.csv') as csvfile:
    reader = csv.DictReader(csvfile)
    for org in reader:
        all_orgs.append((org['id'],org['ancestor_id'],'101','0'))

all_orgs.sort(key=lambda t: int(t[0]))
#print (all_orgs)

style = NodeStyle()
style["fgcolor"] = "#00f000"
#style["bgcolor"] = "lightred"
#style["size"] = 10
style["shape"] = "circle"
style["vt_line_color"] = "#0000aa"
style["hz_line_color"] = "#0000aa"
style["vt_line_width"] = 50
style["hz_line_width"] = 50
style["vt_line_type"] = 2 # 0 solid, 1 dashed, 2 dotted
style["hz_line_type"] = 2

for org in all_orgs:
    #print (org)
    new_node = Tree('(' + org[0]  + ');') #+ ':' + str(int(org[2])^1)
    if (org[3] == '1'):
        #print (org)
        new_node.set_style(style)
    anc = t.search_nodes(name=org[1])[0]
    anc.add_child(new_node)

#for n in t.traverse():
    #print (n.name)
    #if all_orgs[int(n.name)][3] == '1':
        #n.img_style = style2

style2 = NodeStyle()
style2["fgcolor"] = "#000000"
style2["shape"] = "circle"
style2["vt_line_color"] = "#0000aa"
style2["hz_line_color"] = "#0000aa"
style2["vt_line_width"] = 5
style2["hz_line_width"] = 5
style2["vt_line_type"] = 1 # 0 solid, 1 dashed, 2 dotted
style2["hz_line_type"] = 1
for l in t.iter_leaves():
    l.img_style = style2

ts = TreeStyle()
ts.show_leaf_name = True
#ts.rotation = 90
ts.mode = 'c'
ts.arc_start = 180
ts.arc_span = 350
#t.show(tree_style=ts)
#t.show()

t = Tree()
t.populate(8)
style2 = NodeStyle()
style2["fgcolor"] = "darkred"
style2["shape"] = "circle"
style2["vt_line_color"] = "green"
style2["hz_line_color"] = "red"
style2["vt_line_width"] = 5
style2["hz_line_width"] = 5
style2["vt_line_type"] = 1 # 0 solid, 1 dashed, 2 dotted
style2["hz_line_type"] = 1
for l in t.iter_leaves():
    l.img_style = style2
ts = TreeStyle()
ts.show_leaf_name = False
#
t.show(tree_style=ts)


from pyspark.context import SparkContext
import itertools
import collections
from collections import Counter
import time

from operator import add
import os
import sys

def gencomb(freqitem,lenfreqitem):
    cand= []
    for x in range(0,lenfreqitem-1):
         first = freqitem[x]
         firstelem = sorted(first)
         for y in range(x+1,lenfreqitem):
              second = freqitem[y]
              secelem= sorted(second)
              newItem = []
              if firstelem[0:len(firstelem)-1] ==  secelem[0:len(secelem)-1] :
                   newItem=list(firstelem)
                   newItem.append(secelem[len(secelem)-1])
                   newItem.sort()
                   cand.append((newItem))
    #print("candidate ",cand)
    return cand

def checksub(checksub, prev, len, initbasket, ktuple):
    store = Counter()
    candidates = []
    for k in checksub:
        item = set(k)
        count = 0
        for x in prev:
            if set(x).issubset(item):
                count += 1
            if count == ktuple:
                candidates.append(tuple(k))
                break

    candidates = list(dict.fromkeys(candidates))

    for basket in initbasket:
        bas = set(basket)
        for candidate in candidates:
            temp = set(candidate)
            if temp.issubset(bas):
                store[tuple(candidate)] += 1
    return store

def prune(checksubs, threshold):
    freq = []
    for x in checksubs:
        if checksubs[x] >= threshold:
            freq.append(tuple(x))
    return sorted(freq)

def apriori(basket, basket_count, sup):
    initbasket = list(basket)
    support = float(sup)
    partition_baskets = len(initbasket)
    #print("basket ", partition_baskets)
    total_baskets = basket_count
    threshold = support * (float(partition_baskets) / float(total_baskets))
    #print("THRESHOLD ", threshold)
    final_itemsets = []
    single_count = {}
    freqsingle = []
    single_print = []
    # SINGLTON

    for bas in initbasket:
        for item in bas:
            if item in single_count:
                single_count[item] += 1
            else:
                single_count[item] = 1

    for x in single_count:
        if single_count[x] >= threshold:
            freqsingle.append(x)
            single_print.append((x,))
    single = sorted(tuple(freqsingle))
    single_print = sorted(single_print)
    final_itemsets = single
    #print("single  ",len(single))
# PAIRS

    res = list(itertools.combinations(single, 2))
    #print("res ",len(res))
    paircount = collections.defaultdict(int)
    freqpair = []
    cand = []

    for basket in initbasket:
        bas = set(basket)
        for candidate in res:
            temp= set(candidate)
            if temp.issubset(bas):
                if paircount.get(candidate):
                    paircount[candidate] += 1
                else:
                    paircount[candidate] = 1

    for item,count in paircount.items():
        if count >= threshold:
            freqpair.append(tuple(sorted(item)))

    pair = sorted(freqpair)
    pair = list(dict.fromkeys(pair))
    #print("pair   ",len(pair))
    final_itemsets = final_itemsets + pair
    single_print = single_print + pair
    # print("pair   ",single_print)
    frequent = pair
    k = 3
    while frequent != []:

        gencombination = gencomb(frequent, len(frequent))
        # print(k," itemsets",len(gencombination))
        checksubsets = checksub(gencombination, frequent, len(gencombination), initbasket, k)
        # print("candidate ", len(checksubsets))
        freqitemsets = prune(checksubsets, threshold)
        #print(k, " -tuple freqitem ", len(freqitemsets))
        frequent = list(freqitemsets)
        if frequent != []:
            final_itemsets = final_itemsets + frequent
            single_print = single_print + frequent
        k += 1
    # print("Final itemsets ", single_print)

    yield (single_print)

def countmap2(bt):
    initbasket=list(bt)
    count = Counter()

    for basket in initbasket:
        bas=set(basket)
        for item in val:
            if item in bas:
                count[item] += 1
            else:
                temp = set(item)
                if temp.issubset(bas):
                    count[tuple(item)] += 1
    #print("countint ",list(count.items()))

    yield (list(count.items()))

if __name__ == "__main__":

    filterthreshold = sys.argv[1]
    support = sys.argv[2]
    input_path = sys.argv[3]
    output_path = sys.argv[4]

    sc= SparkContext("local[*]")
    sc.setLogLevel("ERROR")
    start = time.time()
    lines = sc.textFile(input_path)
    header = lines.first()
    RDD= lines.filter(lambda x: x != header).map(lambda x: tuple(x.split(",")))
    basket1= RDD.map(lambda x: (x[0],[x[1]])).reduceByKey(lambda x,y: x + y)
    bas=basket1.filter(lambda x: len(x[1]) > int(filterthreshold)).map(lambda x: (x[0],x[1]))
    bt=bas.values()
    basket_count = bt.count()

    mapbasket1 = bt.mapPartitions(lambda x: apriori(x,basket_count,support)).flatMap(lambda x: x)
    phase1_map= mapbasket1.map(lambda itemsets: (itemsets,1))

    phase1_reduce= phase1_map.reduceByKey(lambda x,y : 1).sortBy(keyfunc= lambda x: (len(x[0]),x[0]))
    val= phase1_reduce.keys().collect()
    #print("phase1reduce ", len(val))

    with open(output_path, 'w') as fileop:
        fileop.write("Candidates: \n")

        old = 1
        storestr = ""
        for index in range(len(val)):
            element = val[index]

            if index == len(val) - 1:
                fileop.write(storestr.replace(")(", "),("))
                if len(element) > old:
                    fileop.write("\n\n")
                    fileop.write(str(element))
                    break
                else:
                    fileop.write(","+str(element))
                    break

            if len(element) > old:
                fileop.write(storestr.replace(")(", "),("))
                fileop.write("\n\n")
                old += 1
                storestr = ""
                fileop.write(str(element))
            else:
                if len(element) == 1:
                    storestr += str(element).replace(",)", ")")
                else:
                    storestr += "," + str(element)
        fileop.close()

    phase2_map = bt.mapPartitions(countmap2).flatMap(lambda x: x).reduceByKey(add).sortBy(keyfunc= lambda x: (len(x[0]),x[0]))

    phase2_reduce = phase2_map.filter(lambda x: x[1] >= float(support)).map(lambda x: x[0])
    res = phase2_reduce.collect()
    #print("res ", len(res))

    with open(output_path, 'a') as fileop:
        fileop.write("\n\nFrequent Itemsets: \n")

        old = 1
        storestr = ""
        for index in range(len(res)):
            element = res[index]

            if index == len(res) - 1:
                fileop.write(storestr.replace(")(", "),("))
                if len(element) > old:
                    fileop.write("\n\n")
                    fileop.write(str(element))
                    break
                else:
                    fileop.write(","+str(element))
                    break

            if len(element) > old:
                fileop.write(storestr.replace(")(", "),("))
                fileop.write("\n\n")
                old += 1
                storestr = ""
                fileop.write(str(element))
            else:
                if len(element) == 1:
                    storestr += str(element).replace(",)", ")")
                else:
                    storestr += "," + str(element)
        fileop.close()
    end = time.time()
    print("Duration:", end - start)


from transformers.tokenization_utils import PreTrainedTokenizer
from transformers.utils.dummy_pt_objects import PreTrainedModel
from transformers.data.processors.utils import InputExample, InputFeatures

from .prompts import TemplateGenerator, VerbalizerGenerator
from openprompt import PromptDataLoader, PromptForClassification
from openprompt.prompts import ManualVerbalizer, LMBFFTemplate, ManualTemplate
from typing import List, Optional, Dict, Union
from . import Template, Verbalizer, PromptDataLoader
import copy
import warnings
from .trainer import ClassificationRunner
from yacs.config import CfgNode
from openprompt.utils.logging import logger
from openprompt.utils.cuda import model_to_device

class LMBFFClassificationRunner:
    def __init__(self,
                train_dataset: List[InputExample],
                valid_dataset: List[InputExample],
                test_dataset: List[InputExample],
                model: PreTrainedModel,
                tokenizer: PreTrainedTokenizer,
                template_generator_tokenizer: PreTrainedTokenizer,
                initial_template: Union[LMBFFTemplate, ManualTemplate],
                initial_verbalizer: ManualVerbalizer = None,
                template_generator: TemplateGenerator = None,
                verbalizer_generator: VerbalizerGenerator = None,
                config: CfgNode = None):
        r"""
        This class implements the LM-BFF in paper (https://arxiv.org/pdf/2012.15723.pdf)
        """
        self.model = model
        self.tokenizer = tokenizer
        self.template_generator_tokenizer = template_generator_tokenizer
        self.max_epoch = config.classification.generation_epoch
        self.train_dataset = train_dataset
        self.valid_dataset = valid_dataset
        self.test_dataset = test_dataset
        self.template_generator = template_generator
        self.verbalizer_generator = verbalizer_generator
        self.config = config
        self.metric = self.config.classification.metric[0]
        self.initial_template = initial_template
        self.initial_verbalizer = initial_verbalizer
        self.auto_t = config.classification.auto_t
        self.auto_v = config.classification.auto_v

        if self.auto_t:
            if not (initial_template and initial_verbalizer):
                raise ValueError("initial template and verbalizer must not be None when both auto_t and auto_v are performed")
            if not isinstance(initial_template, LMBFFTemplate):
                raise ValueError("To perform template search, initial_template must be LMBFFTemplate, however {} is provided".format(type(initial_template.__class__.__name__)))
        elif self.auto_v:
            if initial_verbalizer is not None:
                warnings.warn("only auto_v is True, the given initial_verbalizer is ignored")
            if isinstance(initial_template, LMBFFTemplate):
                raise ValueError("To perform template search, initial_template must be ManualTemplate, however LMBFFTemplate is provided")
        else:
            warnings.warn("auto_t and auto_v are both False, the trainer will degenerate to a simple classification trainer")

    def _auto_t(self, dataset, template, verbalizer):
        logger.info("performing auto-t...")
        if self.template_generator is None:
            raise ValueError("no template_generator available")
        dataloader = PromptDataLoader(dataset, template, self.template_generator_tokenizer, batch_size=len(dataset)) # register all data at once
        for data in dataloader:
            data = data.to("cuda:{}".format(self.config.environment.local_rank))
            self.template_generator.register_buffer(data.input_ids, data.attention_mask, data.label)
        template_texts = self.template_generator.generate() # [['text_a', '<mask>', ...]]
        best_template_text = self._get_best_template_text(template_texts, verbalizer)
        return best_template_text

    def _auto_v(self, dataset, template, verbalizer, batch_size=32):
        logger.info("performing auto-v...")
        if self.verbalizer_generator is None:
            raise ValueError("no verbalizer_generator available")
        dataloader = PromptDataLoader(dataset, template, self.tokenizer, batch_size=batch_size)
        for data in dataloader:
            data = template.process_batch(data)
            data = data.to("cuda:{}".format(self.config.environment.local_rank))
            self.verbalizer_generator.register_buffer(data)
        label_words_list = self.verbalizer_generator.generate() # List[List[str]]
        best_label_words = self._get_best_label_words(label_words_list, template, verbalizer)
        return best_label_words

    def run(self):
        '''
        if both auto_v and auto_t are set to True, perform auto_t first and then auto_v
        '''

        best_template = self.initial_template
        best_verbalizer = self.initial_verbalizer
        if self.auto_t:
            best_template_text = self._auto_t(self.train_dataset, self.initial_template, best_verbalizer)
            best_template = ManualTemplate(self.tokenizer, best_template_text)
        if self.auto_v:
            best_label_words = self._auto_v(self.train_dataset, best_template, best_verbalizer)
            best_verbalizer.label_words = best_label_words

        train_dataloader = PromptDataLoader(self.train_dataset, best_template, self.tokenizer)
        valid_dataloader = PromptDataLoader(self.valid_dataset, best_template, self.tokenizer)
        test_dataloader = PromptDataLoader(self.test_dataset, best_template, self.tokenizer)
        model = PromptForClassification(copy.deepcopy(self.model), best_template, best_verbalizer)
        model = model_to_device(model, self.config.environment)
        runner = ClassificationRunner(model, train_dataloader, valid_dataloader, test_dataloader, config=self.config)
        runner.run()

    def _get_best_template_text(self, template_texts_candidates, verbalizer):
        best_metrics = 0.0
        best_template_text = None
        for template_text in template_texts_candidates:
            print(template_text)
            template = ManualTemplate(self.tokenizer, template_text)
            train_dataloader = PromptDataLoader(self.train_dataset, template, self.tokenizer)
            valid_dataloader = PromptDataLoader(self.valid_dataset, template, self.tokenizer)
            score = self._train_eval(template, verbalizer, train_dataloader, valid_dataloader)
            if score > best_metrics:
                best_template_text = template_text
        return best_template_text

    def _get_best_label_words(self, verbalizer_labelwords_candidates, template, verbalizer):
        current_verbalizer = copy.deepcopy(verbalizer)
        best_metrics = 0.0
        best_label_words = None
        for label_words in verbalizer_labelwords_candidates:
            current_verbalizer.label_words = label_words
            train_dataloader = PromptDataLoader(self.train_dataset, template, self.tokenizer)
            valid_dataloader = PromptDataLoader(self.valid_dataset, template, self.tokenizer)
            score = self._train_eval(template, current_verbalizer, train_dataloader, valid_dataloader)
            if score > best_metrics:
                best_label_words = label_words
        return best_label_words

    def _train_eval(self, template, verbalizer, train_dataloader, valid_dataloader):
        model = PromptForClassification(copy.deepcopy(self.model), template, verbalizer)
        model = model_to_device(model, self.config.environment)
        runner = ClassificationRunner(model, train_dataloader, valid_dataloader, config=self.config)
        best_score = 0.0
        for epoch in range(self.max_epoch):
            runner.train_epoch(epoch)
            scores = runner.evaluate(valid_dataloader, 'Valid')
            score = scores[self.metric]
            if score > best_score:
                best_score = score
        return best_score