import streamlit as st
import time
import pandas as pd
# import convokit

# add session state variables
# if "node_list" not in st.session_state:
#     st.session_state["node_list"] = []

# if "edge_list" not in st.session_state:
#     st.session_state["edge_list"] = []

# to track the no of utterances left in the conversation.
if "index_rows" not in st.session_state:
    #Index to keep track of the rows in the datafrme
    st.session_state["index_rows"] = 0

    # Index yto keep track of the rows in a conversation
    st.session_state["index_utterances"] = 0

# Functional Requirements
st.write(
    '''
'''
)

# Uploading the file
st.write('''# Upload the file here''')
file = st.file_uploader("Upload file", type=['csv'])
if file is not None:
    file_df = pd.read_csv(file)
    file_df[["node_label", "edge_label", "remove"]] = None
    st.write(file_df)

st.write('''# Resource section''')
with st.expander("Click to expand info about the sentence categories"):
    st.write(
        """
        #### Agreement :
            - Agreeing to the point made before
            - Positive example/experience or confirming or acknowledging the made point
        #### Answer :
            - Any general response to a Question
            - Many comments underneath a question can be Answers,unless they look more like other options
        #### Appreciation
            - Just praise, appreciation or excitement
            - Has less intent of educating others.

        #### Disagreement
            - comment that is correcting, criticizing, contradicting, or objecting.
            - can also be an experience or story that expresses disagreement
        #### Elaboration
            - Adding extra relevant information to the previous comment
            - A good test would be if I just add this to the previous comment, the flow of the idea remains intact.
        #### Humour
            - A joke, piece of SaRcAsM or just silly puns
            - Could be a fun story
            - Intent here is just to have fun, for example if that joke supports then it might belong in other category
        #### Negative Reaction
            - attacking or mocking the commenter, or expressing emotions like disgust, derision, or anger
            - Has no intent of adding value or justification or correction
        #### Question
            - Is it a question or a request seeking forfeedback, help, or other kinds of responses.
            - Remember not every comment that ends with a "?" is a question
            - If it ends with a "?" does not mean it is a question

        """
    )
with st.expander("Click to expand the relations between the sentences"):
    st.write(
        '''
        #### Asking
            - Ask questions of the other person in a conversation to discover information about them, their situation and what they do or do not know.
        #### Informing
            - Providing information to others helps them understand what you understand and enables them to make informed, sensible decisions.
        #### Asserting
            - Asserting is stating something as if it were true.
            -When a person asserts something, they are also sending a message that they do not want to have an argument whether it is true or not.
        #### Proposing
        #### Summarizing
        #### Checking
        #### Building
        #### Including
        #### Excluding
        #### Self-promotion
        #### Supporting
        #### Disagreeing
        #### Avoiding
        #### Challenging
        #### Attacking
        #### Defendin
        '''
    )

node_list = []
edge_list = []

if file:
    st.write('''# Labelling section''')
    # What to do when this is the first utterance here
    st.warning(' The title of the conversation')

    with st.form('Node1_input'):
        # st.write(st.session_state["index_utterances"])
        # st.info(file_df.iloc[st.session_state["index_utterances"]][2])
        node = st.selectbox('Node_1', ['Agreement',
                                       'Announcement',
                                       'Answer',
                                       'Appreciation',
                                       'Disagreement',
                                       'Elaboration',
                                       'Humour',
                                       'Negative Reaction',
                                       'Question'], key=1)
        sub1 = st.form_submit_button('Submit 1')
        file_df["node_label"][st.session_state["index_utterances"]] = node
        keep_node1 = st.checkbox('Should we exclude this one?', key=2)
        st.write(keep_node1)
        file_df["remove"][st.session_state]

    # Sanity check
    st.write(file_df)
    with st.form('Node2_input'):
        st.info(""" Utterance 2 """)
        node = st.selectbox('Node_2', ['Agreement',
                                       'Announcement',
                                       'Answer',
                                       'Appreciation',
                                       'Disagreement',
                                       'Elaboration',
                                       'Humour',
                                       'Negative Reaction',
                                       'Question'], key=1)
        sub1 = st.form_submit_button('Submit 2')
        keep_node1 = st.checkbox('Should we excluse this one?', key=2)

    # only show it when the count of the utterances is more than one.
    edge = st.selectbox("What kind of realtionship exists between these sentenes", ['Asking',
                                                                                'Informing',
                                                                                'Asserting',
                                                                                'Proposing',
                                                                                'Summarizing',
                                                                                'Checking',
                                                                                'Building',
                                                                                'Including',
                                                                                'Excluding',
                                                                                'Self-promotion',
                                                                                'Supporting',
                                                                                'Disagreeing',
                                                                                'Avoiding',
                                                                                'Challenging',
                                                                                'Attacking',
                                                                                'Defending'], key=3)

    next = st.button("Next")
    if next:
        st.write("Clicked")
        # if index_utterances + 1 > last_page:
        #     index_utterances = 0
        # else:
        #     index_utterances += 1

    # Stuff to do when it is continous

    # stuff when you reach the end of the file.


import unittest
from models.base import Base
from models.rectangle import Rectangle
from models.square import Square

class TestBaseConstructor(unittest.TestCase):
    def test_no_parameters(self):
        b = Base()
        self.assertEqual(b.id, 1)

        b = Base()
        self.assertEqual(b.id, 2)

    def test_id(self):
        b = Base(89)
        self.assertEqual(b.id, 89)

    def test_more_parameters(self):
        with self.assertRaises(TypeError):
            # too many arguments
            b = Base(1, 2)

class TestToDictionary(unittest.TestCase):
    def correct_arguments(self):
        b = Base()

        with self.assertRaises(NotImplementedError):
            b.to_dictionary()

    def incorrect_arguments(self):
        b = Base()

        with self.assertRaises(TypeError):
            b.to_dictionary("hi")

class TestUpdate(unittest.TestCase):
    def test_always_raises_not_implemented(self):
        b = Base()

        with self.assertRaises(NotImplementedError):
            b.update()
            b.update("GOD")
            b.update("IS")
            b.update("NEVER")
            b.update("GONNA")
            b.update("GIVE YOU UP")
            b.update("never", "gonna", let="you down")
            b.update(range(0, 10), int_=int)

class TestToJSONString(unittest.TestCase):
    def test_wrong_amount_of_arguments(self):
        with self.assertRaises(TypeError):
            Base.to_json_string()

        with self.assertRaises(TypeError):
            Base.to_json_string(9, 8)

    def test_none(self):
        result = Base.to_json_string(None)
        self.assertEqual(result, "[]")

    def test_correct_type(self):
        pass

class TestFromJsonString(unittest.TestCase):
    def test_wrong_args(self):
        with self.assertRaises(TypeError):
            Base.from_json_string()

        with self.assertRaises(TypeError):
            Base.from_json_string("h", "i")


#!/usr/bin/python3

from bcc import BPF

prog = '''
int hello(void *ctx)
{
    bpf_trace_printk("hello world\\n");
    return 0;
}
'''

b = BPF(text=prog)
# get_syscall_fnname: 根据输入的名称获取系统调用的全名, For example, given "clone" the helper would return "sys_clone" or "__x64_sys_clone".
b.attach_kprobe(event=b.get_syscall_fnname("clone"), fn_name="hello")

# header
print("%-18s %-16s %-6s %s" % ("TIME(s)", "COMM", "PID", "MESSAGE"))

# 循环获取输出
while(1):
    try:
        # trace_fields 从trace_pipe中读取字段返回
        (task, pid, cpu, flags, timestamp, msg) =  b.trace_fields()
    except ValueError:
        continue
    print("%-18.9f %-16s %-6d %s" % (timestamp, task, pid, msg))


# recursive_circle_direct.py
#
# Jysaw demo - draws recursive circles
#
# Attribution: inspired by the recursion demo in Processing
# from http://processingjs.org/learning/basic/recursion/
# written by Casey Reas and Ben Fry

from jygsaw.graphics import *

# Recursively draw smaller circles
def draw_circle(x, radius, level):
    tt = int(126 * level / 4.0)
    fill(tt)
    circle(x, height() / 2, radius)
    if (level > 1):
        level = level - 1
        draw_circle(x - radius / 2, radius / 2, level)
        draw_circle(x + radius / 2, radius / 2, level)

canvas(640, 360)
no_stroke()

draw_circle(width() / 2, 280, 6)


from decimal import *
class Event:
	def __init__(self, line):
		try:
			data = line.split(' ')
			self.name = data[0]
			self.start = int(data[1])
			self.end = int(data[2])
			self.duration = self.end - self.start
			self.people = int(data[3])
			self.partial_score = self.duration * self.people
			self.compatible_rooms = []
			self.compatible_next_events = []
		except:
			print(line)
			raise

class Room:
	def __init__(self, line):
		try:
			data = line.split(' ')
			self.name = data[0]
			self.places = int(data[1])
		except:
			print(line)
			raise

def unallocatedTime(schedule):
	return ((schedule[-1].end - schedule[0].start)-sum([e.duration for e in schedule]))

def score(schedule, room):
	s = sum([e.people*e.duration/room.places for e in schedule])
	s -= (room.places/max_room_cap)*unallocatedTime(schedule)
	return s

events = []
rooms = []
with open('FIND ROOMS FOR EVENTS/data_50000_100.in','r') as file:
	eventCount, roomCount = [int(k) for k in file.readline().strip().split(' ')]
	for i in range(eventCount):
		line = file.readline().strip()
		events.append(Event(line))
	for i in range(roomCount):
		line = file.readline().strip()
		rooms.append(Room(line))
max_room_cap = max([r.places for r in rooms])
events = [e for e in events if e.people <= max_room_cap]

i = 1
for e in events:
	i *= len([r for r in rooms if r.places >= e.people])
getcontext().prec = 6
a = Decimal(i) / Decimal(1)
print(a)

print('Number of Hostable Events:',len(events))
print('Number of Rooms:',len(rooms))
print('Maximum room capacity',max_room_cap)

STANDARD CODE CHALLENGE 2017 - FIND ROOMS FOR EVENTS/main.py


import scrapy

class YelpSpider(scrapy.Spider):
    name = 'scrape_yelp'

    custom_settings = {
        # "DOWNLOAD_DELAY": 3,
        "CONCURRENT_REQUESTS_PER_DOMAIN": 3,
        "HTTPCACHE_ENABLED": True
    }
    start_urls = ['https://www.yelp.com/search?find_desc=Restaurants&find_loc=Brooklyn%2C+NY']

    def parse(self, response):
        # Extract the links to the individual restaunant
        static_yelp_url = 'https://www.yelp.com'

        for href in response.xpath('//h3/span/a/@href').extract():
            href = static_yelp_url + href
            yield scrapy.Request(
                url=href, callback=self.parse_yelp,
                meta={'url':href})

        # Follow pagination links and repeat
        next_url = response.xpath('//a[@class="u-decoration-none next pagination-links_anchor"]/@href').extract()[0]
        next_url = static_yelp_url + next_url
        yield scrapy.Request(url=next_url,callback=self.parse)

    def parse_yelp(self,response):
        url = response.request.meta['url']
        try:
            name = response.xpath('//h1[@class="biz-page-title embossed-text-white shortenough"]/text()').extract()[0].strip()
        except:
            name = None
        try:
            rating = response.xpath('//div[@class="biz-rating biz-rating-very-large clearfix"]/div/@title').extract()[0].strip()
        except:
            rating = None
        try:
            address = response.xpath('//strong[@class="street-address"]/address/text()').extract()[0].strip()
        except:
            address = None
        try:
            review_count = response.xpath('//span[@class="review-count rating-qualifier"]/text()').extract()[0].strip()
        except:
            review_count = None
        try:
            transit = response.xpath('//b[@class="transit-line inline-block"]/text()').extract().strip()
        except:
            transit = None
        try:
            category = response.xpath('//span[@class="category-str-list"]/a/text()').extract()
            if len(category)%2 == 0:
                idx = len(category)/2
            else:
                idx = (len(category)/2)+1
            category = category[:idx]
        except:
            category = None
        try:
            price_range = response.xpath('//span[@class="business-attribute price-range"]/text()').extract()[0]
        except:
            price_range = None
        try:
            hours = response.xpath('//table[@class="table table-simple hours-table"]/tbody/tr').extract()
        except:
            hours = None
        try:
            price_desc = response.xpath('//dd[@class="nowrap price-description"]/text()').extract()[0].strip()
        except:
            price_desc = None
        try:
            attribute_answers = response.xpath('//dt[@class="attribute-key"]/../dd').extract()[2:].strip()
        except:
            attribute_answers = None
        try:
            attribute_key = response.xpath('//dt[@class="attribute-key"]').extract()[2:].strip()
        except:
            attribute_key = None
        yield{
        'url': url,
        'name': name,
        'rating': rating,
        'address': address,
        'review_count': review_count,
        'transit': transit,
        'category': category,
        'price_range': price_range,
        'hours': hours,
        'price_desc': price_desc,
        'attribute_key': attribute_key,
        'attribute_answers': attribute_answers
        }


from helpers import run

def hit(x0, x1, y0, y1, vx, vy):
    x = 0
    y = 0
    while x <= x1 and (y >= y0 or vy > 0):
        x += vx
        y += vy
        if x in range(x0, x1 + 1) and y in range(y0, y1 + 1):
            return True
        if vx > 0:
            vx -= 1
        vy -= 1
    return False

def solve(inp):
    _, _, xr, yr = inp.split()
    x0, x1 = xr[2:-1].split('..')
    x0 = int(x0)
    x1 = int(x1)
    y0, y1 = yr[2:].split('..')
    y0 = int(y0)
    y1 = int(y1)
    best_h = None
    if y0 >= 0 or y1 >= 0:
        raise Exception("unimplemented")
    cnt = 0
    for vx in range(0, x1 + 1):
        for vy in range(y0 - 1, -y0 + 1):
            if hit(x0, x1, y0, y1, vx, vy):
                h = 0 if vy <= 0 else vy * (vy + 1) // 2
                if best_h is None or best_h < h:
                    best_h = h
                cnt += 1
    print('part 1:', best_h)
    print('part 2:', cnt)

run(17, solve)


# baekjoon 20040

import sys
input = sys.stdin.readline
N, M = map(int, input().split())
lines = []
for i in range(1, M+1): lines.append(list(map(int, input().split())))

parent = [ i for i in range(N) ]

def find(n):
  if n == parent[n]: return n
  else :
    parent[n] = find(parent[n])
    return parent[n]

def union(x, y):
  px, py = find(x), find(y)
  if px == py : return True
  else :
    parent[max(px, py)] = min(px, py)
    return False

ans = 0
for i in range(1, M+1):
  if union(*lines[i-1]):
    ans = i
    break
print(ans)


# storing current game state, check valid moves, keep move log

import numpy as np

class GameState():
    def __init__(self):
        self.dimension = 8
        self.board = np.zeros([self.dimension, self.dimension])
        # 0 is empty space , -1 is black , 1 is red
        for i in range(2):
            for j in range(8):
                if (i % 2 == 0 and j % 2 == 0) or (i % 2 == 1 and j % 2 == 1):
                    self.board[i][j] = -1
                    self.board[i+6][j] = 1
        self.redToMove = True
        self.moveLog = []
        self.turn = 0

    def makeMove(self, move):
        self.board[move.startRow][move.startCol] = 0
        self.board[move.destRow][move.destCol] = move.pieceMoved
        if move.pieceCapture[1] != -100:
            self.board[move.pieceCapture[0][0]][move.pieceCapture[0][1]] = 0
        self.checkPromote(move)
        self.moveLog.append(move)

    def checkPromote(self, move):
        if self.board[move.destRow][move.destCol] == 1 and move.destRow == 0:
            self.board[move.destRow][move.destCol] = 2
            self.redToMove = not self.redToMove
        elif self.board[move.destRow][move.destCol] == -1 and move.destRow == 7:
            self.board[move.destRow][move.destCol] = -2
            self.redToMove = not self.redToMove
        elif move.pieceCapture[1] == -100 or move.pieceCapture[1] == 0:
            self.redToMove = not self.redToMove
        elif move.pieceCapture != -100 and move.pieceCapture != 0 and not self.is_force_move(move.destRow, move.destCol, "BOOL"):
            self.redToMove = not self.redToMove

    def is_force_move(self, r, c, str):
        moves = []
        force_moves = []

        if abs(self.board[r][c]) == 1:
            self.getNormalMove(r, c, moves, force_moves)
        elif abs(self.board[r][c]) == 2:
            self.getBigMove(r, c, moves, force_moves)

        if str == "BOOL":
            return len(force_moves) != 0
        elif str == "MOVE":
            return force_moves

    def undoMove(self):
        if len(self.moveLog) != 0:
            move = self.moveLog.pop()
            self.board[move.startRow][move.startCol] = move.pieceMoved
            self.board[move.destRow][move.destCol] = 0
            if move.pieceCapture[1] != -100 and move.pieceCapture[1] != 0:
                self.board[move.pieceCapture[0][0]][move.pieceCapture[0][1]] = move.pieceCapture[1]
            self.redToMove = move.player

    def getValidMove(self):
        moves = []
        force_moves = []

        if len(self.moveLog) != 0 and self.redToMove == self.moveLog[-1].player:
            return self.is_force_move(self.moveLog[-1].destRow, self.moveLog[-1].destCol, "MOVE")

        for r in range(self.dimension):
            for c in range(self.dimension):
                piece = self.board[r][c]
                if (self.redToMove and piece == 1) or (not self.redToMove and piece == -1):
                    self.getNormalMove(r, c, moves, force_moves)
                elif (self.redToMove and piece == 2) or (not self.redToMove and piece == -2):
                    self.getBigMove(r, c, moves, force_moves)

        if len(force_moves) != 0:
            return force_moves
        else:
            return moves

    def getNormalMove(self, r, c, moves, force_moves):
        allow_change = []
        d = int(self.board[r][c]) * -1
        for i in range(-2, 3):
              if i != 0 and c + i >= 0 and c + i < 8:
                   allow_change.append(i)
        for i in allow_change:
            if abs(i) == 1 and self.board[r+d][c+i] == 0:
                moves.append(Move((r, c), (r+d, c+i), self.board, self.redToMove))
            elif i == -2 and ((self.board[r][c] > 0 and self.board[r+d][c-1] < 0) or (self.board[r][c] < 0 and self.board[r+d][c-1] > 0)):
                if r+d+d >= 0 and r+d+d < 8 and self.board[r+d+d][c+i] == 0:
                        force_moves.append(Move((r, c), (r+d+d, c+i), self.board, self.redToMove))
            elif i == 2 and ((self.board[r][c] > 0 and self.board[r+d][c+1] < 0) or (self.board[r][c] < 0 and self.board[r+d][c+1] > 0)):
                if r+d+d >= 0 and r+d+d < 8 and self.board[r+d+d][c+i] == 0:
                        force_moves.append(Move((r, c), (r+d+d, c+i), self.board, self.redToMove))

    def getBigMove(self, r, c, moves, force_moves):
        direction = [(1, 1), (1, -1), (-1, 1), (-1, -1)]
        for d in direction:
            x = r + d[0]
            y = c + d[1]
            first_force = False
            while(x >= 0 and x < 8 and y >= 0 and y < 8):
                if not first_force:
                    if self.board[x][y] == 0:
                        moves.append(Move((r, c), (x, y), self.board, self.redToMove))
                    elif (self.board[r][c] > 0 and self.board[x][y] < 0) or (self.board[r][c] < 0 and self.board[x][y]):
                        if x+d[0] >= 0 and x+d[0] < 8 and y+d[1] >= 0 and y+d[1] < 8 and self.board[x+d[0]][y+d[1]] == 0:
                                force_moves.append(Move((r, c), (x+d[0], y+d[1]), self.board, self.redToMove))
                        first_force = True
                    else:
                        break
                else:
                    break
                x += d[0]
                y += d[1]

class Move():

    ranksToRows = {"1": 7, "2": 6, "3": 5, "4": 4, "5": 3, "6": 2, "7": 1, "8": 0}
    rowsToRanks = {v: k for k, v in ranksToRows.items()}
    filesToCols = {"a": 0, "b": 1, "c": 2, "d": 3, "e": 4, "f": 5, "g": 6, "h": 7}
    colsToFiles = {v: k for k, v in filesToCols.items()}

    def __init__(self, startSq, destSq, board, player):
        self.startRow = startSq[0]
        self.startCol = startSq[1]
        self.destRow = destSq[0]
        self.destCol = destSq[1]
        self.moveID = self.startRow * 1000 + self.startCol * 100 + self.destRow * 10 + self.destCol
        self.moveDirection = (((self.destRow - self.startRow) // abs(self.destRow - self.startRow)), ((self.destCol - self.startCol) // abs(self.destCol - self.startCol)))
        self.moveDistance = abs(self.startRow - self.destRow)
        self.pieceMoved = board[self.startRow][self.startCol]
        if self.moveDistance > 1:
            self.pieceCapture = ((self.destRow - self.moveDirection[0], self.destCol - self.moveDirection[1]), board[self.destRow - self.moveDirection[0]][self.destCol - self.moveDirection[1]])
        else:
            self.pieceCapture = ((-100, -100), -100)
        self.player = player

    # compare class

    def __eq__(self,other):
        if isinstance(other, Move):
            return self.moveID == other.moveID
        return False

    def getChessNotation(self):
        return self.getRankFile(self.startRow, self.startCol) + " " + self.getRankFile(self.destRow, self.destCol)

    def getRankFile(self, r, c):
        return self.colsToFiles[c] + self.rowsToRanks[r]


import random
import os

import keras
import numpy as np
from keras.callbacks import LambdaCallback
from keras.models import Input, Model, load_model
from keras.layers import LSTM, Dropout, Dense, Flatten, Bidirectional, Embedding, GRU
from keras.optimizers import Adam
import math

from data_utils import *

class PoetryModel(object):
    def __init__(self, config, gr):
        self.model = None
        self.config = config
        self.gr = gr

        # 文件预处理
        self.word2numF, self.num2word, self.words, self.files_content = preprocess_file(self.config)

        # 加载embedding
        self.embedding_matrix = embedding_file(self.config, self.words)

        # 加载模型
        if os.path.exists(self.config.weight_file):
            self.model = load_model(self.config.weight_file)
        else:
            self.build_model()
        self.model.summary()

    def build_model(self):
        '''
        建立模型
        '''
        print("---building new model---")
        cfg = self.config
        # 输入的dimension
        input_tensor = Input(shape=(cfg.max_len,))
        embedd = Embedding(len(self.num2word) + 2,
                           cfg.embedding_size,
                           weights=[self.embedding_matrix],
                           input_length=cfg.max_len,
                           trainable=False)(input_tensor)
        lstm = Bidirectional(GRU(cfg.rnn_1_size, return_sequences=True))(embedd)
        dropout = Dropout(cfg.dropout_rate)(lstm)
        lstm = GRU(cfg.rnn_2_size, return_sequences=True)(dropout)
        dropout = Dropout(cfg.dropout_rate)(lstm)
        flatten = Flatten()(dropout)
        dense = Dense(len(self.words)+1, activation='softmax')(flatten)
        self.model = Model(inputs=input_tensor, outputs=dense)
        optimizer = Adam(lr=cfg.learning_rate)
        self.model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])

    def data_generator(self, batch_size):
        '''
        数据生成器
        '''
        data = self.files_content
        i, data_len, max_len = 0, len(data), self.config.max_len
        while True:
            x_data, y_data = [], []
            for _ in range(batch_size):
                while i + max_len < data_len and data[i: min(data_len - 1, i + max_len + 1)].find(']') != -1: i += 1
                if i + max_len == data_len: break

                x = self.files_content[i: i + self.config.max_len]
                y = self.files_content[i + self.config.max_len]

                y_vec = np.zeros(
                    shape=(len(self.words)+1,),
                    dtype=np.bool
                )
                y_vec[self.word2numF(y)] = 1.0

                x_vec = np.zeros(
                    shape=(max_len,),
                    dtype=np.int32
                )
                for t, char in enumerate(x):
                    x_vec[t] = self.word2numF(char)

                x_data.append(x_vec)
                y_data.append(y_vec)
                i += 1
            if len(x_data) == 0:
                i = 0
                continue
            yield np.array(x_data), np.array(y_data)
        return

    def train(self, epoch_num):
        '''
        训练模型
        '''
        def calc_number_of_batch(data, batch_size, max_len):
            cnt = 0
            data_len = len(data)
            for i in range(data_len):
                if i + max_len < data_len and data[i: min(data_len-1, i + max_len + 1)].find(']') == -1: cnt += 1
            print(len(data), cnt, int(math.ceil(cnt / batch_size)))
            return int(math.ceil(cnt / batch_size))
        number_of_batch = calc_number_of_batch(self.files_content, self.config.batch_size, self.config.max_len)

        tbCallback = keras.callbacks.TensorBoard(log_dir=self.config.tb_file,
                                                 histogram_freq=0,
                                                 write_graph=True,
                                                 write_images=True)
        self.model.fit_generator(
            generator=self.data_generator(self.config.batch_size),
            verbose=True,
            steps_per_epoch=number_of_batch,
            epochs=epoch_num,
            callbacks=[
                keras.callbacks.ModelCheckpoint(self.config.weight_file, save_weights_only=False),
                LambdaCallback(on_epoch_end=self.generate_sample_result),
                tbCallback
            ]
        )

    def generate_sample_result(self, epoch, logs):
        '''
        训练过程中，每个epoch打印出当前的学习情况
        '''
        # if epoch % 5 != 0:
        #     return
        print("\n==================Epoch {}=====================".format(epoch+1))
        for diversity in [0.5, 1.0, 1.5]:
            print("------------Diversity {}--------------".format(diversity))
            start_index = random.randint(0, len(self.files_content) - self.config.max_len - 1)
            generated = ''
            sentence = self.files_content[start_index: start_index + self.config.max_len]
            generated += sentence
            for i in range(20):
                x_pred = np.zeros((1, self.config.max_len))
                for t, char in enumerate(sentence[-self.config.max_len:]):
                    x_pred[0, t] = self.word2numF(char)

                preds = self.model.predict(x_pred, verbose=0)[0]
                next_index = self.sample(preds, diversity)
                next_char = self.num2word[next_index]

                generated += next_char
                sentence = sentence + next_char
            print(sentence)

    def sample(self, preds, temperature=1.0, nline=0, pos=2, p_model=0, used=list()):
        '''
        当temperature=1.0时，模型输出正常
        当temperature=0.5时，模型输出比较open
        当temperature=1.5时，模型输出比较保守
        在训练的过程中可以看到temperature不同，结果也不同
        '''

        '''
        平起首句押韵平平仄仄平，仄仄仄平平。仄仄平平仄，平平仄仄平。
        仄起首句押韵仄仄仄平平，平平仄仄平。平平平仄仄，仄仄仄平平
        '''
        # 改为下标从0开始
        preds = preds[1:]

        pm = [[[0, 0, 1, 1, 0], [1, 1, 1, 0, 0], [1, 1, 0, 0, 1], [0, 0, 1, 1, 0]],
            [[1, 1, 1, 0, 0], [0, 0, 1, 1, 0], [0, 0, 0, 1, 1], [1, 1, 1, 0, 0]]]

        preds = np.asarray(preds).astype('float64')
        preds = np.log(preds) / temperature
        exp_preds = np.exp(preds)
        preds = exp_preds / np.sum(exp_preds)

        if pos == 5:
            if nline & 1:
                return self.word2numF('，')
            else:
                return self.word2numF('，')

        ans = -1
        maxp = 0
        pre = preds.tolist()
        for n in range(len(pre)):
            thischar = self.num2word[n]
            # print(thischar)
            if thischar not in self.gr.rhyme.keys():
                continue
            if thischar not in used and pre[n] > maxp and \
                    (nline == 0 or pos != 4 or self.gr.rhyme[thischar] == self.gr.rhy) and \
                    self.gr.pitch[thischar] == pm[p_model][nline][pos]:
                maxp = pre[n]
                ans = n

        if ans == -1:
            # print('not found')
            probas = np.random.multinomial(1, preds, 1)
            ans = np.argmax(probas)
            if nline == 0 and pos == 4:
                while self.num2word[ans] not in self.gr.rhyme.keys():
                    probas = np.random.multinomial(1, preds, 1)
                    ans = np.argmax(probas)

        if nline == 0 and pos == 4:
            self.gr.rhy = self.gr.rhyme[self.num2word[ans]]

        # print(nline, pos, self.num2word[ans], 'rhy = ',
        #      self.gr.rhyme[self.num2word[ans]], 'rhyme should be ',
        #      self.gr.rhy, 'pitch = ', self.gr.pitch[self.num2word[ans]], 'pitch should be ', pm[p_model][nline][pos])

        return ans

    def predict(self, text):
        '''
        根据给出的文字，生成诗句
        '''
        def is_valid(text):
            if len(text) != 4: return False
            if text[0] not in self.gr.rhyme.keys(): return False
            return True
        if not is_valid(text):
            return 'illegal input'
        used = list()
        self.gr.rhy = -1
        with open(self.config.poetry_file, 'r', encoding='utf-8') as f:
            file_list = f.readlines()
        random_line = random.choice(file_list)

        pitch_model = 0
        if self.gr.pitch[text[0]] == 0:
            pass
        else:
            pitch_model = 1

        # 如果给的text不到四个字，则随机补全
        if not text or len(text) != 4:
            for _ in range(4 - len(text)):
                random_str_index = random.randrange(0, len(self.words))
                if self.num2word.get(random_str_index) not in [',', '。', '，']:
                    text += self.num2word.get(random_str_index)
                else:
                    text += self.num2word.get(random_str_index + 1)
        seed = random_line[-self.config.max_len:-1]

        res = ''

        seed = 'c' + seed
        nl = 0
        for c in text:
            seed = seed[1:] + c
            if nl:
                for i in range(0, 5):
                    used.append(seed[i])
            for j in range(5):
                x_pred = np.zeros((1, self.config.max_len))
                for t, char in enumerate(seed):
                    x_pred[0, t] = self.word2numF(char)

                preds = self.model.predict(x_pred, verbose=0)[0]
                next_index = self.sample(preds, 1.0, nl, j+1, pitch_model, used)
                # print(next_index)
                next_char = self.num2word[next_index]
                seed = seed[1:] + next_char
            res += seed
            nl += 1
        res = res[:5]+','+res[6:17]+','+res[18:]
        return res

def main():
    from config import Config
    from get_ryhme import Ryhme

    model = PoetryModel(Config, Ryhme(Config))

    # 确定是否训练
    opt = int(input('please input the number of epochs to train or 0 to write a poem\n'))
    if opt != 0:
        model.train(opt)
    model.predict('蛤蛤蛤蛤')

    # 念诗
    while True:
        text = input('text: (输入*退出)')
        if text == '*': break

        sentence = model.predict(text)
        print(sentence)

if __name__ == '__main__':
    main()


# import MongoClient
import pymongo

myclient = pymongo.MongoClient("mongodb://localhost:27017/")
mydb = myclient['movie_db']
movies = myclient.movies

movie_ = {
    "Title":    "Mr.Robot",
    "Starting": "Rami Malek, Christian Slater, Carly Chaikin",
    "Created":  "Sam Esmail",
    "Year":     2016
}
# mycol = mydb["customer"]
id = movies.insert_one(movie_).inserted_id
print(id)
mydb.list_collection_names()

print(myclient.list_database_names())


import os
import shutil
import time

def remove_contents(path):
    for root, dirs, files in os.walk(path):
      for file in files:
        print(f'Удаля файл {file}')
        os.remove(file)
        time.sleep(1)
if __name__ == "__main__":
    remove_contents(r'logs back')


import copy
import os
import json
import datetime
import time

from decimal import Decimal
from typing import List
from tortoise import transactions
from fastapi import APIRouter, status, Body, Depends, Request
from fastapi.encoders import jsonable_encoder
from tortoise.expressions import Q
from tortoise.functions import Max
from tortoise.transactions import in_transaction
from core.settings import TIMES
from core.middleware import limiter
from utils import utils
from .job_helper import *
from .schema import CreateJob, UpdateJob
from .model import Jobs
from ..auth.auth_casbin import AuthorityRole
from ..car.model import Cars, CarSensors
from ..controller.model import Controllers
from ..dynamic.model import Dynamics
from ..log.model import Logs
from ..scenario.model import Scenarios
from ..scenario.scenario_helper import json_convert_to_openscenario
from ..sensor.model import Sensors, SensorData
from ..task.model import Tasks
from utils.auth import request
from .job_helper import get_nodes_status
from .report_tool.report_helper import Report
from starlette.responses import FileResponse
from pathlib import Path

router = APIRouter()
rgb_for_replay = {
    "type": "sensor.camera.rgb",
    "id": "view",
    "role_name": "record_rgb",
    "spawn_point": {"x": -7, "y": 0, "z": 2, "roll": 0, "pitch": -15, "yaw": 0},
    "image_size_x": 800,
    "image_size_y": 600,
    "fov": 90,
    "sensor_tick": 0.05,
    "gamma": 2.2,
    "shutter_speed": 200,
    "iso": 100,
    "fstop": 8,
    "min_fstop": 1.2,
    "blade_count": 5,
    "exposure_mode": "histogram",
    "exposure_compensation": 0,
    "exposure_min_bright": 7,
    "exposure_max_bright": 9,
    "exposure_speed_up": 3,
    "exposure_speed_down": 1,
    "calibration_constant": 16,
    "focal_distance": 1000,
    "blur_amount": 1,
    "blur_radius": 0,
    "motion_blur_intensity": 0.45,
    "motion_blur_max_distortion": 0.35,
    "motion_blur_min_object_screen_size": 0.1,
    "slope": 0.88,
    "toe": 0.55,
    "shoulder": 0.26,
    "black_clip": 0,
    "white_clip": 0.04,
    "temp": 6500,
    "tint": 0,
    "chromatic_aberration_intensity": 0,
    "chromatic_aberration_offset": 0,
    "enable_postprocess_effects": "True",
    "lens_circle_falloff": 5,
    "lens_circle_multiplier": 0,
    "lens_k": -1,
    "lens_kcube": 0,
    "lens_x_size": 0.08,
    "lens_y_size": 0.08,
    "bloom_intensity": 0.675,
    "lens_flare_intensity": 0.1,
    "sensor_id": "default",
    "sensor_name": "default_cam",
    "nickname": "RGB相机"
}

@router.post(
    "/create_job",
    summary='创建作业',
    tags=["Job"]
)
@limiter.limit("%d/minute" %TIMES)
async def create_job(request: Request, job_create_model: CreateJob):
    """create a new job.
    car_id 和 scenario_ids 存入，后面直接查询对应的表，不要快照，同时生成task表，吧scenario信息存进去"""
    user = request.state.user
    assert not await Jobs.filter(Q(company_id=user.company_id) | Q(system_data=1), name=job_create_model.name,
                                 invalid=0).first(), '作业名称已存在'
    car_obj = await Cars.filter(Q(company_id=user.company_id) | Q(system_data=1), id=job_create_model.car_id,
                                invalid=0).first()
    assert car_obj, '主车信息不存在'
    car_sensor = await CarSensors.filter(car_id=job_create_model.car_id, invalid=0).first()
    assert car_sensor, '主车上没有添加传感器'
    sensor_obj = await Sensors.filter(Q(company_id=user.company_id) | Q(system_data=1), id=car_sensor.sensor_id,
                                      invalid=0).first()
    assert sensor_obj, '传感器信息不存在'
    scenario_infos = await Scenarios.filter(id__in=job_create_model.scenario_ids, company_id=user.company_id, invalid=0).all()
    assert len(scenario_infos) == len(job_create_model.scenario_ids), '场景存在异常，检查是否有无效场景'

    async with transactions.in_transaction():
        job = Jobs(**job_create_model.dict())
        job.status = "waiting"
        job.user_id = user.id
        job.company_id = user.company_id
        await job.save()
        job_id = job.id
        job_name = job.name
        task_snaps = []
        for scenario_id in job_create_model.scenario_ids:
            scenario = await Scenarios.filter(id=scenario_id, company_id=user.company_id, invalid=0).first()
            task_snap = Tasks()
            task_snap.desc = scenario.desc
            task_snap.user_id = user.id
            task_snap.job_id = job_id
            task_snap.company_id = user.company_id
            task_snap.job_name = job_name
            assert len(scenario.name) <= 64, "场景名称超过了64个字符"
            task_snap.name = scenario.name
            task_snap.scenario_id = scenario.id
            task_snap.scenario_tags = scenario.tags
            task_snap.status = "waiting"
            task_snaps.append(task_snap)

        await Tasks.bulk_create(task_snaps)
    return job_id

@router.put(
    "/update_job/{job_id}",
    summary='更新作业信息',
    tags=["Job"]
)
@limiter.limit("%d/minute" %TIMES)
async def update_job(request: Request, job_id: int, job_update_model: UpdateJob):
    """update job， 这个时候更新job表里面的所有信息，重新刷一遍保存；同时如果场景和主车发生改变，要将task表刷新，
    删掉的scenario变为无效，新增的添加到task"""
    user = request.state.user
    job_model = await Jobs.filter(Q(company_id=user.company_id) | Q(system_data=1), id=job_id, invalid=0).first()
    assert job_model, "作业不存在"
    if job_model.name != job_update_model.name:
        assert not await Jobs.filter(name=job_update_model.name, company_id=user.company_id, invalid=0).first(), \
            '作业名称已存在'
    assert job_model.status == "waiting", '作业状态错误'
    car_obj = await Cars.filter(Q(company_id=user.company_id) | Q(system_data=1), id=job_update_model.car_id,
                                invalid=0).first()
    assert car_obj, '主车信息不存在'
    car_sensor = await CarSensors.filter(car_id=job_update_model.car_id, invalid=0).first()
    assert car_sensor, '主车上没有添加传感器'
    sensor_obj = await Sensors.filter(Q(company_id=user.company_id) | Q(system_data=1), id=car_sensor.sensor_id,
                                      invalid=0).first()
    assert sensor_obj, '传感器信息不存在'

    scenario_infos = await Scenarios.filter(id__in=job_update_model.scenario_ids, invalid=0).all()
    assert len(scenario_infos) == len(job_update_model.scenario_ids), '场景存在异常，检查是否有无效场景'
    # 获取所有的 scenario id,删除掉前端删掉的任务
    task_all = await Tasks.filter(Q(company_id=user.company_id) | Q(system_data=1), job_id=job_id, invalid=0).all()
    assert task_all, "场景不存在，检查作业是否存在场景"

    async with transactions.in_transaction():
        if job_update_model.scenario_ids != job_model.scenario_ids:
            for t in task_all:
                await Tasks.filter(id=t.id).update(invalid=t.id)

            for scenario in scenario_infos:
                task_snap = Tasks()
                task_snap.desc = scenario.desc
                task_snap.job_id = job_id
                task_snap.job_name = job_update_model.name
                task_snap.name = scenario.name
                task_snap.scenario_id = scenario.id
                task_snap.scenario_tags = scenario.tags
                task_snap.status = "waiting"
                task_snap.user_id = user.id
                task_snap.company_id = user.company_id
                await task_snap.save()
        await job_model.update_from_dict(job_update_model.dict()).save()
    return

@router.get(
    "/job/{job_id}",
    summary='获取作业详细情',
    tags=["Job"]
)
async def get_job(job_id: int):
    """
    get job details;
    if status is waiting,return edit page info
    if status is not waiting, return detail info
    """
    user = request.state.user
    job_info = await Jobs.filter(Q(company_id=user.company_id) | Q(system_data=1), id=job_id, invalid=0).first()
    assert job_info, '任务不存在'
    status = job_info.status

    if status == "waiting":
        car_obj = await Cars.filter(Q(company_id=user.company_id) | Q(system_data=1), id=job_info.car_id,
                                    invalid=0).first()
        job_info.car_snap = car_obj if car_obj else []
        tasks = await Scenarios.filter(id__in=job_info.scenario_ids, invalid=0).all()
        if not tasks:
            job_info.scenario_ids = []
        # job_info.sensor_snap = sensor_obj
    else:
        tasks = await Tasks.filter(job_id=job_id, invalid=0, company_id=user.company_id).order_by("index").all()

    job_info = jsonable_encoder(job_info)
    controller_version = await Controllers.filter(id=job_info['controller_version'], invalid=0).first()
    job_info['controller_version_name'] = controller_version.version if controller_version else ''
    controller = await Controllers.filter(id=job_info['controller'], invalid=0).first()
    job_info['controller_name'] = controller.name if controller else ''
    job_info['tasks'] = jsonable_encoder(tasks) if tasks else []
    if status != "waiting":
        job_info = handle_finish_pass_job(job_info)
    return job_info

@router.get(
    "/job_detail/{job_id}",
    summary='获取主车信息',
    tags=["Job"]
)
async def get_car_info(job_id: int):
    """
    get job car snap details;
    """
    user = request.state.user
    job_info = await Jobs.filter(Q(company_id=user.company_id) | Q(system_data=1), id=job_id, invalid=0).first()
    assert job_info, '任务不存在'
    car_info = dict()
    car_info['car_snap'] = job_info.car_snap
    car_info['sensor_snap'] = job_info.sensors_snap
    return car_info

@router.delete(
    "/{job_ids}",
    summary='删除作业',
    tags=["Job"],
    dependencies=[Depends(AuthorityRole('admin'))]
)
@limiter.limit("%d/minute" %TIMES)
async def delete_job(request: Request, job_ids: str):
    job_id_list = job_ids.split(",")
    assert len(job_id_list) > 0, "未添加要删除作业id"
    user = request.state.user
    for job_id in job_id_list:
        job = await Jobs.filter(id=job_id, company_id=user.company_id, invalid=0).first()
        assert job, "作业不存在"
        assert job.status in ['waiting', 'finish'], "仅支持删除已完成和未运行状态作业"
        job.invalid = job_id
        await job.save()

        tasks = await Tasks.filter(id=job_id, company_id=user.company_id, invalid=0).all()
        if tasks:
            for task in tasks:
                task.invalid = task.job_id
            await Tasks.bulk_update(tasks, fields=['invalid'])
    return True

@router.delete(
    "/task_by_ids/{job_id}",
    summary='删除任务',
    tags=["Job"],
    dependencies=[Depends(AuthorityRole('admin'))]
)
@limiter.limit("%d/minute" %TIMES)
async def delete_task(request: Request, job_id: int, task_ids: List[int] = Body(..., embed=True)):
    user = request.state.user
    tasks = await Tasks.filter(job_id=job_id, id__in=task_ids, company_id=user.company_id, invalid=0).all()
    assert tasks, "任务不存在"
    assert len(tasks) == len(task_ids), '存在错误的任务id'

    for task in tasks:
        task.invalid = task.id
    await Tasks.bulk_update(tasks, fields=["invalid"])

@router.post(
    "/job_list",
    summary='获取作业列表',
    tags=["Job"]
)
async def get_job_list(pagenum: int = 1, pagesize: int = 15, asc: int = -1, status: List[str] = Body([], embed=True),
                       content: str = "", recent: str = "", rets: List[str] = Body([], embed=True)):
    user = request.state.user
    cond = []
    vcs = []

    if status:
        cond.append(' jobs.status in %s ')
        vcs += [tuple(status)]

    if content:
        cond.append(' ( jobs.name like %s or jobs.desc like %s ) ')
        vcs += ['%{}%'.format(content), '%{}%'.format(content)]

    if rets:
        cond.append(' tasks.ret_status in %s ')
        vcs += [tuple(rets)]

    if recent:
        time_str = filter_recent(recent)
        cond.append(' jobs.modified_at >= %s ')
        vcs += [time_str]

    cond.append(' jobs.invalid = 0 and jobs.company_id = %s or jobs.system_data = 1 ')
    vcs += [user.company_id]

    cond = ' and '.join(cond)

    async with in_transaction() as trans:
        data = await trans.execute_query(
            'select count(1) as total from jobs left join tasks on tasks.job_id = jobs.id where {} group by jobs.id,jobs.name,jobs.status '.format(
                cond if cond else 1), vcs)
        total_num = data[0]
        vcs += [pagesize, max(pagenum - 1, 0) * pagesize]
        sql = "select jobs.id as job_id," \
              "jobs.name as job_name," \
              "jobs.status as job_status," \
              "jobs.name_en as job_name_en," \
              "jobs.desc as job_desc," \
              "jobs.desc_en as job_desc_en," \
              "jobs.render_mode as render_mode," \
              "jobs.controller as controller," \
              "jobs.system_data as system_data," \
              "jobs.controller_version as controller_version," \
              "max(jobs.modified_at) as modified_at," \
              "count(tasks.id) as task_num," \
              "count(if(tasks.ret_status='pass',true,null)) as pass_num," \
              "count(if(tasks.ret_status='failure',true,null)) as failure_num," \
              "count(if(tasks.status='inqueue',true,null)) as inqueue_num," \
              "count(if(tasks.status='running',true,null)) as running_num," \
              "count(if(tasks.status='finish',true,null)) as finish_num," \
              "count(if(tasks.ret_status='exception',true,null)) as exception_num " \
              "from jobs left join tasks on tasks.job_id = jobs.id where {} group by jobs.id,jobs.name,jobs.status order by jobs.modified_at " + (
                  'desc' if asc == -1 else 'asc') + " limit %s offset %s"

        datas = await trans.execute_query(sql.format(cond), vcs)
        for data in datas[1]:
            data['modified_at'] = data['modified_at'].strftime("%Y-%m-%d %H:%M:%S")
    pageinfo = utils.paginate(total_num, pagesize)

    return {'pageinfo': pageinfo, 'datas': datas[1]}

@router.post(
    "/run_job/{job_id}",
    summary='运行作业',
    tags=["Job"]
)
async def run_job(job_id: int):
    user = request.state.user
    job_model = await Jobs.filter(Q(company_id=user.company_id) | Q(system_data=1), id=job_id, invalid=0,
                                  status="waiting").first()
    assert job_model, "作业不存在"
    task_snap = await Tasks.filter(Q(company_id=user.company_id) | Q(system_data=1), job_id=job_id, invalid=0,
                                   status="waiting").order_by('id').all()
    assert task_snap, "任务不存在"

    car_info = await Cars.filter(Q(company_id=user.company_id) | Q(system_data=1), id=job_model.car_id,
                                 invalid=0).first()
    assert car_info, '主车信息不存在'

    dynamic_info = await Dynamics.filter(Q(company_id=user.company_id) | Q(system_data=1), id=car_info.dynamics_id,
                                         invalid=0).first()
    assert dynamic_info, '动力学模型不存在'

    sensors_snap = await CarSensors.filter(car_id=car_info.id, invalid=0).all()
    assert sensors_snap, '主车上传感器不存在'
    controller = await Controllers.filter(id=job_model.controller, invalid=0).first()
    controller_version = await Controllers.filter(id=job_model.controller_version, invalid=0).first()
    assert controller, '控制系统不存在'
    assert controller_version, '控制系统版本不存在'
    sensors_list = []
    sensors_snap = jsonable_encoder(sensors_snap)
    for s in sensors_snap:
        sensor_info = await Sensors.filter(Q(company_id=user.company_id) | Q(system_data=1), id=s["sensor_id"]).first()
        # 判断camera sensor上若semantic、instance字段值为Ture则增加对应的sensor
        if "camera" in s["type"] and s["semantic"]:
            sensors_list.append({"type": "sensor.camera.semantic_segmentation",
                                 'car_sensor_id': s['id'],
                                 "semantic": s.get("semantic"),
                                 "spawn_point": {"x": s.get("position_x"),
                                                 "y": s.get("position_y"),
                                                 "z": s.get("position_z"),
                                                 "roll": s.get("roll"),
                                                 "pitch": s.get("pitch"),
                                                 "yaw": s.get("yaw"),
                                                 },
                                 "fov": sensor_info.param.get("fov"),
                                 "image_size_x": sensor_info.param.get("image_size_x"),
                                 "image_size_y": sensor_info.param.get("image_size_y"),
                                 "sensor_tick": sensor_info.param.get("sensor_tick")
                                 })
        if "camera" in s["type"] and s["instance"]:
            sensors_list.append({"type": "sensor.camera.instance_segmentation",
                                 'car_sensor_id': s['id'],
                                 "instance": s.get("instance"),
                                 "spawn_point": {"x": s.get("position_x"),
                                                 "y": s.get("position_y"),
                                                 "z": s.get("position_z"),
                                                 "roll": s.get("roll"),
                                                 "pitch": s.get("pitch"),
                                                 "yaw": s.get("yaw"),
                                                 },
                                 "fov": sensor_info.param.get("fov"),
                                 "image_size_x": sensor_info.param.get("image_size_x"),
                                 "image_size_y": sensor_info.param.get("image_size_y"),
                                 "sensor_tick": sensor_info.param.get("sensor_tick")
                                 })
        sensors_list.append({
            **{"type": s["type"], "name": s["name"], "data_record": s["data_record"], "car_sensor_id": s["id"],
               "semantic": s["semantic"], "instance": s["instance"], 'nick_name': s['nick_name']},
            **{
                "spawn_point": {
                    "x": s["position_x"],
                    "y": s["position_y"],
                    "z": s["position_z"],
                    "roll": s["roll"],
                    "pitch": s["pitch"],
                    "yaw": s["yaw"],
                }
            },
            **jsonable_encoder(sensor_info.param)})
        # sensors_list.append(rgb_for_replay)
    car_snap = jsonable_encoder(car_info)
    dynamic_info.param['dynamics_id'] = dynamic_info.id
    dynamic_info.param['dynamics_name'] = dynamic_info.name
    dynamic_info.param['dynamics_name_en'] = dynamic_info.name_en
    car_snap["vehicle_physics_control"] = jsonable_encoder(dynamic_info.param)

    queue_sess = request.app.state.redis
    index = 100
    async with transactions.in_transaction():
        sensor_data_snap = []
        for task in task_snap:
            scenario_param = await Scenarios.filter(id=task.scenario_id, invalid=0).first()
            assert scenario_param, '{}场景信息缺失,检查场景是否存在'.format(task.name)
            scenario_param = jsonable_encoder(scenario_param)
            try:
                openscenario, ego_start_postion, ego_target_postion = await json_convert_to_openscenario(
                    scenario_param, car_snap, request.state.user.id)
            except Exception as e:
                assert False, f'任务{task.name}场景格式转换失败'
            scenario_param.setdefault("dynamic_scene", {})
            scenario_param["dynamic_scene"]["scene_script"] = openscenario
            scenario_param["dynamic_scene"]["type"] = "openScenario"
            scenario_param["ego_start_and_end_position"] = {"start_position": ego_start_postion,
                                                            "end_position": ego_target_postion}
            task.scenario_param = scenario_param
            task.status = "inqueue"
            task.index = Decimal(index / 100)
            await task.save()
            new_sensors = []
            for num, sensor in enumerate(sensors_list):
                # 对sensors_list中的各个sensor添加role_name
                sensor["role_name"] = ".".join([str(task.id), sensor["type"], str(num)])
                if (sensor.get("data_record") or sensor.get("semantic") or sensor.get("instance")) \
                        and (("instance" not in sensor.get("type")) and ("semantic" not in sensor.get("type"))):
                    sensor_data_info = SensorData()
                    sensor_data_info.sensor_name = sensor["name"]
                    sensor_data_info.sensor_type = sensor["type"]
                    sensor_data_info.car_sensor_id = sensor["car_sensor_id"]
                    sensor_data_info.task_id = task.id
                    sensor_data_snap.append(sensor_data_info)
                if job_model.render_mode == 'norender':
                    if sensor.get('type') in ['sensor.camera.goal', 'sensor.lidar.goal']:
                        new_sensors.append(sensor)
                else:
                    new_sensors.append(sensor)

            task_info = jsonable_encoder(task)
            task_info['id'] = str(task_info['id'])
            task_info['job_id'] = str(task_info['job_id'])
            task_info['car_snap'] = car_snap
            task_info.setdefault('sensors_snap', {})
            task_info['sensors_snap'].setdefault('sensors', new_sensors)
            task_info['agent'] = controller_version.setup_file_name
            task_info['conf'] = controller_version.config_file_name
            task_info['data_flow'] = controller_version.data_flow_file_name
            task_info['view_record'] = job_model.view_record
            task_info['show_game_window'] = job_model.show_game_window
            dic = {"action": "RUN", "task": task_info}

            assert task_info['scenario_param']["evaluation_standard"], "evaluation_standard not exits"
            await queue_sess.hset('task_infos', task.id, json.dumps(dic))
            index += 100
            sensor_cam_info = SensorData()
            sensor_cam_info.task_id = task.id
            sensor_cam_info.sensor_name = "record_rgb"
            sensor_cam_info.sensor_type = "sensor.camera.rgb"
            sensor_cam_info.car_sensor_id = 0
            sensor_data_snap.append(sensor_cam_info)
        await SensorData.bulk_create(sensor_data_snap)
        job_model.car_snap = car_snap
        job_model.sensors_snap = sensors_snap
        job_model.status = "inqueue"
        job_model.start_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        await job_model.save()
        return "执行成功"

@router.post(
    "/create_and_run_new_job",
    summary='创建并运行新的作业',
    tags=["Job"]
)
async def create_and_run_new_job(request: Request, job_create_model: CreateJob):
    job_id = await create_job(request, job_create_model)
    await run_job(job_id)
    return job_id

@router.post(
    "/stop_jobs",
    summary='终止作业',
    tags=["Job"]
)
async def stop_jobs(job_ids: List[int] = Body(..., embed=True)):
    user = request.state.user
    jobs = await Jobs.filter(Q(company_id=user.company_id) | Q(system_data=1), id__in=job_ids, invalid=0).all()
    assert jobs, "作业不存在"
    task_snap = await Tasks.filter(Q(company_id=user.company_id) | Q(system_data=1), job_id__in=job_ids,
                                   invalid=0).all()
    assert task_snap, "任务不存在"

    async with transactions.in_transaction():
        for task in task_snap:
            await task.update_from_dict({'status': 'waiting', 'start_time': None, 'end_time': None}).save()
            sensor_data_item = await SensorData.filter(task_id=task.id, invalid=0).all()
            for data_item in sensor_data_item:
                data_item.modified_at = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                await data_item.update_from_dict({"invalid": task.id}).save()
        for job in jobs:
            job.modified_at = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
            await job.update_from_dict({'status': 'waiting', 'start_time': None, 'end_time': None}).save()
    return

@router.post(
    "/retry_task",
    summary='重试任务',
    tags=["Job"]
)
async def retry_task(job_id: str, task_ids: List[int] = Body(..., embed=True)):
    user = request.state.user
    job_model = await Jobs.filter(Q(company_id=user.company_id) | Q(system_data=1), id=job_id, invalid=0).first()
    assert job_model, "作业不存在"
    task_snap = await Tasks.filter(Q(company_id=user.company_id) | Q(system_data=1), job_id=job_id, invalid=0,
                                   id__in=task_ids).all()
    assert task_snap, "任务不存在"

    queue_sess = request.app.state.redis
    controller = await Controllers.filter(id=job_model.controller, invalid=0).first()
    controller_version = await Controllers.filter(id=job_model.controller_version, invalid=0).first()
    assert controller, '控制系统不存在'
    assert controller_version, '控制系统版本不存在'
    async with transactions.in_transaction():
        sensor_data_snap = []
        for t in task_snap:
            now_max_index = await Tasks.filter(invalid=0,
                                               job_id=job_id,
                                               index__gte=t.index,
                                               index__lt=int(t.index) + 1) \
                .annotate(max_index=Max('index')).first().values('max_index')
            now_max_index = now_max_index['max_index']
            task = Tasks()
            task.status = "inqueue"
            task.end_time = None
            task.scenario_id = t.scenario_id
            task.job_id = t.job_id
            task.job_name = t.job_name
            task.scenario_param = t.scenario_param
            task.scenario_tags = t.scenario_tags
            task.user_id = t.user_id
            task.company_id = t.company_id
            task.desc = t.desc
            task.index = Decimal((now_max_index * 100 + 1) / 100).quantize(Decimal('0.00'))
            ori_task = await Tasks.filter(invalid=0, job_id=job_id, scenario_id=t.scenario_id,
                                          index=int(t.index)).first()
            if str(task.index) == '01':
                task.name = t.name + '_' + str(task.index).split(".")[-1]
            else:
                task.name = ori_task.name + '_' + str(task.index).split(".")[-1]
            task.result = {}
            await task.save()

            sensors_list = []
            sensors_snap = job_model.sensors_snap
            for s in sensors_snap:
                sensor_info = await Sensors.filter(Q(company_id=user.company_id) | Q(system_data=1),
                                                   id=s['sensor_id']).first()
                # 判断camera sensor上若semantic、instance字段值为Ture则增加对应的sensor
                if "camera" in s["type"] and s["semantic"]:
                    sensors_list.append({"type": "sensor.camera.semantic_segmentation",
                                         "semantic": s.get("semantic"),
                                         'car_sensor_id': s['id'],
                                         "spawn_point": {"x": s.get("position_x"),
                                                         "y": s.get("position_y"),
                                                         "z": s.get("position_z"),
                                                         "roll": s.get("roll"),
                                                         "pitch": s.get("pitch"),
                                                         "yaw": s.get("yaw"),
                                                         },
                                         "fov": sensor_info.param.get("fov"),
                                         "image_size_x": sensor_info.param.get("image_size_x"),
                                         "image_size_y": sensor_info.param.get("image_size_y"),
                                         "sensor_tick": sensor_info.param.get("sensor_tick")
                                         })
                if "camera" in s["type"] and s["instance"]:
                    sensors_list.append({"type": "sensor.camera.instance_segmentation",
                                         "instance": s.get("instance"),
                                         'car_sensor_id': s['id'],
                                         "spawn_point": {"x": s.get("position_x"),
                                                         "y": s.get("position_y"),
                                                         "z": s.get("position_z"),
                                                         "roll": s.get("roll"),
                                                         "pitch": s.get("pitch"),
                                                         "yaw": s.get("yaw"),
                                                         },
                                         "fov": sensor_info.param.get("fov"),
                                         "image_size_x": sensor_info.param.get("image_size_x"),
                                         "image_size_y": sensor_info.param.get("image_size_y"),
                                         "sensor_tick": sensor_info.param.get("sensor_tick")
                                         })
                sensors_list.append({**{'type': s['type'], 'name': s['name'], 'data_record': s['data_record'],
                                        'car_sensor_id': s['id'], "semantic": s["semantic"], "instance": s["instance"],
                                        'nick_name': s['nick_name']},
                                     **{
                                         'spawn_point': {
                                             "x": s['position_x'],
                                             "y": s['position_y'],
                                             "z": s['position_z'],
                                             "roll": s['roll'],
                                             "pitch": s['pitch'],
                                             "yaw": s['yaw'],
                                         }
                                     }, **jsonable_encoder(sensor_info.param)})
                # sensors_list.append(rgb_for_replay)
            new_sensors = []
            for num, sensor in enumerate(sensors_list):
                # 对sensors_list中的各个sensor添加role_name
                sensor["role_name"] = ".".join([str(task.id), sensor["type"], str(num)])
                new_sensors.append(sensor)
                if (sensor.get("data_record") or sensor.get("semantic") or sensor.get("instance")) \
                        and (("instance" not in sensor.get("type")) and ("semantic" not in sensor.get("type"))):
                    sensor_data_info = SensorData()
                    sensor_data_info.sensor_name = sensor["name"]
                    sensor_data_info.sensor_type = sensor["type"]
                    sensor_data_info.car_sensor_id = sensor["car_sensor_id"]
                    sensor_data_info.task_id = task.id
                    sensor_data_snap.append(sensor_data_info)

            sensor_cam_info = SensorData()
            sensor_cam_info.task_id = task.id
            sensor_cam_info.sensor_name = "record_rgb"
            sensor_cam_info.sensor_type = "sensor.camera.rgb"
            sensor_cam_info.car_sensor_id = 0
            sensor_data_snap.append(sensor_cam_info)

            task_info = jsonable_encoder(task)
            task_info['id'] = str(task_info['id'])
            task_info['job_id'] = str(task_info['job_id'])
            task_info['car_snap'] = job_model.car_snap
            task_info['agent'] = controller_version.setup_file_name
            task_info['conf'] = controller_version.config_file_name
            task_info['data_flow'] = controller_version.data_flow_file_name
            task_info.setdefault('sensors_snap', {})
            task_info['sensors_snap'].setdefault('sensors', new_sensors)
            task_info['view_record'] = job_model.view_record
            task_info['show_game_window'] = job_model.show_game_window
            dic = {"action": "RUN", "task": task_info}

            assert task_info['scenario_param']["evaluation_standard"], "evaluation_standard not exits"
            await queue_sess.hset('task_infos', task.id, json.dumps(dic))

        await SensorData.bulk_create(sensor_data_snap)
        job_model.modified_at = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
        job_model.status = "running"
        await job_model.save()

@router.get(
    "/total_task_info",
    summary='获取总的任务信息',
    tags=["Job"]
)
async def get_total_task_info():
    ret_dic = {}
    failure_num = 0
    passed_num = 0
    exception_num = 0
    runing_num = 0
    waiting_num = 0
    finished_num = 0
    queue_num = 0

    user = request.state.user
    jobs = await Jobs.filter(company_id=user.company_id, invalid=0).all()
    for job_model in jobs:
        task_snap = await Tasks.filter(company_id=user.company_id, invalid=0, job_id=job_model.id).all()
        for task in task_snap:
            if task.status == "inqueue":
                queue_num += 1
            if task.status == "finish":
                finished_num += 1
            if task.status == "waiting":
                waiting_num += 1
            if task.status == "running":
                runing_num += 1
            if task.ret_status == "exception":
                exception_num += 1
                continue
            if task.ret_status == "failure":
                failure_num += 1
                continue
            if task.ret_status == "pass":
                passed_num += 1
    queue_sess = request.app.state.redis
    free_num = await queue_sess.hget('node_status', 'free_num')
    busy_num = await queue_sess.hget('node_status', 'busy_num')
    exct_num = await queue_sess.hget('node_status', 'exct_num')
    free_nodes_num = int(free_num) if free_num else 0
    busy_nodes_num = int(busy_num) if busy_num else 0
    exct_nodes_num = int(exct_num) if exct_num else 0
    ret_dic["queue_num"] = queue_num
    ret_dic["failure_num"] = failure_num
    ret_dic["passed_num"] = passed_num
    ret_dic["exception_num"] = exception_num
    ret_dic["finished_num"] = finished_num
    ret_dic["waiting_num"] = waiting_num
    ret_dic["runing_num"] = runing_num
    ret_dic["free_nodes_num"] = free_nodes_num
    ret_dic["busy_nodes_num"] = busy_nodes_num
    ret_dic["exct_nodes_num"] = exct_nodes_num
    ret_dic["error_nodes_num"] = 0

    return ret_dic

@router.get(
    "/task_info",
    summary='获取任务详情',
    tags=["Job"]
)
async def get_task_info(task_id: int):
    user = request.state.user
    task = await Tasks.filter(id=task_id, company_id=user.company_id, invalid=0).first()
    assert task, "任务不存在"
    assert task.status in ['timeout', 'finish'], "仅支持已完成状态任务信息查询"
    job = await Jobs.filter(id=task.job_id, company_id=user.company_id, invalid=0).first()
    assert job, "作业信息不存在"
    ret_dic = {}
    task_dic = jsonable_encoder(task)
    task_dic["show_game_window"] = job.show_game_window
    task_dic["view_record"] = job.view_record
    timer, task_dic['running_time'] = cal_running_time(task_dic['start_time'], task_dic['end_time'])
    car_info = await assember_car_info(job)
    ret_dic.update(car_info)
    record_data_sensors = record_data_sensor_list(job)
    ret_dic.update(record_data_sensors)
    if task.result:
        car_event_list = handle_car_event(task.result.get("criteria", []))
        ret_list = sort_criteria(task_dic.get('result').get('criteria'))
        task_dic['result']['criteria'] = ret_list
    else:
        car_event_list = []
        evaluat_list = handel_evaluation(task)
        ret_dic.update({'evaluat_list': evaluat_list})
    ret_dic.update({"task": task_dic})
    ret_dic.update({'car_event_list': car_event_list})
    logs = await Logs.filter(task_id=task_id, type='').all()
    for lo in logs:
        if lo.game_time:
            try:
                time_str = str(round(float(lo.game_time), 2))
                lo.game_time = "%.2d:%.2d.%s" % (int(time_str.split(".")[0]) // 60, int(time_str.split(".")[0]) % 60,
                                                 time_str.split(".")[1])
            except:
                print('格式化失败')
    logs.sort(key=lambda log: (log.created_at, log.id))
    scenario_logs = await Logs.filter(task_id=task_id, type="scenario").all()
    for log in scenario_logs:
        log.msg = json.loads(log.msg)
        if log.msg.get('current_game_time') or log.msg.get("finish_game_time"):
            handel_game_time(log.msg, 'current_game_time')
    scenario_logs.sort(key=lambda log: log.created_at)
    ret_dic.update({"scenario_event_list": scenario_logs})
    controller = await Controllers.filter(id=job.controller, invalid=0).first()
    controller_version = await Controllers.filter(id=job.controller_version, invalid=0).first()
    ret_dic.update({"controller": controller.name if controller else '',
                    "controller_version": controller_version.version if controller_version else '',
                    "log_info": logs, "render_mode": job.render_mode})
    return ret_dic

@router.get(
    "/task_scenario_infos",
    summary='获取任务场景信息',
    tags=["Job"]
)
async def get_task_scenario_infos(task_id: int):
    user = request.state.user
    task = await Tasks.filter(id=task_id, company_id=user.company_id, invalid=0).first()
    assert task, "任务不存在"
    assert task.status in ['timeout', 'finish'], "仅支持已完成状态任务信息查询"
    task_dic = jsonable_encoder(task)
    return task_dic.get('scenario_param')

@router.get(
    "/job_report/{job_id}",
    summary='获取作业测试报告',
    tags=["Job"]
)
async def get_job_report(job_id: int, locale: str):
    """
    获取测试报告的接口
    params: 作业ID
    return: FileObject
    """
    job_info = await get_job(job_id)
    assert job_info["status"] == "finish", "仅提供完成状态的作业测试报告"
    try:
        # 作业信息 ["作业名称"，"测试主车", "受测系统", "测试环境", "开始时间", "结束时间"]
        # 结果汇总 ["场景总数", "通过", "失败", "无效", "总测试里程", "平均得分", "通过率"]
        job_info_desc, job_res_summary = get_job_info_desc(job_info, locale)

        # 获取任务详情数据 ["No.", "场景名称", "测试里程", "仿真时长", "任务结果", "评价通过率"]
        task_datas_list = get_task_detail_info(job_info, locale)

        # 调用report类函数生成报告
        report_name = "{}_{}_{}.pdf".format(job_info["name"], job_id, time.strftime("%Y%m%d-%H%M%S"))
        report_path = os.path.join('/tmp/', report_name)
        report = Report(job_info_desc, job_res_summary, task_datas_list, report_path, report_name, locale)
        report_path = report.create_file()
        print(report_path)
        return FileResponse(report_path, filename=Path(report_path).name)
    except Exception as e:
        assert False, "{}生成测试报告失败".format(job_info["name"])

@router.get(
    "/sensor_data",
    summary='获取传感器数据',
    tags=["Job"]
)
async def get_sensor_data(task_id: int, sensor_id: int, data_type: str):
    """
    获取传感器数据接口
    params: task_id sensor_id
    return: {}
    """
    sensor_data = await SensorData.filter(task_id=task_id, car_sensor_id=sensor_id, invalid=0).first()
    assert sensor_data, "传感器数据不存在"
    sensor_data = jsonable_encoder(sensor_data)
    if "camera" in sensor_data["sensor_type"]:
        if data_type == 'semantic':
            sensor_data["status"] = 'empty' if (sensor_data["process_rate_semantic"] == -1 or
                                                sensor_data["process_rate_semantic"] is None) else \
                ('data' if sensor_data["process_rate_semantic"] == 100 else 'loading')
        elif data_type == 'instance':
            sensor_data["status"] = 'empty' if (sensor_data["process_rate_instance"] == -1 or
                                                sensor_data["process_rate_instance"] is None) else \
                ('data' if sensor_data["process_rate_instance"] == 100 else 'loading')
        elif data_type == 'origin':
            sensor_data["status"] = 'empty' if (sensor_data["process_rate_img"] == -1 or
                                                sensor_data["process_rate_video"] == -1 or
                                                sensor_data["process_rate_img"] is None or
                                                sensor_data["process_rate_video"] is None) else \
                ('video' if (sensor_data["process_rate_img"] == 100 and sensor_data[
                    "process_rate_video"] == 100) else 'loading')
    elif "lidar" in sensor_data["sensor_type"] and sensor_data["data_url"]:
        sensor_data["status"] = "data"
    elif ("lidar" in sensor_data["sensor_type"] and (
            sensor_data["process_rate_data"] == -1 or sensor_data["process_rate_data"] is None)) or (
            "camera" in sensor_data["sensor_type"] and (
            sensor_data["process_rate_img"] == -1 or sensor_data["process_rate_img"] is None or
            sensor_data["process_rate_video"] == -1 or sensor_data["process_rate_video"] is None or
            sensor_data["process_rate_semantic"] == -1 or sensor_data["process_rate_semantic"] is None or
            sensor_data["process_rate_instance"] == -1 or sensor_data["process_rate_instance"] is None)):
        sensor_data["status"] = "empty"
    else:
        sensor_data["status"] = "loading"
    return sensor_data


#!/usr/bin/python

# --------------------------------------------------------
#           PYTHON PROGRAM
# Here is where we are going to define our set of...
# - Imports
# - Global Variables
# - Functions
# ...to achieve the functionality required.
# When executing > python 'this_file'.py in a terminal,
# the Python interpreter will load our program,
# but it will execute nothing yet.
# --------------------------------------------------------

import sys
import codecs
from operator import itemgetter, attrgetter
#Print to output stream
def print_key_value(lang, value, percentage, output_stream):

    res = lang + "\t" + str(value) + "\t" + str(percentage) + "\n"
    output_stream.write(res)

def get_key_value(line):
    # 1. Get rid of the end of line at the end of the string
    line = line.replace('\n', '')

    # 2. Split the string by the tabulator delimiter
    words = line.split('\t')

    # 3. Get the key and the value and return them
    lang = words[0]
    value = words[1]

    return value, lang

# ------------------------------------------
# FUNCTION my_reduce
# ------------------------------------------
def my_reduce(input_stream, total_petitions, output_stream):

#   I create a number of variables similar to other reducers with addition to an overall list that
#	stores a the total temp views, language and percentage(answer)
    current_lang = ""
    current_page_views = 0
    percentage = 0.0
    temp_total_views = 0
    overall_list = []

    # I read the stream and extract the data from each line using get_key_value function
    for text_line in input_stream:
        #New variables from return of function
        (new_page_views, new_lang) = get_key_value(text_line)

        #Similar to other reducers I commented on
        if new_lang != current_lang:
           if current_lang != "":

			   #Calculate the percentage, round and then append all to an overall list. (nested list)
               percentage = (temp_total_views / total_petitions) * 100
               answer = round(percentage, 2)
               overall_list.append((temp_total_views, current_lang, answer))
           #need to resent the total views for next language/project
           temp_total_views = 0
           current_lang = new_lang
#       Set rest of current variables to new variables. Also add the total views for that project as you loop.
        current_page_views = new_page_views
        temp_total_views = temp_total_views + int(current_page_views)

	#Need to account for last project or language
    if current_lang != "":
        percentage = (temp_total_views / total_petitions) * 100
        overall_list.append((temp_total_views, current_lang, answer))

     #sorting the overall list by page views since that will also show highest percentage.
	#Then print so that language is first since I had page views as first item in list
    overall_list.sort(key=itemgetter(0), reverse = True)
    for item in overall_list:
        print_key_value(item[1], item[0], item[2], output_stream )
# ------------------------------------------
# FUNCTION my_main
# ------------------------------------------
def my_main(debug, i_file_name, total_petitions, o_file_name):
    # We pick the working mode:

    # Mode 1: Debug --> We pick a file to read test the program on it
    if debug == True:
        my_input_stream = codecs.open(i_file_name, "r", encoding='utf-8')
        my_output_stream = codecs.open(o_file_name, "w", encoding='utf-8')
    # Mode 2: Actual MapReduce --> We pick std.stdin and std.stdout
    else:
        my_input_stream = sys.stdin
        my_output_stream = sys.stdout

    # We launch the Map program
    my_reduce(my_input_stream, total_petitions, my_output_stream)

# ---------------------------------------------------------------
#           PYTHON EXECUTION
# This is the main entry point to the execution of our program.
# It provides a call to the 'main function' defined in our
# Python program, making the Python interpreter to trigger
# its execution.
# ---------------------------------------------------------------
if __name__ == '__main__':
    # 1. Input parameters
    debug = True

    # This variable must be computed in the first stage
    total_petitions = 21996631

    i_file_name = "sort_simulation.txt"
    o_file_name = "reduce_simulation.txt"

    my_main(debug, i_file_name, total_petitions, o_file_name)


import os
import re

from flask import Blueprint, g, current_app, request, session, redirect, url_for, render_template, abort, flash

from flask.ext.login import current_user, login_required, confirm_login, fresh_login_required

from wok.config import ConfigBuilder

from intogensm.utils import normalize_id
from intogensm.paths import get_project_path, get_temp_path
from intogensm.constants.oncodrivefm import ONCODRIVEFM_GENES_THRESHOLD, ONCODRIVEFM_PATHWAYS_THRESHOLD
from intogensm.constants.oncodriveclust import ONCODRIVECLUST_MUTATIONS_THRESHOLD

from intogensm.common import project_analysis

from constants import *

bp = Blueprint("analysis", __name__)

@bp.route("/<type>", methods=["GET", "POST"])
@login_required
def run(type):
	if type not in [COHORT_ANALYSIS, SINGLE_TUMOR_ANALYSIS]:
		abort(400)

	if request.method == "GET":
		form = dict(
			ofm_genes_threshold=ONCODRIVEFM_GENES_THRESHOLD,
			ofm_pathways_threshold=ONCODRIVEFM_PATHWAYS_THRESHOLD,
			oclust_genes_threshold=ONCODRIVECLUST_MUTATIONS_THRESHOLD)

		return render_template("analysis.html", type=type, form=form)

	mutations_file = request.files['mutations_file']
	file_name = os.path.basename(mutations_file.filename)

	project_id = request.form['project_name']
	if len(project_id) == 0:
		project_id = os.path.splitext(file_name)[0]

	project_id = normalize_id(project_id)

	i = 0
	base_id = project_id
	while current_app.wok.exists_case(current_user, project_id):
		i += 1
		project_id = "{}-{}".format(base_id, i)

	'''
	if current_app.wok.exists_case(current_user, project_id):
		flash("An analysis with this name already exists. Please give it a different name or remove the previous one before starting again.", "error")
		return render_template("analysis.html", type=type, form=request.form)
	'''

	'''
	if g.demo and current_user.max_analysis != -1 and proj_manager.get_projects_count(g.conn, g.user_id) >= current_user.max_analysis:
		flash("""The online version is for demo only and there is a limit for the number of simultaneous analysis a user can manage.
		You must remove finished analysis before running new ones.
		Please download the pipeline and install in your system to avoid these limitations.""", "error")
		return redirect(url_for("download"))

	if not current_user.validated:
		flash("""You can not run an analysis with your data until you are completely registered.
		Please check your email and follow the instructions to validate this account.""", "error")
		flash("Meanwhile you can play with the included examples.")
		return redirect(url_for("examples"))
	'''

	cb = ConfigBuilder(current_app.wok.conf_builder)
	cb.add_value("workspace", "default")
	cb.add_value("project.id", project_id)

	#case_name = "-".join([current_user.nick, project_id])
	#cb.add_value("wok.instance.name", case_name)

	conf = cb.get_conf()

	results_path = os.path.join(conf["results_path"], current_user.nick)
	cb.add_value("results_path", results_path)

	temp_path = os.path.join(conf["temp_path"], current_user.nick)
	cb.add_value("temp_path", temp_path)

	conf = cb.get_conf()

	project_path = get_project_path(conf, project_id)
	if not os.path.exists(project_path):
		os.makedirs(project_path)

	project_temp_path = get_temp_path(conf, project_id)
	if not os.path.exists(project_temp_path):
		os.makedirs(project_temp_path)

	if not current_user.is_anonymous():
		cb.add_value("website.user_id", current_user.nick)

	# FIXME ? type == SINGLE_TUMOR_ANALYSIS
	if request.form.get("variants_only") == "1":
		cb.add_value("variants_only", True)
		cb.add_value("skip_oncodrivefm", True)
		cb.add_value("skip_oncodriveclust", True)

	try:
		threshold = request.form["ofm_genes_threshold"]
		if re.match(r"^[1-9]\d*%?$", threshold):
			cb.add_value(ONCODRIVEFM_GENES_THRESHOLD_KEY, threshold)
	except:
		if type == COHORT_ANALYSIS:
			current_app.logger.warn("Undefined form input: {}".format("ofm_genes_threshold"))

	try:
		threshold = request.form["ofm_pathways_threshold"]
		if re.match(r"^[1-9]\d*%?$", threshold):
			cb.add_value(ONCODRIVEFM_PATHWAYS_THRESHOLD_KEY, threshold)
	except:
		if type == COHORT_ANALYSIS:
			current_app.logger.warn("Undefined form input: {}".format("ofm_pathways_threshold"))

	try:
		threshold = int(request.form["oclust_genes_threshold"])
		if threshold >= 1:
			cb.add_value(ONCODRIVECLUST_GENES_THRESHOLD_KEY, threshold)
	except:
		if type == COHORT_ANALYSIS:
			current_app.logger.warn("Undefined form input: {}".format("oclust_genes_threshold"))

	genes_filter_enabled = request.form.get('genes_filter_enabled') == "1"
	cb.add_value(ONCODRIVEFM_FILTER_ENABLED_KEY, genes_filter_enabled)
	cb.add_value(ONCODRIVECLUST_FILTER_ENABLED_KEY, genes_filter_enabled)
	if genes_filter_enabled:
		try:
			genes_filter_file = request.files['genes_filter_file']
			genes_filter_file_path = os.path.join(project_temp_path, "genes-filter.txt")
			genes_filter_file.save(genes_filter_file_path)
			if os.path.getsize(genes_filter_file_path) != 0:
				cb.add_value(ONCODRIVEFM_GENES_FILTER_KEY, genes_filter_file_path)
				cb.add_value(ONCODRIVECLUST_GENES_FILTER_KEY, genes_filter_file_path)
		except:
			current_app.logger.exception("Error retrieving genes filter from form")

	mutations_path = os.path.join(project_temp_path, file_name)
	try:
		mutations_file.save(mutations_path)
	except:
		current_app.logger.exception("Error while saving mutations file {} into {}".format(mutations_file.filename, mutations_path))
		flash("""There were some problem with the input file for mutations.
			Please check that a file has been loaded before submitting a new analysis.
			This error has been already submitted to the application administrators
			who will take care of it as soon as possible.""")
		return render_template("analysis.html", type=type, form=request.form)

	assembly = request.form.get("assembly", "hg19").lower()

	cb, flow_uri = project_analysis(
				mutations_path,
				assembly=assembly,
				conf_builder=cb)

	properties = dict(
		analysis_type=type,
		path=project_path,
		temp_path=project_temp_path,
		data_file=mutations_path)

	current_app.logger.info("[{}] Starting analysis {} ...".format(
							current_user.nick, project_id))

	case = current_app.wok.create_case(current_user, project_id, cb, flow_uri,
									   properties=properties, start=True)

	current_app.logger.info("[{}] Analysis {} started on case {}...".format(
							current_user.nick, project_id, case.engine_name))

	return redirect(url_for("cases.index", highlight=case.id))


#!/usr/bin/env python3
'''
    covid19_api.py
    author: Claire Schregardus
    16 October 2020
'''
import sys
import argparse
import flask
import json
import psycopg2

app = flask.Flask(__name__)

try:
    connection = psycopg2.connect(database = 'covid19')
except Exception as e:
    print(e)
    exit()

def get_abbreviations_dict():
    '''Returns a dictionery of state abbreviations and state ids'''
    try:
        cursor = connection.cursor()
        query = 'SELECT abbreviation, id FROM states'
        cursor.execute(query)
    except Exception as e:
        print(e)
        exit()
    abbreviations_dict = {}
    for row in cursor:
        abbreviation = row[0]
        id_num = row[1]
        abbreviations_dict[id_num] = abbreviation
    return abbreviations_dict

def get_state_abbreviation(state_id):
    '''Returns the corresponding state abbreviation given a state id'''
    abbreviations_dict = get_abbreviations_dict()
    state_abbreviation = abbreviations_dict[state_id]
    return state_abbreviation

def get_cumulative_dictionary(state_abbreviation):
    '''Returns a dictionary with cumulative information about a given state'''
    try:
        cursor = connection.cursor()
        query = 'SELECT date, state_id, new_deaths, new_positive_tests, new_negative_tests, new_hospitalizations FROM covid19_days'
        cursor.execute(query)
    except Exception as e:
        print(e)
        exit()
    state_cumulative_info = {}
    state_list = []
    date_list = []
    deaths = 0
    positive_tests = 0
    negative_tests = 0
    hospitalizations = 0
    for row in cursor:
        state = get_state_abbreviation(row[1])
        if state.startswith(state_abbreviation):
            if state in state_list:
                deaths += row[2]
                positive_tests += row[3]
                negative_tests += row[4]
                hospitalizations += row[5]
                date_list.append(row[0])
            else:
                deaths = row[2]
                positive_tests = row[3]
                negative_tests = row[4]
                hospitalizations = row[5]
                state_list.append(state)
                date_list.append(row[0])
    start_date = str(date_list[-1])
    end_date = str(date_list[0])
    state_cumulative_info = {'start_date': start_date, 'end_date': end_date, 'state': state_abbreviation, 'deaths': deaths, \
        'positive_tests': positive_tests, 'negative_tests': negative_tests, 'hospitalizations': hospitalizations}
    return state_cumulative_info

@app.route('/state/<state_abbreviation>/daily')
def get_state(state_abbreviation):
    ''' Returns a list of dictionaries, each representing the COVID-19 statistics from the specified state on a single date.
        Each dictionary will have the following fields:
        date -- YYYY-MM-DD (e.g. "2020-10-08")
        state -- upper-case two-letter state abbreviation (e.g. "MN")
        deaths -- (integer) the number of deaths on this date
        positive -- (integer) the number of positive COVID-19 tests on this date
        negative -- (integer) the number of negative COVID-19 tests on this date
        hospitalizations -- (integer) the number of new hospitalizations for COVID-19 on this date'''
    try:
        cursor = connection.cursor()
        query = 'SELECT date, state_id, new_deaths, new_positive_tests, new_negative_tests, new_hospitalizations FROM covid19_days'
        cursor.execute(query)
    except Exception as e:
        print(e)
        exit()
    state_info = {}
    state_info_list = []
    for row in cursor:
        state = get_state_abbreviation(row[1])
        if state.startswith(state_abbreviation):
            state_info = {'date': str(row[0]), 'state': state, 'deaths': str(row[2]), 'positive_tests': str(row[3]),
            'negative_tests': str(row[4]), 'hospitalizations': str(row[5])}
            state_info_list.append(state_info)
    return json.dumps(state_info_list)

@app.route('/state/<state_abbreviation>/cumulative')
def get_state_cumulative(state_abbreviation):
    '''Returns a single dictionary representing the cumulative statistics for the specified state.
       Each dictionary will have the following fields:
       start_date -- YYYY-MM-DD (e.g. "2020-10-08")
       end_date -- YYYY-MM-DD (e.g. "2020-03-11")
       state -- upper-case two-letter state abbreviation (e.g. "MN")
       deaths -- (integer) the total number of deaths between the start and end dates (inclusive)
       positive -- (integer) the number of positive COVID-19 tests between the start and end dates (inclusive)
       negative -- (integer) the number of negative COVID-19 tests between the start and end dates (inclusive)
       hospitalizations -- (integer) the number of hospitalizations between the start and end dates (inclusive)'''
    state_cumulative_info = get_cumulative_dictionary(state_abbreviation)
    return json.dumps(state_cumulative_info)

@app.route('/states/cumulative')
def get_all_states_cumulative():
    '''Returns a list of dictionaries each representing the cumulative COVID-19 statistics for each state.
    The dictionaries are sorted in decreasing order of deaths, cases (i.e. positive tests), or hospitalizations,
    depending on the value of the GET parameter "sort". If sort is not present, then the list will be sorted by deaths.
    Each dictionary will have the following fields:
    start_date -- YYYY-MM-DD (e.g. "2020-10-08")
    end_date -- YYYY-MM-DD (e.g. "2020-03-11")
    state -- upper-case two-letter state abbreviation (e.g. "MN")
    deaths -- (integer) the total number of deaths between the start and end dates (inclusive)
    positive -- (integer) the number of positive COVID-19 tests between the start and end dates (inclusive)
    negative -- (integer) the number of negative COVID-19 tests between the start and end dates (inclusive)
    hospitalizations -- (integer) the number of hospitalizations between the start and end dates (inclusive)'''
    try:
        cursor = connection.cursor()
        query = 'SELECT abbreviation FROM states'
        cursor.execute(query)
    except Exception as e:
        print(e)
        exit()
    state_list = []
    state_dict_list = []
    for row in cursor:
        state_list.append(row[0])
    for state in state_list:
        state_cumulative_info = get_cumulative_dictionary(state)
        state_dict_list.append(state_cumulative_info)

    if flask.request.args.get('sort') == 'deaths':
        states_deaths_sort = sorted(state_dict_list, key=lambda k: k['deaths'], reverse=True)
        return json.dumps(states_deaths_sort)
    elif flask.request.args.get('sort') == 'cases':
        states_cases_sort = sorted(state_dict_list, key=lambda k: k['positive_cases'], reverse=True)
        return json.dumps(states_cases_sort)
    elif flask.request.args.get('sort') == 'hospitalizations':
        states_hospitalizations_sort = sorted(state_dict_list, key=lambda k: k['hospitalizations'], reverse=True)
        return json.dumps(states_hospitalizations_sort)
    else:
        states_deaths_sort = sorted(state_dict_list, key=lambda k: k['deaths'], reverse=True)
        return json.dumps(states_deaths_sort)

if __name__ == '__main__':
    parser = argparse.ArgumentParser('A sample Flask application/API')
    parser.add_argument('host', help='the host on which this application is running')
    parser.add_argument('port', type=int, help='the port on which this application is listening')
    arguments = parser.parse_args()
    app.run(host=arguments.host, port=arguments.port, debug=True)