"""
app.py
Final version
Updated January 18th, 2022

Written by: Kai Wang
"""

""" -----imports------"""
from flask import Flask, render_template, request, redirect
from flask_mysqldb import MySQL
import csv
import MySQLdb

app = Flask(__name__)

""" -----MySQL connection setup-----
	Change the password to the
	password of your system 		"""

app.config['MYSQL_HOST'] = 'localhost'
app.config['MYSQL_USER'] = 'root'
app.config['MYSQL_PORT'] = 3306

app.config['MYSQL_PASSWORD'] = ""
app.config['MYSQL_DB'] = "shopify_inventory"
app.config['MYSQL_CURSORCLASS'] = 'DictCursor'

mysql = MySQL(app)

""" ---- MainMenu API ----- """
@app.route('/', methods=['GET', 'POST'])
def mainmenu():
    if request.method == 'POST':
        if request.form['submit_button'] == 'Create':
            return redirect('/create')
        elif request.form['submit_button'] == 'View':
            return redirect('/view')

    return render_template('mainmenu.html')

""" --- Create API ----- """
@app.route('/create', methods=['GET', 'POST'])
def create():
    if request.method == 'POST':
        if request.form['submit_button'] == 'Create':
        # Fetch form data
            userDetails = request.form
            name = userDetails['name']
            price = userDetails['price']
            department = userDetails['department']
            description = userDetails['description']
            store_URL = userDetails['store_URL']

            #Insert(Create) query
            cur = mysql.connection.cursor()
            cur.execute("INSERT INTO inventory(name,price,department,description,store_URL) VALUES (%s, %s,%s,%s,%s)", (name,price,department,description,store_URL))
            mysql.connection.commit()
            cur.close()

        elif request.form['submit_button'] == 'View':
            return redirect('/view')

        elif request.form['submit_button'] == 'Return':
            return redirect('/')

    return render_template('create.html')

""" ---- Edit API ----- """
@app.route('/edit',methods=['GET', 'POST'])
def edit():
    if request.method == 'POST':
        if request.form['submit_button'] == 'Return':
          return redirect('/')

        elif request.form['submit_button'] == 'View':
          return redirect('/view')

        elif request.form['submit_button'] == 'Edit':
            #Get all the details for what we want changed
            userDetails = request.form
            prevDetails = request.form
            product_id = userDetails['product_id']
            name = userDetails['name']
            price = userDetails['price']
            department = userDetails['department']
            description = userDetails['description']
            store_URL = userDetails['store_URL']

            #update query, used product id as primary key and overwrites the rest
            cur = mysql.connection.cursor()
            cur.execute("UPDATE inventory SET name = (%s), price = (%s),department = (%s),description = (%s), store_URL = (%s) WHERE product_id = (%s)",(name,price,department,description,store_URL, [product_id]))
            mysql.connection.commit()
            cur.close()
    return render_template('edit.html', prevDetails = prevDetails)

""" ---- Delete API ----- """
@app.route('/delete',methods=['GET', 'POST'])
def delete():
    if request.method == 'POST':
        if request.form['submit_button'] == 'Return':
          return redirect('/')

        if request.form['submit_button'] == 'Delete':
            userDetails = request.form
            product_id = userDetails['product_id']
            name = userDetails['name']
            cur = mysql.connection.cursor()
            #delete query, uses product id to identify unique item to delete
            cur.execute("DELETE From inventory WHERE product_id = (%s)", [product_id])
            mysql.connection.commit()
            cur.close()
        if request.form['submit_button'] == 'View':
          return redirect('/view')
          #render deleted product id and name on the delete page
    return render_template('delete.html',product_id = product_id, name = name)

""" ---- View API ----- """
@app.route('/view',methods=['GET', 'POST'])
def view():
    if request.method == 'POST':
        #return to main maenu
        if request.form['submit_button'] == 'Return':
          return redirect('/')

        #redirect to edit page
        elif request.form['submit_button'] == 'Edit':
          return redirect('/edit')

        #redirect to delete page
        elif request.form['submit_button'] == 'Delete':
          return redirect('/delete')

        #redirect to export function
        elif request.form['submit_button'] == 'Export':
          userDetails = request.form
          product_id = userDetails['product_id']
          export(product_id)

    cur = mysql.connection.cursor()
    cur.execute("SELECT * FROM inventory")
    userDetails = cur.fetchall()
    return render_template('view.html',userDetails=userDetails)

""" ---- export function ----- """
def export(product_id):
    # export to csv functionality
               header = ['Product ID', 'Name','Price', 'Department', 'Description','Store URL']

               #Had to use MySQLdb to connect instead of mysql.connection for it to work
               db = MySQLdb.connect("localhost","root","","shopify_inventory")
               cur = db.cursor()
               cur.execute("SELECT * FROM inventory WHERE product_id = (%s)", [product_id])

               result = cur.fetchall()
               c = csv.writer(open('product_data.csv', 'w',encoding='utf-8'))
               c.writerow(header)
               #writes information in the csv file
               for row in result:
                   print(product_id)
                   c.writerow(row)

if __name__ == '__main__':
    app.run(debug=True)


#!/usr/bin/env python3

import argparse
import os
import sys
import io

import numpy as np
import torch

__dir__ = os.path.normpath(
    os.path.join(
        os.path.dirname(os.path.realpath(__file__)), '..', '..'))
sys.path[1:1] = [__dir__]

from sharpf.utils.abc_utils.hdf5.dataset import PreloadTypes, Hdf5File
import sharpf.fusion.io as fusion_io
import sharpf.data.datasets.sharpf_io as sharpf_io
from sharpf.utils.convertor_utils import convertors_io
from sharpf.metrics.torch_metrics import MRMSE, Q95RMSE, MRecall, MFPR

def main(options):
    print(options)
    if options.single_view:
        true_data_io = convertors_io.AnnotatedViewIO
        pred_data_io = fusion_io.ImagePredictionsIO
    elif options.single_patch:
        true_data_io = sharpf_io.WholePointCloudIO
        pred_data_io = sharpf_io.WholePointCloudIO
    else:
        true_data_io = fusion_io.FusedPredictionsIO
        pred_data_io = fusion_io.FusedPredictionsIO

    # true_data_io = fusion_io.ComparisonsIO
    # pred_data_io = fusion_io.ComparisonsIO

    assert len(options.true_filenames) == len(options.pred_filenames), '-t and -p must be added the same number of times'

    true_distances, pred_distances = [], []
    for true_filename, pred_filename in zip(options.true_filenames, options.pred_filenames):
        true_dataset = Hdf5File(
            true_filename,
            io=true_data_io,
            preload=PreloadTypes.LAZY,
            labels=[options.true_key])
        pred_dataset = Hdf5File(
            pred_filename,
            io=pred_data_io,
            preload=PreloadTypes.LAZY,
            labels=[options.pred_key])
        assert len(true_dataset) == len(pred_dataset), 'lengths of files: {} and {} are not equal'.format(true_filename, pred_filename)

        sharpness_masks = [item[options.true_key] != options.sharpness_bg_value for item in true_dataset]
        #for item, mask in zip(true_dataset, sharpness_masks):
        #    print(item[options.true_key][mask].shape, item[options.true_key].shape, mask.shape)
        true_distances.extend([item[options.true_key][mask] for item, mask in zip(true_dataset, sharpness_masks)])
        pred_distances.extend([item[options.pred_key][mask] for item, mask in zip(pred_dataset, sharpness_masks)])
    #for item in pred_distances:
    #    for val in item:
    #        print(val)
    mfpr = MFPR()
    mrec = MRecall()
    mrmse = MRMSE()
    q95rmse = Q95RMSE()

    if options.is_binary:
        metrics = [
            mfpr,
            mrec,
        ]
    else:
        metrics = [
            mfpr,
            mrec,
            mrmse,
            q95rmse,
        ]
    thresh_r = options.resolution_3d * options.resolution_multiplier
    for idx, (true_item, pred_item) in enumerate(zip(true_distances, pred_distances)):
        print(idx)
        for metric in metrics:
            if isinstance(metric, MFPR) or isinstance(metric, MRecall):
                pred_tensor = torch.tensor(np.array(pred_item, dtype=np.float32), dtype=torch.float32).reshape(1, -1)
                if not options.is_binary:
                    pred_tensor = pred_tensor < thresh_r
                metric.update(
                    pred_tensor,
                    torch.tensor(np.array(true_item, dtype=np.float32), dtype=torch.float32).reshape(1, -1) < thresh_r
                )
            else:
                metric.update(
                    torch.tensor(pred_item, dtype=torch.float32).reshape(1, -1),
                    torch.tensor(true_item, dtype=torch.float32).reshape(1, -1)
                )

    values = [metric.compute() for metric in metrics]
    fp = options.out_filename
    fp = open(fp, 'w') if not isinstance(fp, io.TextIOBase) else fp
    print(
        '{metrics_names}\n{items_values}'.format(
            metrics_names=','.join([str(metric) for metric in metrics]),
            items_values=','.join([str(value) for value in values])),
        file=fp)
    fp.close()

def parse_args():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('--verbose', action='store_true', default=False, dest='verbose',
                        help='verbose output [default: False].')

    parser.add_argument(
        '-t', '--true-filename',
        dest='true_filenames',
        required=True,
        action='append',
        help='path to GT file with fused points and distances.')
    parser.add_argument(
        '-p', '--pred-filename',
        dest='pred_filenames',
        required=True,
        action='append',
        help='path to PRED file with fused points and distances.')
    parser.add_argument(
        '-o', '--out-filename',
        dest='out_filename',
        default=sys.stdout,
        help='path to OUTPUT file with metrics (if None, print metrics to stdout).')

    parser.add_argument(
        '-bin', '--is_binary',
        dest='is_binary',
        default=False,
        action='store_true',
        help='whether predict is a binary vector, where 1 means sharp, 0 - not sharp')

    parser.add_argument(
        '-sv', '--single_view',
        dest='single_view',
        default=False,
        action='store_true',
        help='if set, this .')
    parser.add_argument(
        '-sp', '--single_patch',
        dest='single_patch',
        default=False,
        action='store_true',
        help='if set, this .')

    parser.add_argument(
        '-r', '--resolution_3d',
        dest='resolution_3d',
        default=1.0,
        type=float,
        help='presupposed 3d resolution.')
    parser.add_argument(
        '-s', '--max_distance_to_feature',
        dest='max_distance_to_feature',
        type=float,
        default=1.0,
        help='max distance to sharp feature to evaluate.')
    parser.add_argument(
        '-sbg', '--sharpness_bg_value',
        dest='sharpness_bg_value',
        default=float('inf'),
        type=float,
        help='if set, specifies sharpness value to be treated as background.')

    parser.add_argument(
        '-tk', '--true_key',
        dest='true_key',
        default='distances',
        type=str,
        help='key to use as ground truth.')
    parser.add_argument(
        '-pk', '--pred_key',
        dest='pred_key',
        default='distances',
        type=str,
        help='key to use as predictions.')
    parser.add_argument(
        '-rm', '--resolution_multiplier',
        dest='resolution_multiplier',
        default=1,
        type=float,
        help='resolution multiplier for the probability thresholding')

    return parser.parse_args()

if __name__ == '__main__':
    options = parse_args()
    main(options)


import scipy.io
import os
import numpy as np
import cv2
from src.dataset_loader.datasetUtils import *
from src.module.function import *

# AEI_kmeans = np.array(
#     [[-0.37959412,  0.0896221,   0.05287072,  0.90319805,  0.99185765,  0.99559656],
#      [-0.83103055,  0.10626562,  0.0299934 ,  0.51789511,  0.9870735 ,  0.99758095],
#      [ 0.96297726,  0.68287062,  0.09528844, -0.21831189,  0.72485644,  0.99080621],
#      [ 0.53015532,  0.10014143,  0.02805288, -0.83132568,  0.99192195,  0.99745991],
#      [ 0.86861489,  0.10909378, -0.01550593,  0.45770347,  0.98740085,  0.99779991],
#      [-0.93655641,  0.08003212,  0.00114281, -0.29706842,  0.98999588,  0.99767647],
#      [ 0.02219816,  0.03370241,  0.00365841, -0.98923496,  0.9974772 ,  0.99869783],
#      [ 0.92599029,  0.07402331,  0.01549187, -0.33420168,  0.99187946,  0.99774758],
#      [-0.52019853,  0.10331311, -0.03536063, -0.8353981 ,  0.98985149,  0.99392136],
#      [ 0.45297786,  0.10298805, -0.06113805,  0.8753308 ,  0.98987731,  0.99512509]]
# )

AEI_kmeans = np.array(
    [
     [-0.47621441,  0.12350617,  0.0664845  , 0.87744119 , 0.9890067  , 0.9958762 ],
     [-0.79701111,  0.0897479 ,  0.01711335 , 0.59947227 , 0.99189595 , 0.99781593],
     [ 0.24718982,  0.03466764, -0.04019557 , 0.96675895 , 0.99782538 , 0.99749265],
     [-0.92900045,  0.12796586, -0.02959894 ,-0.36313565 , 0.98995881 , 0.998478],
     [-0.99093385,  0.30963827,  0.04399168 ,-0.03849395 , 0.94132017 , 0.99682202],
     [ 0.63991822,  0.08676728,  0.01266549, -0.76207316 , 0.99382095,  0.99825609],
     [-0.97041808, -0.04349611,  0.05179524, -0.2217277 ,  0.99625264,  0.99575635],
     [-0.75931431,  0.34058989,  0.08872104,  0.64092271 , 0.93278592,  0.99194674],
     [ 0.36390551,  0.10402673,  0.04212833, -0.92760847 , 0.99118484,  0.9969696 ],
     [-0.02068825,  0.04477927,  0.00249808,  0.99825918 , 0.99533083,  0.99785909],
     [-0.93060167,  0.09966637,  0.02097796,  0.35872446 , 0.99066755,  0.99873714],
     [ 0.559026  ,  0.1759681 , -0.07896671,  0.82671968,  0.98174557,  0.99525479],
     [-0.98538106, -0.04002435,  0.01965351,  0.13296903,  0.99573206,  0.99765323],
     [ 0.98356559,  0.00115694, -0.00883092, -0.14943953,  0.99380634,  0.99798697],
     [-0.63086791,  0.09089955,  0.04917653,  0.77313566,  0.99302347,  0.9975129 ],
     [ 0.96297726,  0.68287062,  0.09528844 ,-0.21831189,  0.72485644,  0.99080621],
     [ 0.89273093,  0.11288946,  0.02832545, -0.43880685,  0.99064152 , 0.99760898],
     [-0.36066788,  0.23448267,  0.3728395 ,  0.92279967,  0.95020208 , 0.91946087],
     [ 0.37875795,  0.19846965, -0.14137089,  0.92281866,  0.97781774 , 0.98612405],
     [ 0.43960992,  0.05180215 ,-0.03794313,  0.89593427,  0.99690567,  0.99759366],
     [ 0.95803985,  0.08486974 ,-0.00948285,  0.27001179,  0.98916025,  0.99752139],
     [-0.3050887 ,  0.06243963 , 0.05041205,  0.95028344,  0.99524137,  0.99639665],
     [-0.74702989,  0.11269824 ,-0.01525002, -0.65815991,  0.99057759,  0.9981728 ],
     [-0.29650311,  0.05262828 ,-0.0096012 , -0.95147564,  0.99610075,  0.99613081],
     [-0.51872658,  0.27368644 ,-0.43666126, -0.8388962 ,  0.94688394,  0.89374456],
     [-0.5108259 ,  0.11407817 ,-0.03612973, -0.85536679,  0.98887529,  0.99624544],
     [ 0.68169805,  0.07487039 ,-0.02669289,  0.72808247,  0.99466217,  0.99823521],
     [ 0.84553889,  0.14025878 ,-0.02059248,  0.52720822,  0.98423765,  0.99785371],
     [ 0.65659539,  0.52835238 ,-0.2444696,  0.74462741 , 0.84188589,  0.94337074],
     [ 0.01983669,  0.02214991,  0.00109092, -0.99691492 , 0.99825084,  0.99899499]
    ]
)

class dataLoaderSingleObject(object):
    def __init__(self,
                 Pascal3DDataPath=None,
                 trainOrVal='train',
                 ):
        self.dataStart = 0
        self.dataLength = 0
        self.epoch = 0

        self._Pascal3DDataPath = Pascal3DDataPath
        self._trainOrVal = trainOrVal
        self._isTrain = True
        if self._trainOrVal == 'train':
            self._isTrain = True
        elif self._trainOrVal == 'val':
            self._isTrain = False
        else:
            print('choose \'train\' or \'val\'')
            return
        self._dataPathList = []
        self._CAD3DShapes = {}

        print('set pascal3d dataset...')

        self._getTrainList()
        self._loadDataPath()
        self._load3DShapes()
        self._dataPathShuffle()

    def _getTrainList(self):
        print('set train or val list...')
        self._Pascal3DTrainList = []
        datasetList = os.listdir(os.path.join(self._Pascal3DDataPath, 'Image_sets/'))
        for datasetName in datasetList:
            if os.path.isdir(os.path.join(self._Pascal3DDataPath, 'Image_sets/', datasetName)):
                txtFileList = os.listdir(os.path.join(self._Pascal3DDataPath, 'Image_sets/',datasetName))
                for txtFileName in txtFileList:
                    className = txtFileName.split('.')[0].split('_')[0]
                    trainval = txtFileName.split('.')[0].split('_')[-1]
                    if trainval==self._trainOrVal:
                        with open(os.path.join(self._Pascal3DDataPath, 'Image_sets/',datasetName,txtFileName)) as txtFilePointer:
                            dataPointList = txtFilePointer.readlines()
                            for i, dataPoint in enumerate(dataPointList):
                                if datasetName == 'pascal':
                                    dp = dataPoint.split('\n')[0].split(' ')[0]
                                    isTrue = int(dataPoint.split('\n')[0].split(' ')[-1])
                                    if int(isTrue)==1:
                                        self._Pascal3DTrainList.append(dp)
                                else:
                                    dp = dataPoint.split('\n')[0].split(' ')[0]
                                    self._Pascal3DTrainList.append(dp)
        print('done!')

    def _loadDataPath(self):
        print('load datapoint path...')
        datasetList = os.listdir(os.path.join(self._Pascal3DDataPath, 'training_data'))
        for datasetName in datasetList:
            if datasetName == 'imagenet' or datasetName == 'pascal':
                dataPointList = os.listdir(os.path.join(self._Pascal3DDataPath, 'training_data', datasetName))
                for dataPointName in dataPointList:
                    if dataPointName in self._Pascal3DTrainList:
                        if os.path.isdir(os.path.join(self._Pascal3DDataPath, 'training_data', datasetName, dataPointName)):
                            dataPointPath = os.path.join(self._Pascal3DDataPath, 'training_data', datasetName, dataPointName)
                            self._dataPathList.append(dataPointPath)
        self._dataPathList = np.array(self._dataPathList)
        self.dataLength = len(self._dataPathList)
        print('done!')

    def _dataPathShuffle(self):
        # print('')
        # print('data path shuffle...')
        self.dataStart = 0
        np.random.shuffle(self._dataPathList)
        self.dataLength = len(self._dataPathList)
        # print('done! : ' + str(self.dataLength))

    def _load3DShapes(self):
        print('load 3d shapes for pascal3d...')
        self._classIndex = {}
        self._CAD3DShapes = {}
        classList = os.listdir(os.path.join(self._Pascal3DDataPath, 'CAD'))
        classList.sort(key=natural_keys)
        classNum = 0
        for className in classList:
            if os.path.isdir(os.path.join(self._Pascal3DDataPath, 'CAD', className)):
                if self._CAD3DShapes.get(className) is None:
                    self._classIndex[className] = classNum
                    self._CAD3DShapes[className] = []
                    classNum += 1
                CADModelList = os.listdir(os.path.join(self._Pascal3DDataPath, 'CAD', className))
                CADModelList.sort(key=natural_keys)
                for CADModel in CADModelList:
                    if CADModel.split(".")[-1] == 'npy':
                        shape = np.load(os.path.join(self._Pascal3DDataPath, 'CAD', className, CADModel)).reshape((64, 64, 64, 1))
                        self._CAD3DShapes[className].append(shape)
                self._CAD3DShapes[className] = np.array(self._CAD3DShapes[className])
                self._CAD3DShapes[className] = np.where(self._CAD3DShapes[className] > 0, 1.0, 0.0)
        self._classNum = classNum
        print('done!')

    def getKmeansAEI(self, k=None, max_iter=1000):
        AEI = []
        i = 0
        for dataPath in self._dataPathList:
            print(i,len(self._dataPathList))
            i += 1
            objFolderList = os.listdir(dataPath)
            objSelectedList = []
            for objFolder in objFolderList:
                if os.path.isdir(os.path.join(dataPath, objFolder)):
                    objInfoTXT = os.path.join(dataPath, objFolder, 'objInfo.txt')
                    with open(objInfoTXT) as objInfoPointer:
                        objInfo = objInfoPointer.readline()
                    className = objInfo.split(' ')[0]
                    if self._CAD3DShapes[className] is not None:
                        objSelectedList.append(objInfo)
                    for objIndex, objSelected in enumerate(objSelectedList):
                        className,imagePath,CADModelPath,colMin,rowMin,colMax,rowMax,azimuth,elevation,inPlaneRot=objSelected.split(' ')
                        azimuth, elevation, inPlaneRot = float(azimuth) / 180.0 * np.pi, float(elevation) / 180.0 * np.pi, float(inPlaneRot) / 180.0 * np.pi
                        sin = np.sin([azimuth, elevation, inPlaneRot])
                        cos = np.cos([azimuth, elevation, inPlaneRot])
                        AEI.append([sin[0], sin[1], sin[2], cos[0], cos[1], cos[2]])
        AEI = np.array(AEI)
        kmeans = kmeans_cosine(X=AEI, k=k, max_iter=max_iter)
        centers, xtoc, dist = kmeans.kmeansSample()
        return centers, xtoc, dist

    def getNextBatch(self, batchSizeof3DShape=32, imageSize=None, augmentation=True):
        inputImages = []
        outputImages, classList, instList, EulerRadList = [], [], [], []
        item_num = 0
        while len(outputImages)==0:
            for dataPath in self._dataPathList[self.dataStart:]:
                objFolderList = os.listdir(dataPath)
                np.random.shuffle(objFolderList)
                # objFolderList.sort()
                objSelectedList = []
                for objFolder in objFolderList:
                    objFolder = objFolder.decode("utf-8")
                    if os.path.isdir(os.path.join(dataPath, objFolder)):
                        objInfoTXT = os.path.join(dataPath, objFolder, 'objInfo.txt')
                        with open(objInfoTXT) as objInfoPointer:
                            objInfo = objInfoPointer.readline()
                        className = objInfo.split(' ')[0]
                        # if className == 'car':
                        #     objSelectedList.append(objInfo)
                        if self._CAD3DShapes[className] is not None:
                            objSelectedList.append(objInfo)
                np.random.shuffle(objSelectedList)
                if len(objSelectedList) > 0:
                    if len(outputImages) >= batchSizeof3DShape and len(inputImages)>0:
                        break
                    try:
                        image2DPath = os.path.join(self._Pascal3DDataPath, objSelectedList[0].split(' ')[1])
                        inputImage = cv2.imread(image2DPath, cv2.IMREAD_COLOR)
                        row_org, col_org, _ = inputImage.shape
                        for objIndex, objSelected in enumerate(objSelectedList):
                            className,imagePath,CADModelPath,colMin,rowMin,colMax,rowMax,azimuth,elevation,inPlaneRot=objSelected.split(' ')
                            colMin, rowMin, colMax, rowMax, azimuth, elevation, inPlaneRot = float(colMin), float(rowMin), float(
                                colMax), float(rowMax), float(azimuth), float(elevation), float(inPlaneRot)
                            width, height = colMax - colMin, rowMax - rowMin
                            if augmentation:
                                border_ratio = np.random.rand() * 0.2
                            else:
                                border_ratio = 0.1
                            colMin, rowMin = np.max((0, colMin - width * border_ratio)), np.max(
                                (0, rowMin - height * border_ratio))
                            colMax, rowMax = np.min((col_org, colMax + width * border_ratio)), np.min(
                                (row_org, rowMax + height * border_ratio))
                            image = inputImage[int(rowMin):int(rowMax), int(colMin):int(colMax), :]

                            is_flip = False
                            if augmentation:
                                is_flip = np.random.rand() > 0.5
                            if is_flip:
                                image = cv2.flip(image, 1)
                                if np.random.rand() > 0.5:
                                    image = cv2.flip(image, 0)
                            # image augmentation
                            image, _, _, _, _, _ = imageRandomAugmentation(
                                inputImage=image, imageRowFinal=imageSize[0], imageColFinal=imageSize[1],
                                imAug=augmentation, imAugPr=0.5, padding=False,
                                randomTrans=augmentation, randomScale=augmentation,
                                transPr=0.5, scalePr=0.5, transRatioMax=0.2,
                                scaleRatioMin=0.8, scaleRatioMax=1.2
                            )
                            image = image / 255.

                            azimuth, elevation, inPlaneRot = float(azimuth) / 180.0 * np.pi, float(elevation) / 180.0 * np.pi, float(inPlaneRot) / 180.0 * np.pi
                            # for crossdataset
                            # inPlaneRot = 0.
                            if is_flip:
                                azimuth = -azimuth
                            EulerRad = np.array([azimuth, elevation, inPlaneRot])
                            classVector = np.zeros(self._classNum)
                            classVector[self._classIndex[className]] = 1
                            cadIndex = int(CADModelPath.split('/')[-1])
                            instVector = np.zeros(10)
                            instVector[cadIndex - 1] = 1
                            # car 3d shape
                            if (cadIndex - 1 < 0) or (cadIndex > len(self._CAD3DShapes)):
                                for i in range(1000):
                                    print('pascal ', cadIndex - 1)
                                    return
                            inst3DCAD = self._CAD3DShapes[className][cadIndex - 1]

                            inputImages.append(image)
                            outputImages.append(inst3DCAD)
                            classList.append(classVector)
                            instList.append(instVector)
                            EulerRadList.append(EulerRad)
                            item_num += 1
                            if item_num >= batchSizeof3DShape:
                                break
                    except:
                        pass
                self.dataStart += 1
                if self.dataStart >= self.dataLength:
                    self.epoch += 1
                    self._dataPathShuffle()
                    break
        inputImages = np.array(inputImages).astype('float32')
        outputImages = np.array(outputImages).astype('float32')
        classList = np.array(classList).astype('float32')
        instList = np.array(instList).astype('float32')
        EulerRadList = np.array(EulerRadList).astype('float32')

        return instList, classList, np.sin(EulerRadList), np.cos(EulerRadList), inputImages, outputImages

# average image size of pascal3D : (508.54, 404.47)
class dataLoader(object):
    def __init__(self,
                 imageSize=(640,480),
                 gridSize=(20,15),
                 predNumPerGrid=5,
                 Pascal3DDataPath=None,
                 trainOrVal='train',
                 ):
        self.dataStart = 0
        self.dataLength = 0
        self.epoch = 0

        self._imageSize = imageSize
        self._gridSize = gridSize
        self._predNumPerGrid = predNumPerGrid
        self._Pascal3DDataPath = Pascal3DDataPath
        self._trainOrVal = trainOrVal
        self._isTrain = True
        if self._trainOrVal == 'train':
            self._isTrain = True
        elif self._trainOrVal == 'val':
            self._isTrain = False
        else:
            print('choose \'train\' or \'val\'')
            return
        self._dataPathList = []
        self._CAD3DShapes = None

        print('set pascal3d dataset...')

        self._getTrainList()
        self._loadDataPath()
        self._load3DShapes()
        self._dataPathShuffle()

    def _getTrainList(self):
        print('set train or val list...')
        self._Pascal3DTrainList = []
        datasetList = os.listdir(os.path.join(self._Pascal3DDataPath, 'Image_sets/'))
        for datasetName in datasetList:
            if os.path.isdir(os.path.join(self._Pascal3DDataPath, 'Image_sets/', datasetName)):
                txtFileList = os.listdir(os.path.join(self._Pascal3DDataPath, 'Image_sets/',datasetName))
                for txtFileName in txtFileList:
                    className = txtFileName.split('.')[0].split('_')[0]
                    trainval = txtFileName.split('.')[0].split('_')[-1]
                    if className=='car' and trainval==self._trainOrVal:
                        with open(os.path.join(self._Pascal3DDataPath, 'Image_sets/',datasetName,txtFileName)) as txtFilePointer:
                            dataPointList = txtFilePointer.readlines()
                            for i, dataPoint in enumerate(dataPointList):
                                if datasetName == 'pascal':
                                    dp = dataPoint.split('\n')[0].split(' ')[0]
                                    isTrue = int(dataPoint.split('\n')[0].split(' ')[-1])
                                    if int(isTrue)==1:
                                        self._Pascal3DTrainList.append(dp)
                                else:
                                    dp = dataPoint.split('\n')[0].split(' ')[0]
                                    self._Pascal3DTrainList.append(dp)
        print('done!')

    def _loadDataPath(self):
        print('load datapoint path...')
        datasetList = os.listdir(os.path.join(self._Pascal3DDataPath, 'training_data'))
        for datasetName in datasetList:
            if datasetName == 'imagenet' or datasetName == 'pascal':
                dataPointList = os.listdir(os.path.join(self._Pascal3DDataPath, 'training_data', datasetName))
                for dataPointName in dataPointList:
                    if dataPointName in self._Pascal3DTrainList:
                        if os.path.isdir(os.path.join(self._Pascal3DDataPath, 'training_data', datasetName, dataPointName)):
                            dataPointPath = os.path.join(self._Pascal3DDataPath, 'training_data', datasetName, dataPointName)
                            self._dataPathList.append(dataPointPath)
        self._dataPathList = np.array(self._dataPathList)
        self.dataLength = len(self._dataPathList)
        print('done!')

    def _dataPathShuffle(self):
        print('')
        print('data path shuffle...')
        self.dataStart = 0
        np.random.shuffle(self._dataPathList)
        self.dataLength = len(self._dataPathList)
        print('done! : ' + str(self.dataLength))

    def _load3DShapes(self):
        print('load 3d shapes for pascal3d...')
        self._CAD3DShapes = []
        CADModelList = os.listdir(os.path.join(self._Pascal3DDataPath, 'CAD', 'car'))
        CADModelList.sort()
        for CADModel in CADModelList:
            if CADModel.split(".")[-1] == 'npy':
                shape = np.load(os.path.join(self._Pascal3DDataPath, 'CAD', 'car', CADModel)).reshape(64, 64, 64, 1)
                self._CAD3DShapes.append(shape)
        self._CAD3DShapes = np.array(self._CAD3DShapes)
        self._CAD3DShapes = np.where(self._CAD3DShapes>0, 1.0, 0.0)
        print('done!')

    def _getOffset(self, batchSize):
        offsetX = np.transpose(np.reshape(
            np.array([np.arange(self._gridSize[0])]*self._gridSize[1]*self._predNumPerGrid),
            (self._predNumPerGrid, self._gridSize[1], self._gridSize[0])), (1,2,0))
        offsetX = np.tile(np.reshape(offsetX, (1,self._gridSize[1],self._gridSize[0],self._predNumPerGrid)),[batchSize,1,1,1])
        offsetY = np.transpose(np.reshape(
            np.array([np.arange(self._gridSize[1])]*self._gridSize[0]*self._predNumPerGrid),
            (self._predNumPerGrid, self._gridSize[0], self._gridSize[1])), (2,1,0))
        offsetY = np.tile(np.reshape(offsetY, (1,self._gridSize[1],self._gridSize[0],self._predNumPerGrid)),[batchSize,1,1,1])
        return offsetX.astype('float32'), offsetY.astype('float32')

    def getNextBatch(self, batchSizeof3DShape=32, imageSize=None, gridSize=None):
        if imageSize!=None:
            self._imageSize = imageSize
        if gridSize!=None:
            self._gridSize = gridSize
        inputImages, bboxImages, objnessImages, eulerRadImages = [], [], [], []
        outputImages, instList = [], []
        while len(outputImages)==0:
            for dataPath in self._dataPathList[self.dataStart:]:
                objFolderList = os.listdir(dataPath)
                np.random.shuffle(objFolderList)
                # objFolderList.sort()
                objSelectedList = []
                for objFolder in objFolderList:
                    if os.path.isdir(os.path.join(dataPath, objFolder)):
                        objInfoTXT = os.path.join(dataPath, objFolder, 'objInfo.txt')
                        with open(objInfoTXT) as objInfoPointer:
                            objInfo = objInfoPointer.readline()
                        className = objInfo.split(' ')[0]
                        if className == 'car':
                            objSelectedList.append(objInfo)
                if len(objSelectedList) > 0:
                    if len(outputImages) + len(objSelectedList) > batchSizeof3DShape and len(inputImages)>0:
                        break
                    try:
                        image2DPath = os.path.join(self._Pascal3DDataPath, objSelectedList[0].split(' ')[1])
                        inputImage = cv2.imread(image2DPath, cv2.IMREAD_COLOR)

                        is_flip = np.random.rand() > 0.5
                        if is_flip:
                            inputImage = cv2.flip(inputImage, flipCode=1)

                        # imageRow, imageCol, channel = inputImage.shape
                        # # make this a square image
                        # imgRowColMax = np.max((imageRow, imageCol))
                        # heightBorderSize = (imgRowColMax - imageRow) / 2
                        # widthBorderSize = (imgRowColMax - imageCol) / 2
                        # inputImage = cv2.copyMakeBorder(
                        #     inputImage, top=heightBorderSize, bottom=heightBorderSize,
                        #     left=widthBorderSize, right=widthBorderSize, borderType=cv2.BORDER_CONSTANT,
                        #     value=[0, 0, 0])

                        imageRowOrg, imageColOrg, _ = inputImage.shape
                        inputImage, rowPad, dRow, colPad, dCol, scale = imageRandomAugmentation(
                            inputImage=inputImage, imageRowFinal=self._imageSize[1], imageColFinal=self._imageSize[0],
                            imAug=self._isTrain, imAugPr=0.5, padding=False,
                            randomTrans=self._isTrain, randomScale=self._isTrain,
                            transPr=0.5, scalePr=0.5, transRatioMax=0.2,
                            scaleRatioMin=0.8, scaleRatioMax=1.2
                        )
                        inputImage = inputImage/255.

                        imageRow = imageRowOrg + 2*rowPad
                        imageCol = imageColOrg + 2*colPad

                        objOrderingImage = -1 * np.ones([self._gridSize[1], self._gridSize[0], self._predNumPerGrid])
                        bboxImage = np.zeros([self._gridSize[1], self._gridSize[0], self._predNumPerGrid, 4])
                        objnessImage = np.zeros([self._gridSize[1], self._gridSize[0], self._predNumPerGrid, 1])
                        eulerRadImage = np.zeros([self._gridSize[1], self._gridSize[0], self._predNumPerGrid, 3])
                        outputPerImage, instPerImage, EulerPerImage = [], [], []
                        itemIndex = 0
                        for objIndex, objSelected in enumerate(objSelectedList):
                            className,imagePath,CADModelPath,colMin,rowMin,colMax,rowMax,azimuth,elevation,inPlaneRot=objSelected.split(' ')
                            colMin, rowMin, colMax, rowMax, azimuth, elevation = float(colMin), float(rowMin), float(
                                colMax), float(rowMax), float(azimuth), float(elevation)
                            if is_flip:
                                colMin, colMax = imageColOrg - colMax, imageColOrg - colMin
                                azimuth, inPlaneRot = -azimuth, -inPlaneRot
                            # #border for square
                            # rowMin, rowMax = float(rowMin) + heightBorderSize, float(rowMax) + heightBorderSize
                            # colMin, colMax = float(colMin) + widthBorderSize, float(colMax) + widthBorderSize
                            # augmentation
                            colMin = (float(colMin) + colPad - imageCol/2) * scale + imageCol/2 + dCol
                            colMax = (float(colMax) + colPad - imageCol/2) * scale + imageCol/2 + dCol
                            rowMin = (float(rowMin) + rowPad - imageRow/2) * scale + imageRow/2 + dRow
                            rowMax = (float(rowMax) + rowPad - imageRow/2) * scale + imageRow/2 + dRow
                            # colMin = np.max((0.0, colMin))
                            # colMax = np.min((colMax, imageCol))
                            # rowMin = np.max((0.0, rowMin))
                            # rowMax = np.min((rowMax, imageRow))
                            azimuth,elevation,inPlaneRot = -float(azimuth)/180.0*np.pi,-float(elevation)/180.0*np.pi,float(inPlaneRot)/180.0*np.pi
                            cadIndex = int(CADModelPath.split('/')[-1])

                            rowCenterOnGrid = (rowMax+rowMin)/2.0*self._gridSize[1]/imageRow
                            colCenterOnGrid = (colMax+colMin)/2.0*self._gridSize[0]/imageCol
                            rowIndexOnGrid = int(rowCenterOnGrid)
                            colIndexOnGrid = int(colCenterOnGrid)
                            dx,dy = colCenterOnGrid - colIndexOnGrid, rowCenterOnGrid - rowIndexOnGrid
                            bboxHeight,bboxWidth = np.min((1.0, (rowMax-rowMin)/imageRow)),np.min((1.0, (colMax-colMin)/imageCol))
                            for predIndex in range(self._predNumPerGrid):
                                if (rowIndexOnGrid>=0 and rowIndexOnGrid<self._gridSize[1]) \
                                    and (colIndexOnGrid>=0 and colIndexOnGrid<self._gridSize[0]) \
                                    and bboxHeight > 0 and bboxWidth > 0 \
                                    and objnessImage[rowIndexOnGrid, colIndexOnGrid, predIndex] != 1:
                                    # objectness and bounding box
                                    objnessImage[rowIndexOnGrid,colIndexOnGrid,predIndex]=1
                                    bboxImage[rowIndexOnGrid,colIndexOnGrid,predIndex,0:4] = bboxHeight,bboxWidth,dx,dy
                                    eulerRadImage[rowIndexOnGrid,colIndexOnGrid,predIndex,:] = azimuth, elevation, inPlaneRot

                                    # car instance vector
                                    carInstVector = np.zeros(len(self._CAD3DShapes))
                                    carInstVector[cadIndex-1] = 1
                                    # car 3d shape
                                    if(cadIndex-1<0) or (cadIndex>len(self._CAD3DShapes)):
                                        for i in range(1000):
                                            print('pascal ', cadIndex-1)
                                            return
                                    car3DCAD = self._CAD3DShapes[cadIndex-1]
                                    # # Euler angle in rad
                                    # EulerRad = np.array([azimuth,elevation,inPlaneRot])

                                    # append items
                                    outputPerImage.append(car3DCAD)
                                    instPerImage.append(carInstVector)
                                    # EulerPerImage.append(EulerRad)
                                    # set item order
                                    objOrderingImage[rowIndexOnGrid, colIndexOnGrid, predIndex] = itemIndex
                                    itemIndex += 1
                                    break
                        if itemIndex > 0:
                            inputImages.append(inputImage)
                            bboxImages.append(bboxImage)
                            objnessImages.append(objnessImage)
                            eulerRadImages.append(eulerRadImage)

                            for gridRow in range(self._gridSize[1]):
                                for gridCol in range(self._gridSize[0]):
                                    for predIndex in range(self._predNumPerGrid):
                                        objOrder = int(objOrderingImage[gridRow, gridCol, predIndex])
                                        if objOrder>=0:
                                            outputImages.append(outputPerImage[objOrder])
                                            instList.append(instPerImage[objOrder])
                                            # EulerRadList.append(EulerPerImage[objOrder])
                    except:
                        pass
                self.dataStart += 1
                if self.dataStart >= self.dataLength:
                    self.epoch += 1
                    self._dataPathShuffle()
                    break
        inputImages = np.array(inputImages).astype('float32')
        objnessImages = np.array(objnessImages).astype('float32')
        bboxImages = np.array(bboxImages).astype('float32')
        eulerRadImages = np.array(eulerRadImages).astype('float32')
        outputImages = np.array(outputImages).astype('float32')
        instList = np.array(instList).astype('float32')
        # EulerRadList = np.array(EulerRadList).astype('float32')

        offsetX,offsetY = self._getOffset(batchSize=len(inputImages))

        # print inputImages.shape
        # print instList.shape
    #     return offsetX, offsetY, inputImages, objnessImages,\
    # bboxImages, np.sin(EulerRadList), np.cos(EulerRadList), \
    # outputImages, instList
        return offsetX, offsetY, inputImages, objnessImages, \
               bboxImages, np.sin(eulerRadImages), np.cos(eulerRadImages), \
               outputImages, instList

# data_loader_pascal = dataLoaderSingleObject(trainOrVal='val',
#                                              Pascal3DDataPath='/media/yonsei/4TB_HDD/dataset/PASCAL3D+_release1.1/')
#
# center, xtoc, dist = data_loader_pascal.getKmeansAEI(k=30*12)
# print(center)
# print(np.median(dist))
# np.save('../../val.npy', center)


import requests
from bs4 import BeautifulSoup
import re
headers = {
    'User-Agent':'Mozilla/4.0(compatible;MSIE 5.5;Windows NT)',}
url = 'http://www.biqukan.com/3_3039/1351331.html'
html = requests.get(url,headers=headers)
# 获取到网页的文本格式
html = html.text
soup = BeautifulSoup(html,'lxml')
# 选择器得到的是一个列表，只有一个元素，要选取第一个
zhangjie = soup.select('div.content h1')[0]
# 得到章节的文本内容
zhangjie = zhangjie.text
print(zhangjie)
# 查找属性是id为'content'且class为'showtxt'的标签
text = soup.find(id='content', class_='showtxt')
# 得到文本内容
text=text.text
# 将得到的文本存入文件中
with open('xiaoshuo.txt','w',encoding='utf-8') as f:
    f.write(zhangjie)
    f.write(text)


from Gaudi.Configuration import *

# Data service
from Configurables import FCCDataSvc
podioevent = FCCDataSvc("EventDataSvc")

# DD4hep geometry service
# Parses the given xml file
from Configurables import GeoSvc, SimG4SingleParticleGeneratorTool
geoservice = GeoSvc("GeoSvc", detectors=['file:Detector/DetFCChhBaseline1/compact/FCChh_DectEmptyMaster.xml',
  'file:Detector/DetFCChhTrackerTkLayout/compact/Tracker.xml'],
                    OutputLevel = DEBUG)

# Geant4 service
# Configures the Geant simulation: geometry, physics list and user actions
from Configurables import SimG4Svc
# giving the names of tools will initialize the tools of that type
geantservice = SimG4Svc("SimG4Svc", detector='SimG4DD4hepDetector', physicslist="SimG4FtfpBert", actions="SimG4FullSimActions")

#geantservice.g4PostInitCommands  += ["/tracking/verbose 1"]

# Geant4 algorithm
# Translates EDM to G4Event, passes the event to G4, writes out outputs via tools
from Configurables import SimG4Alg, SimG4SaveTrackerHits
# first, create a tool that saves the tracker hits
# Name of that tool in GAUDI is "XX/YY" where XX is the tool class name ("SimG4SaveTrackerHits")
# and YY is the given name ("saveTrackerHits")
savetrackertool = SimG4SaveTrackerHits("saveTrackerHits", readoutNames = ["TrackerBarrelReadout", "TrackerEndcapReadout"])
savetrackertool.positionedTrackHits.Path = "positionedHits"
savetrackertool.trackHits.Path = "hits"
# next, create the G4 algorithm, giving the list of names of tools ("XX/YY")
pgun = SimG4SingleParticleGeneratorTool("GeantinoGun", etaMin=-5, etaMax=5, particleName="geantino")
geantsim = SimG4Alg("SimG4Alg",
                    outputs= ["SimG4SaveTrackerHits/saveTrackerHits" ],
                    eventProvider=pgun)

# PODIO algorithm
from Configurables import PodioOutput
out = PodioOutput("out",
                   OutputLevel=DEBUG)
out.outputCommands = ["keep *"]

# ApplicationMgr
from Configurables import ApplicationMgr
ApplicationMgr( TopAlg = [geantsim, out],
                EvtSel = 'NONE',
                EvtMax   = 10000,
                # order is important, as GeoSvc is needed by SimG4Svc
                ExtSvc = [podioevent, geoservice, geantservice],
                OutputLevel=DEBUG
 )


from keras.preprocessing import sequence
from sklearn.cluster import KMeans
from sklearn.metrics.pairwise import euclidean_distances
from sklearn.preprocessing import LabelEncoder

import multiprocessing
import os
import pandas

import numpy as np

def get_input_sequences(input_data, padding="avg", return_class_encoder=False):
    classes = sorted([f for f in os.listdir(input_data)
                      if os.path.isdir(os.path.join(input_data, f)) and not f.startswith('.')],
                     key=lambda f: f.lower())
    X = []
    Y = []
    sequence_lengths = []
    for class_name in classes:
        files = sorted([f for f in os.listdir(os.path.join(input_data, class_name))
                        if os.path.isfile(os.path.join(input_data, class_name, f)) and not f.startswith('.')
                        and f.endswith(".csv")], key=lambda f: f.lower())
        for file in files:
            df = pandas.read_csv(os.path.join(input_data, class_name, file))
            values = df.values
            nan_inds = np.where(np.isnan(values))
            values[nan_inds] = 0
            inf_inds = np.where(np.isinf(values))
            values[inf_inds] = np.sign(values[inf_inds])
            X.append(values)
            sequence_lengths.append(len(df.values))
            Y.append(class_name)
    sequence_lengths = np.array(sequence_lengths)
    X = np.array(X)
    if padding == "avg":
        length = int(sequence_lengths.mean())
    elif padding == "max":
        length = max(sequence_lengths)
    elif padding == "min":
        length = min(sequence_lengths)
    else:
        print("No padding selected, returning raw sequences.")
        length = None
    if length is not None:
        X = sequence.pad_sequences(X, maxlen=length, dtype="float64")
    Y = np.array(Y)
    encoder = LabelEncoder()
    encoder.fit(Y)
    Y = encoder.transform(Y)
    if return_class_encoder:
        return X, Y, encoder
    else:
        return X, Y

def kmeans_frame_selection(frames, k=20, seed=0):

    kmeans = KMeans(n_clusters=k, random_state=seed, n_jobs=multiprocessing.cpu_count()).fit(frames)
    centers = kmeans.cluster_centers_
    frame_numbers = []
    for center in centers:
        distances = [distance[0] for distance in euclidean_distances(frames, [center])]
        frame_number = distances.index(min(distances))
        frame_numbers.append(frame_number)
    frame_numbers.sort()

    representative_frames = []
    for i in range(len(frame_numbers)):
        frame = frames[frame_numbers[i]]
        representative_frames.append(frame)

    return np.array(representative_frames), np.array(frame_numbers)

def kmeans_seq_reduction(dataset, k=20, seed=0):

    for stream_idx, stream in enumerate(dataset):
        new_dataset = []
        for instance in stream:
            instance_length = instance.shape[0]
            new_instance, frames = kmeans_frame_selection(instance, min(k, instance_length), seed)
            new_dataset.append(new_instance)
        dataset[stream_idx] = sequence.pad_sequences(new_dataset, maxlen=k, dtype="float64")
    return dataset

def kmeans_sync_seq_reduction(dataset, k=20, seed=0):

    for instance_idx in range(dataset[0].shape[0]):
        view_vectors = []
        for stream in dataset:
            view_vectors.append(stream[instance_idx])
        instance = np.concatenate(tuple(view_vectors), axis=1)
        instance_length = instance.shape[0]
        conc_instance, frames = kmeans_frame_selection(instance, min(k, instance_length), seed)

        for stream_idx in range(len(dataset)):
            dataset[stream_idx][instance_idx] = dataset[stream_idx][instance_idx][frames]
    for idx, stream in enumerate(dataset):
        dataset[idx] = sequence.pad_sequences(stream.tolist(), maxlen=k, dtype="float64")
    return dataset

def multiple_sequence_padding(input_streams, padding="max"):

    lengths = []
    for stream in input_streams:
        length = stream.shape[-2]
        lengths.append(length)
    lengths = np.array(lengths)

    if padding == "avg":
        max_length = int(lengths.mean())
    elif padding == "max":
        max_length = max(lengths)
    elif padding == "min":
        max_length = min(lengths)
    else:
        raise ValueError("padding must be either avg, max or min")

    for idx, x in enumerate(input_streams):
        input_streams[idx] = sequence.pad_sequences(x, maxlen=max_length, dtype="float64")

    return input_streams

def multiple_sequence_padding_means(input_streams, padding="max"):

    lengths = []
    for stream in input_streams:
        for instance in stream:
            length = instance.shape[-2]
            lengths.append(length)
    lengths = np.array(lengths)

    if padding == "avg":
        max_length = int(lengths.mean())
    elif padding == "max":
        max_length = max(lengths)
    elif padding == "min":
        max_length = min(lengths)
    else:
        raise ValueError("padding must be either avg, max or min")

    for stream_idx, stream in enumerate(input_streams):
        input_streams[stream_idx] = kmeans_seq_reduction(stream, k=max_length)

    for idx, x in enumerate(input_streams):
        input_streams[idx] = sequence.pad_sequences(x, maxlen=max_length, dtype="float64")

    return input_streams

def pad_views_sequences(views_list, length):
    padded_list = []
    for view in views_list:
        view_len = view.shape[0]
        turn = 0
        while view_len > length:
            view = np.delete(view, (turn), axis=0)
            view_len = view.shape[0]
            turn = (turn + 1) * -1
        padded_list.append(view)
    return padded_list

def synchronize_views(input_streams):

    num_instances = input_streams[0].shape[0]

    for instance_idx in range(num_instances):

        seq_lengths = []
        samples = []
        for view in input_streams:
            seq_lengths.append(view[instance_idx].shape[0])
            samples.append(view[instance_idx])
        length = min(seq_lengths)
        padded_instances = pad_views_sequences(samples, length)

        for view_idx in range(len(padded_instances)):
            input_streams[view_idx][instance_idx] = padded_instances[view_idx]

    return input_streams


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Задание 7.3b

Сделать копию скрипта задания 7.3a.

Переделать скрипт:
- Запросить у пользователя ввод номера VLAN.
- Выводить информацию только по указанному VLAN.

Ограничение: Все задания надо выполнять используя только пройденные темы.

"""

#Решение

vlan = input('Введите номер VLAN: ')
vlans = []
with open('CAM_table.txt','r') as f:
    for line in f:
        line = line.strip().split()
        if line and line[0].isdigit():
            vlans.append([int(line[0]),line[1:]])
    for v in sorted(vlans):
        if vlan.isdigit() and int(vlan) == v[0]:
            print('{:>4} {:>18} {:>8}'.format(v[0],v[1][0],v[1][2]))


import xml.etree.ElementTree as xml
import json
import argparse
import re
import copy

def build_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument('-xml', '--xml-file', type=str, required=True)
    parser.add_argument('-out', '--out-file', type=str, required=True)
    group1 = parser.add_mutually_exclusive_group()
    group1.add_argument('-dets-only', '--detections-only', action='store_true')
    group1.add_argument('-imgs-info', '--images-info', action='store_true')
    parser.add_argument('-shortened-file-names', '--shortened-file-names', action='store_true')
    return parser

def cvat_segmentation_to_coco(cvat_segmentation):
    coco_segmentation = [[float(x) for x in re.split(',|;', cvat_segmentation)]]
    return coco_segmentation

def segmentation_to_bbox(segmentation):  # json segmentation
    segmentation = segmentation[0]
    x = segmentation[0::2]
    y = segmentation[1::2]
    bbox = [min(x), min(y), max(x)-min(x), max(y)-min(y)]
    return bbox

def shorten_file_names(images_to_shorten):
    images = copy.deepcopy(images_to_shorten)
    if len(images) == 0:
        return images
    if len(images) == 1:
        images[0]['file_name'] = images[0]['file_name'].split('/')[-1]
        return images
    splitted_file_name = images[0]['file_name'].split('/')
    shortening_deep = len(splitted_file_name)-1
    if shortening_deep == 0:
        return
    for image in images[1:]:
        splitted_prev_file_name = splitted_file_name
        splitted_file_name = image['file_name'].split('/')
        for i in range(shortening_deep):
            if splitted_file_name[i] != splitted_prev_file_name[i]:
                shortening_deep = i
                if shortening_deep == 0:
                    return images
                break
    for image in images:
        image['file_name'] = '/'.join(image['file_name'].split('/')[shortening_deep:])
    return images

def cvat_root2coco_dict(root, detections_only=False, images_info=False, shortened_file_names=False):
    assert (not detections_only) or (not images_info)
    images = []
    annotations = []
    categories = []

    # get categories
    labels = root.find('meta').find('task').find('labels')
    category_name_to_id = {}
    cat_id = 1
    for label in labels:
        category = {"id": cat_id, "name": label[0].text}
        categories.append(category)
        category_name_to_id[label.find('name').text] = cat_id
        cat_id += 1

    # get images and annotations
    ann_id = 1
    for cvat_image in root.findall('image'):
        image = dict()
        image["id"] = int(cvat_image.attrib["id"])
        image["height"] = int(cvat_image.attrib["height"])
        image["width"] = int(cvat_image.attrib["width"])
        image["file_name"] = cvat_image.attrib.pop("name")
        images.append(image)
        for box in cvat_image.findall('box'):
            annotation = dict()
            bbox = [float(box.attrib["xtl"]), float(box.attrib["ytl"]),
                    float(box.attrib["xbr"]) - float(box.attrib["xtl"]),
                    float(box.attrib["ybr"]) - float(box.attrib["ytl"])]
            annotation["area"] = bbox[2] * bbox[3]
            annotation["iscrowd"] = 0
            annotation["image_id"] = int(cvat_image.attrib["id"])
            annotation["bbox"] = bbox
            annotation["category_id"] = category_name_to_id[box.attrib["label"]]
            annotation["id"] = ann_id
            if "score" in box.attrib.keys():
                annotation["score"] = float(box.attrib["score"])
            annotations.append(annotation)
            ann_id += 1
        for polygon in cvat_image.findall('polygon'):
            annotation = dict()
            annotation["iscrowd"] = 0
            annotation["image_id"] = int(cvat_image.attrib["id"])
            annotation["segmentation"] = cvat_segmentation_to_coco(polygon.attrib["points"])
            annotation["bbox"] = segmentation_to_bbox(annotation["segmentation"])
            annotation["area"] = annotation["bbox"][2] * annotation["bbox"][3]
            annotation["category_id"] = category_name_to_id[polygon.attrib["label"]]
            annotation["id"] = ann_id
            if "score" in polygon.attrib.keys():
                annotation["score"] = float(polygon.attrib["score"])
            annotations.append(annotation)
            ann_id += 1

    if shortened_file_names:
        images = shorten_file_names(images)
    if detections_only:
        json_dict = {annotations}
    elif images_info:
        json_dict = {'images': images, 'categories': categories}
    else:
        json_dict = {'images': images, 'annotations': annotations, 'categories': categories}
    return json_dict

def cvat2coco(xml_file, out_file, detections_only=False, images_info=False, shortened_file_names=False):
    tree = xml.parse(xml_file)
    root = tree.getroot()
    json_dict = cvat_root2coco_dict(root, detections_only, images_info, shortened_file_names)
    with open(out_file, 'w') as f:
        json.dump(json_dict, f, indent=2)

if __name__ == '__main__':
    parser = build_parser()
    args = parser.parse_args()
    cvat2coco(**vars(args))


#!/usr/bin/env python
# -*- coding: ascii -*-
from __future__ import print_function
from __future__ import unicode_literals
from future import standard_library
standard_library.install_aliases()
from builtins import str
from builtins import range
from builtins import object

import random
import numpy as np

class UpdateWVector( object ):
    """
    Update the w_vector of a FeatureFunction object (i.e. self.feature_func.w_vector).
    """

    def __init__(self, feature_func):

        self.feature_func = feature_func
        self.sim = feature_func.sim

    def get_best_eps_greedy_action(self, s_vector, epsgreedy_obj=None ):
        """
        Pick the best action for state "s_vector" based on max Q(s,a)
        If epsgreedy_obj is given, apply Epsilon Greedy logic to choice.
        """
        a_descL = self.sim.get_state_legal_action_list( s_vector )
        if a_descL:
            best_a_desc, best_a_val = a_descL[0], float('-inf')
            bestL = [best_a_desc]
            for a in a_descL:
                q = self.feature_func.get_QsaEst( a, s_vector )
                if q > best_a_val:
                    best_a_desc, best_a_val = a, q
                    bestL = [ a ]
                elif q == best_a_val:
                    bestL.append( a )

            best_a_desc = random.choice( bestL )
            if epsgreedy_obj is not None:
                best_a_desc = epsgreedy_obj( best_a_desc, a_descL )

            return best_a_desc
        return None

    def get_best_greedy_action(self, s_vector):
        """Use eps greedy logic without an epsgreedy_obj."""
        return self.get_best_eps_greedy_action( s_vector )

    def get_max_Qsa(self, s_vector):
        """return the maximum Q(s,a) for state, s_vector."""
        a_best = self.get_best_greedy_action( s_vector )
        if a_best is None:
            return None, None
        return a_best, self.feature_func.get_QsaEst( a_best, s_vector )

    def sarsa_update(self, s_vector='', a_desc='', alpha=0.1, gamma=1.0,
                     sn_vector='', an_desc='', reward=0.0):
        """
        Do a SARSA, Temporal-Difference-style learning rate update.
        Use estimated Q(s,a) values by evaluating linear function approximation.
        w = w + alpha * [R + gamma*QEst(s',a') - QEst(s,a)] * grad(s,a)
        """
        Qsat = self.feature_func.get_QsaEst( a_desc, s_vector )

        if self.sim.is_terminal_state( s_vector=sn_vector ):
            delta = alpha * (reward - Qsat)
        else:
            Qsatp1 = self.feature_func.get_QsaEst( an_desc, sn_vector )
            target_val = reward + gamma*Qsatp1

            delta = alpha * (target_val - Qsat)

        delta_vector = delta * self.feature_func.get_gradient( a_desc, s_vector )
        self.feature_func.w_vector += delta_vector

        # remember max amount of change due to [s_vector][a_desc]
        #delta = np.max( np.absolute( delta_vector ) )
        #self.record_changes( s_vector, a_desc, delta )

        #return abs(delta) # return the absolute value of change

    def qlearning_update(self, s_vector='', a_desc='', sn_vector='',
                         alpha=0.1, gamma=1.0, reward=0.0):
        """
        Do a Q-Learning, Temporal-Difference-style learning rate update.
        Use estimated Q(s,a) values by evaluating linear function approximation.
        w = w + alpha * [R + gamma* max(QEst(s',a')) - QEst(s,a)] * grad(s,a)
        """
        Qsat = self.feature_func.get_QsaEst( a_desc, s_vector )

        if self.sim.is_terminal_state( s_vector=sn_vector ):
            delta = alpha * (reward - Qsat)
        else:
            # find best Q(s',a')
            an_descL = self.sim.get_state_legal_action_list( sn_vector )

            if an_descL:
                best_a_desc, best_a_val = an_descL[0], float('-inf')
                for a in an_descL:
                    q = self.feature_func.get_QsaEst( a, sn_vector )
                    if q > best_a_val:
                        best_a_desc, best_a_val = a, q
            else:
                best_a_val = 0.0

            # use best Q(s',a') to update Q(s,a)
            target_val = reward + gamma * best_a_val
            delta = alpha * (target_val - Qsat)

        delta_vector = delta * self.feature_func.get_gradient( a_desc, s_vector )
        self.feature_func.w_vector += delta_vector

        # remember max amount of change due to [s_vector][a_desc]
        #delta = np.max( np.absolute( delta_vector ) )
        #self.record_changes( s_vector, a_desc, delta )

        #return abs(delta) # return the absolute value of change

if __name__=="__main__":

    from introrl.continuous_sims.sim_continuous import ContinuousSimulation
    from introrl.continuous_sims.feature_func import FeatureFunction
    from introrl.agent_supt.epsilon_calc import EpsilonGreedy

    sim = ContinuousSimulation(name='Mountain Car', step_reward=-1.0)

    ff = FeatureFunction( sim, name='Proportional', init_w_val=None)

    uv = UpdateWVector( ff )

    s_vector = sim.get_s_vector()
    epsgreedy_obj = EpsilonGreedy(epsilon=0.5,
                                  const_epsilon=True, half_life=200,
                                  N_episodes_wo_decay=0)

    a_best = uv.get_best_eps_greedy_action( s_vector, epsgreedy_obj=epsgreedy_obj )
    print('at',s_vector,'  a_best=',a_best)
    print('Best Qsa =', uv.get_max_Qsa(s_vector) )


# simple_device.py

from my_states import LocalizeState, CompletedState

class StateMachine(object):
    """
    A simple state machine that mimics the functionality of a device from a
    high level.
    """

    def __init__(self, robot):
        """ Initialize the components. """
        self.robot = robot
        # Start with a default state.
        self.state = LocalizeState(self.robot, 'A')

    async def run(self):
        while self.state is not CompletedState:
            await self.state.do_action()
            self.state = self.state.go_to_next_state()

    # def on_event(self, event, robot):
    #     """
    #     This is the bread and butter of the state machine. Incoming events are
    #     delegated to the given states which then handle the event. The result is
    #     then assigned as the new state.
    #     """

    #     # The next state will be the result of the on_event function.
    #     self.state = self.state.on_event(event, robot)
    #     self.state = self.state.return_to_idle()


import colander
import deform.widget
from persistent.dict import PersistentDict
from zope.interface import implementer

from substanced.content import content
from substanced.schema import NameSchemaNode
from substanced.util import renamer, get_oid

from dace.descriptors import (
    SharedUniqueProperty, CompositeMultipleProperty,
    SharedMultipleProperty)
from dace.util import getSite, get_obj
from pontus.core import VisualisableElementSchema
from pontus.widget import (
    Select2Widget, AjaxSelect2Widget)
from pontus.schema import Schema
from pontus.file import ObjectData, File

from .interface import IComment
from novaideo.core import (
    Commentable, Emojiable, can_access, SignalableEntity,
    CorrelableEntity)
from novaideo import _, log
from novaideo.content import get_file_widget
from novaideo.utilities.util import (
    text_urls_format,
    get_emoji_form,
    get_files_data, connect, disconnect)
from novaideo.content.correlation import CorrelationType
from novaideo.widget import FilesWidget

@colander.deferred
def intention_choice(node, kw):
    root = getSite()
    values = [(str(i), i) for i in root.comment_intentions]
    values.insert(0, ('', _('- Select -')))
    return Select2Widget(
        values=values,
        item_css_class="comment-form-group comment-intention-form")

@colander.deferred
def relatedcontents_choice(node, kw):
    request = node.bindings['request']
    context = node.bindings['context']
    root = getSite()
    values = []
    if isinstance(context, Comment) and\
       context.related_correlation:
        values = [(get_oid(t), t.title) for
                  t in context.associated_contents]

    def title_getter(id):
        try:
            obj = get_obj(int(id), None)
            if obj:
                return obj.title
            else:
                return id
        except Exception as e:
            log.warning(e)
            return id

    ajax_url = request.resource_url(
        root, '@@novaideoapi',
        query={'op': 'find_correlable_entity'})
    return AjaxSelect2Widget(
        values=values,
        ajax_url=ajax_url,
        ajax_item_template="related_item_template",
        css_class="search-idea-form",
        title_getter=title_getter,
        multiple=True,
        page_limit=50,
        item_css_class="comment-form-group comment-related-form")

class RelatedContentsSchema(Schema):
    """Schema for associtation"""

    associated_contents = colander.SchemaNode(
        colander.Set(),
        widget=relatedcontents_choice,
        title=_('Associated contents'),
        description=_('Choose the contents to be associated'),
        missing=[],
        default=[],
        )

    associate = colander.SchemaNode(
        colander.Boolean(),
        widget=deform.widget.CheckboxWidget(
            item_css_class="hide-bloc"),
        label='',
        title='',
        default=False,
        missing=False
        )

@colander.deferred
def comment_textarea(node, kw):
    request = node.bindings['request']
    emoji_form = get_emoji_form(
        request, emoji_class='comment-form-group')
    return deform.widget.TextAreaWidget(
        rows=2, cols=60, item_css_class="comment-form-group comment-textarea",
        emoji_form=emoji_form,
        template='novaideo:views/templates/textarea_comment.pt',
        request=request)

def context_is_a_comment(context, request):
    return request.registry.content.istype(context, 'comment')

class CommentSchema(VisualisableElementSchema):
    """Schema for comment"""

    name = NameSchemaNode(
        editing=context_is_a_comment,
        )

    intention = colander.SchemaNode(
        colander.String(),
        widget=intention_choice,
        title=_('Intention'),
        description=_('Choose your intention'),
        default=_('Remark')
        )

    associated_contents = colander.SchemaNode(
        colander.Set(),
        widget=relatedcontents_choice,
        title=_('Associated contents'),
        description=_('Choose contents to associate'),
        missing=[],
        default=[],
        )

    files = colander.SchemaNode(
        colander.Sequence(),
        colander.SchemaNode(
            ObjectData(File),
            name=_("File"),
            widget=get_file_widget()
            ),
        widget=FilesWidget(
            add_subitem_text_template='',
            item_css_class="files-block comment-form-group comment-files"),
        missing=[],
        description=_('Add files to your comment'),
        title=_('Attached files'),
        )

    comment = colander.SchemaNode(
        colander.String(),
        validator=colander.Length(max=2000),
        widget=comment_textarea,
        title=_("Message")
        )

    anonymous = colander.SchemaNode(
        colander.Boolean(),
        widget=deform.widget.CheckboxWidget(
            item_css_class="comment-form-group comment-anonymous-form"),
        label=_('Remain anonymous'),
        description=_('Check this box if you want to remain anonymous.'),
        title='',
        missing=False,
        default=False
        )

@content(
    'comment',
    icon='glyphicon glyphicon-align-left',
    )
@implementer(IComment)
class Comment(Commentable, CorrelableEntity, Emojiable, SignalableEntity):
    """Comment class"""
    icon = 'icon ion-chatbubbles'
    templates = {'default': 'novaideo:views/templates/comment_result.pt'}
    name = renamer()
    author = SharedUniqueProperty('author')
    files = CompositeMultipleProperty('files')
    url_files = CompositeMultipleProperty('url_files')
    related_correlation = SharedUniqueProperty('related_correlation', 'targets')
    contextualized_correlations = SharedMultipleProperty(
        'contextualized_correlations', 'context')

    def __init__(self, **kwargs):
        super(Comment, self).__init__(**kwargs)
        self.set_data(kwargs)
        self.urls = PersistentDict({})
        self.edited = False
        self.pinned = False

    @property
    def channel(self):
        """Return the channel of th commented entity"""

        if not isinstance(self.__parent__, Comment):
            return self.__parent__
        else:
            return self.__parent__.channel

    @property
    def root(self):
        """Return the root comment"""

        if not isinstance(self.__parent__, Comment):
            return self
        else:
            return self.__parent__.root

    @property
    def comment_parent(self):
        """Return the root comment"""

        if isinstance(self.__parent__, Comment):
            return self.__parent__
        else:
            return None

    @property
    def subject(self):
        """Return the commented entity"""
        return self.channel.get_subject()

    @property
    def challenge(self):
        return getattr(self.subject, 'challenge', None)

    @property
    def relevant_data(self):
        return [getattr(self, 'comment', ''),
                getattr(self.author, 'title',
                        getattr(self.author, '__name__', ''))]

    @property
    def related_contents(self):
        subject = self.subject
        return [content[0] for content in self.contextualized_contents
                if content[0] is not subject]

    @property
    def associated_contents(self):
        subject = self.subject
        return [content[0] for content in self.contextualized_contents
                if content[0] is not subject and not getattr(content[1], 'tags', [])]

    def set_associated_contents(self, associated_contents, user):
        subject = self.subject
        current_associated_contents = self.associated_contents
        associated_contents_to_add = [i for i in associated_contents
                                      if i not in current_associated_contents]
        associated_contents_to_del = [i for i in current_associated_contents
                                      if i not in associated_contents and
                                      i not in associated_contents_to_add]
        correlations = connect(
            subject,
            associated_contents_to_add,
            {'comment': _('Add related contents'),
             'type': _('Edit the comment')},
            author=user,)
        for correlation in correlations:
            correlation.setproperty('context', self)

        disconnect(
            subject,
            associated_contents_to_del)

    def get_title(self):
        return self.subject.title

    def presentation_text(self, nb_characters=400):
        return getattr(self, 'comment', "")[:nb_characters]+'...'

    def get_discuss_url(self, request, user):
        subject = self.channel.get_subject(user)
        return request.resource_url(
            subject, "@@index") + '#comment-' + str(get_oid(self, 'None'))

    def get_url(self, request):
        return request.resource_url(
            request.root, "@@seecomment", query={'comment_id': get_oid(self)})

    def get_related_contents(self, user):
        return [r for r in self.related_contents if can_access(user, r)]

    def format(self, request, is_html=False):
        comment = getattr(self, 'comment', '')
        all_urls, url_files, text_urls, formatted_text = text_urls_format(
            comment, request, is_html)
        self.urls = PersistentDict(all_urls)
        self.setproperty('url_files', url_files)
        self.formatted_comment = formatted_text
        self.formatted_urls = text_urls

    def get_attached_files_data(self):
        return get_files_data(self.files)

    def can_add_reaction(self, user, process):
        return True


def _to_webdav(self, docs_base, release):
        """Upload to WebDAV store."""
        try:
            git_path = subprocess.check_output('git remote get-url origin 2>/dev/null', shell=True)
        except subprocess.CalledProcessError:
            git_path = ''
        else:
            git_path = git_path.decode('ascii').strip()
            git_path = git_path.replace('http://', '').replace('https://', '').replace('ssh://', '')
            git_path = re.search(r'[^:/]+?[:/](.+)', git_path)
            git_path = git_path.group(1).replace('.git', '') if git_path else ''
        url = None
        with self._zipped(docs_base) as handle:
            url_ns = dict(name=self.cfg.project.name, version=release, git_path=git_path)
            reply = requests.put(self.params['url'].format(**url_ns),
                                 data=handle.read(), headers={'Accept': 'application/json'})
            if reply.status_code in range(200, 300):
                notify.info("{status_code} {reason}".format(**vars(reply)))
                try:
                    data = reply.json()
                except ValueError as exc:
                    notify.warning("Didn't get a JSON response! ({})".format(exc))
                else:
                    if 'downloadUri' in data:  # Artifactory
                        url = data['downloadUri'] + '!/index.html'
            elif reply.status_code == 301:
                url = reply.headers['location']
            else:
                data = self.cfg.copy()
                data.update(self.params)
                data.update(vars(reply))
                notify.error("{status_code} {reason} for PUT to {url}".format(**data))

        if not url:
            notify.warning("Couldn't get URL from upload response!")
        return url


################################################
### SVM best case's recall & fall-out graph  ###
###            SVM best case's AUC           ###
###    SVM best case's decision boundary     ###
################################################

import matplotlib.pyplot as plt
import matplotlib as mpl

from sklearn import datasets
from sklearn.externals.six.moves import xrange

from sklearn import svm
from sklearn.svm import SVC
import numpy as np
import time

from sklearn.metrics import accuracy_score
from sklearn import metrics
from sklearn.metrics import confusion_matrix
import seaborn as sns
import mglearn
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.svm import LinearSVC
from sklearn.metrics import classification_report
from sklearn.metrics import log_loss
from sklearn.metrics import roc_curve

from sklearn.metrics import auc
from scipy import interp
from itertools import cycle

def plot_svc_decision_function(model, ax=None, plot_support=True):
    """Plot the decision function for a 2D SVC"""
    if ax is None:
        ax = plt.gca()
    xlim = ax.get_xlim()
    ylim = ax.get_ylim()

    # create grid to evaluate model
    x = np.linspace(xlim[0], xlim[1], 30)
    y = np.linspace(ylim[0], ylim[1], 30)
    Y, X = np.meshgrid(y, x)
    xy = np.vstack([X.ravel(), Y.ravel()]).T
    P = model.decision_function(xy).reshape(X.shape)

    # plot decision boundary and margins
    ax.contour(X, Y, P, colors='k',
               levels=[-1, 0, 1], alpha=0.5,
               linestyles=['--', '-', '--'])

    # plot support vectors
    if plot_support:
        ax.scatter(model.support_vectors_[:, 0],
                   model.support_vectors_[:, 1],
                   s=300, linewidth=1, facecolors='none')
    ax.set_xlim(xlim)
    ax.set_ylim(ylim)

## Choose only one of the three flags.
recall_fallout = False
roc_curve_flag = False
svm_show = True

problem = 0
# problem = 1

if problem == 0:
    train_input = np.loadtxt('./data/p1_train_input.txt')
    train_target = np.loadtxt('./data/p1_train_target.txt')
    test_input = np.loadtxt('./data/p1_test_input.txt')
    test_target = np.loadtxt('./data/p1_test_target.txt')
else:
    train_input = np.loadtxt('./data/p2_train_input.txt')
    train_target = np.loadtxt('./data/p2_train_target.txt')
    test_input = np.loadtxt('./data/p2_test_input.txt')
    test_target = np.loadtxt('./data/p2_test_target.txt')

xx, yy = np.meshgrid(np.linspace(-3, 3, 500),
                     np.linspace(-3, 3, 500))

clf = SVC(kernel='rbf', C=10, gamma=10, random_state=100)
clf.fit(train_input, train_target)

y_pred = clf.predict(test_input)

cm = confusion_matrix(test_target, y_pred)
print("confusion_matrix")
print(cm)

print("classification_report")
print(classification_report(test_target, y_pred, target_names=['class 0', 'class 1']))

recall = cm[0][0] / (cm[0][0] + cm[0][1])
fallout = cm[1][0] / (cm[1][0] + cm[1][1])
print("recall = {:.5f}".format(recall))
print("fallout = {:.5f}".format(fallout))

fpr, tpr, thresholds = roc_curve(test_target, y_pred)
_fpr = 1-fpr[np.argmax(np.abs(tpr - fpr))]
_tpr = tpr[np.argmax(np.abs(tpr - fpr))]
print("Specificity = {:.5f}".format(_fpr))
print("Sensitivity = {:.5f}".format(_tpr))

_auc = auc(fpr, tpr)
print("AUC = {:.5f}".format(_auc))

########################################
############ Draw ROC curve ############
########################################
lw = 2
n_classes = 2

_test_target = []
for tar in test_target:
    if tar == 0.0:
        _test_target.append([1, 0])
    else:
        _test_target.append([0, 1])
_test_target = np.array(_test_target)

_y_pred = []
for pre in y_pred:
    if pre == 0.0:
        _y_pred.append([1, 0])
    else:
        _y_pred.append([0, 1])
_y_pred = np.array(_y_pred)

if recall_fallout == True:
    plt.plot(fpr, tpr, 'o-', label="Logistic Regression")
    plt.plot([0, 1], [0, 1], 'k--', label="random guess")
    plt.plot([fallout], [recall], 'ro', ms=10)
    plt.xlabel('Fall-Out')
    plt.ylabel('Recall')
    plt.show()

if roc_curve_flag == True:
    # Compute ROC curve and ROC area for each class
    fpr = dict()
    tpr = dict()
    roc_auc = dict()
    for i in range(n_classes):
        fpr[i], tpr[i], _ = roc_curve(_test_target[:, i], _y_pred[:, i])
        roc_auc[i] = auc(fpr[i], tpr[i])

    # Compute micro-average ROC curve and ROC area
    fpr["micro"], tpr["micro"], _ = roc_curve(_test_target.ravel(), _y_pred.ravel())
    roc_auc["micro"] = auc(fpr["micro"], tpr["micro"])

    # Compute macro-average ROC curve and ROC area

    # First aggregate all false positive rates
    all_fpr = np.unique(np.concatenate([fpr[i] for i in range(n_classes)]))

    # Then interpolate all ROC curves at this points
    mean_tpr = np.zeros_like(all_fpr)
    for i in range(n_classes):
        mean_tpr += interp(all_fpr, fpr[i], tpr[i])

    # Finally average it and compute AUC
    mean_tpr /= n_classes

    fpr["macro"] = all_fpr
    tpr["macro"] = mean_tpr
    roc_auc["macro"] = auc(fpr["macro"], tpr["macro"])
    ## Plot all ROC curves
    plt.figure(1)
    plt.plot(fpr["micro"], tpr["micro"],
            label='micro-average ROC curve (area = {0:0.5f})'
                ''.format(roc_auc["micro"]),
            color='deeppink', linestyle=':', linewidth=4)

    plt.plot(fpr["macro"], tpr["macro"],
            label='macro-average ROC curve (area = {0:0.5f})'
                ''.format(roc_auc["macro"]),
            color='navy', linestyle=':', linewidth=4)

    colors = cycle(['aqua', 'darkorange', 'cornflowerblue'])
    for i, color in zip(range(n_classes), colors):
        plt.plot(fpr[i], tpr[i], color=color, lw=lw,
                label='ROC curve of class {0} (area = {1:0.5f})'
                ''.format(i, roc_auc[i]))

    plt.plot([0, 1], [0, 1], 'k--', lw=lw)
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('ROC curve')
    plt.legend(loc="lower right")
    plt.show()

if svm_show == True:
    plt.scatter(train_input[:, 0], train_input[:, 1], c=train_target, s=50, cmap='autumn')
    plot_svc_decision_function(clf)
    plt.scatter(clf.support_vectors_[:, 0], clf.support_vectors_[:, 1],
                s=300, lw=1, facecolors='none')
    plt.show()


from django.shortcuts import render
from blog.models import ArticleModel
from django.core.paginator import Paginator
from django.db.models import Q
#from IpBlock.ip_blocking import BlockedIpMiddleware

def homePage(request):
    search = request.GET.get('search')
    articles = ArticleModel.objects.order_by('-id')

    if search:
        articles = articles.filter(
            Q(title__icontains=search) |
            Q(content__icontains=search)
        ).distinct()

    page = request.GET.get('page')
    paginator = Paginator(articles, 3)

    return render(request, 'pages/homepage.html', context={
        'articles': paginator.get_page(page)
    })


from flask import Flask, jsonify, request

app = Flask(__name__, template_folder='templates', static_folder='')
app.debug = True

@app.route("/api/getData", methods=["GET"])
def get_data():
	"""get请求"""
	return jsonify({"code": 200, "msg": "success", "data": request.args}), 200

@app.route("/api/post", methods=["POST"])
def post():
	"""post请求"""
	request_content_type = request.headers['Content-Type'].split(";")[0].strip()
	if request_content_type == 'application/json':
		return jsonify({"code": 200, "msg": "success", "data": request.get_json()}), 200
	return jsonify({"code": 200, "msg": "success"}), 200

if __name__ == '__main__':
	app.run()


#!/usr/bin/python3

""" Load typology questionnaires and parse them into a working format """

import pandas
from xlrd.biffh import XLRDError

class UnexpectedTypologyFormatError (ValueError):
    """Exception to be thrown when a questionnaire is misformatted"""
    pass

def parse_typology_questionnaire(filename):
    questionnaire = pandas.ExcelFile(filename)

    metadata_sheet_name = 'Metadata'
    if metadata_sheet_name in questionnaire.sheet_names:
        metadata_sheet = questionnaire.parse(metadata_sheet_name, header=None)
        try:
            metadata_blob = "; ".join(metadata_sheet[0])
        except KeyError:
            metadata_blob = ""
    else:
        metadata_blob = ""
    metadata = metadata_blob

    weird_rows = set()
    answers = pandas.DataFrame(columns = ["a_text", "a_notes" ,"a_reference"])
    features = pandas.DataFrame(columns = ["q_id: Reesink", "q_text", "q_note: Nijmegen Typological Survey"])
    data_sheet_name = 'Questionnaire'
    try:
        data_sheet = questionnaire.parse(
            data_sheet_name,
            skiprows=[0, 1, 2, 3, 4, 5, 7]).dropna('columns', 'all')
    except XLRDError:
        raise UnexpectedTypologyFormatError("Typology sheet did not have a 'Questionnaire' sheet")
    for i, row in data_sheet.iterrows():
        if pandas.isnull(row["q_text"]):
            weird_rows.add(i)
        if pandas.isnull(row["a_text"]):
            weird_rows.add(i)
        else:
            features = features.append(row[["q_id: Reesink", "q_text", "q_note: Nijmegen Topological Survey"]])
            answer = row[["a_text", "a_notes", "a_reference"]]
            answer.name = row["q_text"].lower()
            answers = answers.append(answer)
            assert(len(features) == len(answers))
    return metadata, features, answers

if __name__=='__main__':
    import sys
    metadata, features, answers = parse_questionnaire(sys.argv[1])
    print(metadata)
    print(features)
    print(answers.to_string())


import logging

logger = logging.getLogger('upgrade')

def migrate(cr, installed_version):

	logger.info("Migrating Staff management to version %s", installed_version)

	cr.execute("UPDATE staff_scheduler s SET work_time = (Select (hour_to - hour_from) w from staff_scheduler t where s.id = t.id)")

	logger.info("Staff Management update... done!")


import numpy as np
from keras.models import Sequential
from keras.layers import BatchNormalization, Conv1D, LSTM, Dense, Dropout
from keras.callbacks import *
from keras.utils import np_utils
def get_available_gpus():
    from tensorflow.python.client import device_lib
    local_device_protos = device_lib.list_local_devices()
    return [x.name for x in local_device_protos if x.device_type == 'GPU']

#load data
file_path = 'data\\GoT.txt'
book = open(file_path, encoding = "utf8").read().lower()

#covert unique chars to int
chars = sorted(list(set(book)))

char_to_int = dict((c, i) for i, c in enumerate(chars))
int_to_char = dict((i, c) for i, c in enumerate(chars))

char_int_dict = {}
for integer, character in enumerate(chars):
	char_int_dict[character] = integer

total_chars = len(book)
unique_chars = len(chars)

filter_len = 250 #100
dataX = []
dataY = []
for i in range(0, total_chars - filter_len):
    lineX = book[i: i + filter_len]
    lineY = book[i + filter_len]

	#char_int_list = []
	#for char in lineX:
	#	char_int_list.append(char_int_dict[char])

	#dataX.append(char_int_list)
	#dataY.append(char_int_dict[lineY])
    dataX.append([char_to_int[char] for char in lineX])
    dataY.append(char_to_int[lineY])
total_filters = len(dataX)

#prepare training data
X_train = np.reshape(dataX, (total_filters, filter_len, 1))
X_train = X_train / float(unique_chars)
y_train = np_utils.to_categorical(dataY)

#print(X_train.shape)
# (144225, 100, 1)
#print(y_train.shape)
#(144225, 45)

'''
LSTM RNN MODEL

Pre: Input into this model (train,valid) should have been preprocessed and
reshaped into the following data format for X: [samples, time steps, features]
Post: Generates a keras sequential model instance to generate text predictions.
Model consists of a LSTM (Long Short Term Memory] layers.
'''

#========================MODEL==================++

starting_neurons =  16 # starting filters

if len(get_available_gpus())>0:
    # https://twitter.com/fchollet/status/918170264608817152?lang=en
    from keras.layers import CuDNNLSTM as LSTM # this one is about 3x faster on GPU instances
stroke_read_model = Sequential()
stroke_read_model.add(LSTM(starting_neurons * 16,input_shape=(X_train.shape[1], X_train.shape[2]) ,return_sequences = True))
stroke_read_model.add(Dropout(0.2))
stroke_read_model.add(LSTM(starting_neurons * 16, return_sequences = False))
stroke_read_model.add(Dropout(0.2))
#stroke_read_model.add(Dense(starting_neurons * 32))
#stroke_read_model.add(Dropout(0.2))
stroke_read_model.add(Dense(y_train.shape[1], activation = 'softmax'))
stroke_read_model.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['categorical_accuracy'])

#=======================HyperParameters===========

weight_path="weights_got.best.hdf5"
checkpoint = ModelCheckpoint(weight_path, monitor='loss', verbose=1,
                             save_best_only=True, mode='min', save_weights_only = True)
reduceLROnPlat = ReduceLROnPlateau(monitor='loss', factor=0.8, patience=4,
                                   verbose=1, mode='auto', min_lr=0.0001)
early = EarlyStopping(monitor="val_loss", mode="min", patience=20)
callbacks_list = [checkpoint, reduceLROnPlat] #early,

#========================TRAINING=================
from IPython.display import clear_output
stroke_read_model.fit(X_train, y_train,
                      #validation_split = 0.2,
                      batch_size = 512,#128,
                      epochs = 100,
                      callbacks = callbacks_list,
                      verbose = 1)
clear_output()

#========================PREDICTIONS==============

'''
model.load_weights('weights.best.hdf5')
stroke_read_model.compile(optimizer = 'adam', loss = 'categorical_crossentropy')

# reverse mapping to convert integers back into characters
int_to_char = dict((i, c) for i, c in enumerate(chars))

# pick a random seed
start = numpy.random.randint(0, len(dataX)-1)
pattern = dataX[start]
print ("Seed:")
print "\"", ''.join([int_to_char[value] for value in pattern]), "\""
# generate characters
for i in range(1000):
	x = numpy.reshape(pattern, (1, len(pattern), 1))
	x = x / float(n_vocab)
	prediction = model.predict(x, verbose=0)
	index = numpy.argmax(prediction)
	result = int_to_char[index]
	seq_in = [int_to_char[value] for value in pattern]
	sys.stdout.write(result)
	pattern.append(index)
	pattern = pattern[1:len(pattern)]
print "Done."
'''


from __future__ import absolute_import, division, print_function
from six.moves import range
import math
import scitbx.math
from scitbx import lbfgs
from scitbx.array_family import flex

class fit_cdf(object):
  """
  =============================================================================
  Class fits a distribution according to its cumulative distribution function

  Arguments:
    x_data - measured property (list)
    y_data - fraction of points with measured property (cdf) (list)
    distribution - type of distribution to be fit

  Useful accessible attributes:
    self.x - the final parameters (list)
    self.distribution - the distribution being modeled (see notes)

  Notes:
    The available distributions are,
      gaussian
      rayleigh
      rice
    The distribution argument should not have quotes since it is used directly
    to access the class.
    The lbfgs method is used to minimize, f = (predicted - observed)^2
  -----------------------------------------------------------------------------
  """
  def __init__(self,x_data=None,y_data=None,distribution=None,
               max_iterations=50):
    # setup data
    assert(len(x_data) == len(y_data))
    self.n = len(x_data)
    self.x_data = flex.double(x_data)
    self.y_data = flex.double(y_data)
    if (flex.max(self.y_data) > 1.0):
      print("The cumulative distribution function (y_data) should only have values be between 0 and 1.")
      exit()

    # intialize distribution with guess
    self.distribution = distribution()
    self.distribution.estimate_parameters_from_cdf(x_data=self.x_data,
                                                   y_data=self.y_data)
    self.x = self.distribution.get_parameters()

    # optimize parameters
    self.minimizer = lbfgs.run(target_evaluator=self)

    # set optimized parameters
    self.distribution.set_parameters(p=self.x)

  def compute_functional_and_gradients(self):
    # caculate difference between predicted and observed values
    self.distribution.set_parameters(p=self.x)
    is_cpp_ = getattr(self.distribution,"interface","Python")=="C++"
    if is_cpp_:
      predicted = self.distribution.cdf(x=self.x_data)
    else:
      predicted = flex.double(self.n)
      for i in range(self.n):
        predicted[i] = self.distribution.cdf(x=self.x_data[i])
    difference = predicted - self.y_data

    # target function for minimization is sum of rmsd
    f = flex.sum(flex.sqrt(difference*difference))
    if is_cpp_:
      gradients = self.distribution.gradients(x=self.x_data, nparams=len(self.x), difference=difference)
      return f,gradients
    gradients = flex.double(len(self.x))
    for i in range(self.n):
      g_i = self.distribution.cdf_gradients(x=self.x_data[i])
      for j in range(len(self.x)):
        gradients[j] = gradients[j] + difference[i]*g_i[j]
    gradients = 2.0*gradients
    return f,gradients

# =============================================================================
class rayleigh(object):
  """
  =============================================================================
  Class models a 1-d Rayleigh distribution using one parameter, sigma.

              x                x^2
    pdf = --------- exp(- ------------)
           sigma^2         2 sigma^2

                        x^2
    cdf = 1 - exp(- -----------)
                     2 sigma^2

  The derivative of the cdf with respect to sigma is,

      d(cdf)          x^2               x^2             x
    ---------- = - --------- exp( - -----------) = - ------- pdf
     d(sigma)       sigma^3          2 sigma^2        sigma

  Methods:
    set_parameters
    get_parameters
    estimate_parameters_from_cdf
    pdf
    cdf
    d_cdf_d_sigma
    d_cdf_d_sigma_finite
    cdf_gradients
  -----------------------------------------------------------------------------
  """
  def __init__(self,mean=None,sigma=None):

    # parameter for Rayleigh distribution
    if (sigma is None):
      sigma = 1.0
    self.sigma = sigma

  def set_parameters(self,p=None):
    """
    Function sets the all the parameters
    """
    assert(len(p) == 1)
    self.sigma = float(p[0])

  def get_parameters(self):
    """
    Function returns all the parameters
    """
    return flex.double([self.sigma])

  def estimate_parameters_from_cdf(self,x_data=None,y_data=None):
    """
    Function estimates the parameter values based on the data (cdf)
    """
    # sigma is the mode of the distribution
    # approximate with the median (cdf = 0.5)
    midpoint = None
    for i in range(len(x_data)):
      if (y_data[i] > 0.5):
        midpoint = i
        break
    if (midpoint is None):
      midpoint = len(x_data) - 1
    self.sigma = x_data[midpoint]

  def pdf(self,x=None):
    """
    Function returns the probability density function at x
    """
    x_sigma = x/self.sigma
    f = (x_sigma/self.sigma)*math.exp(-0.5*x_sigma*x_sigma)
    return f

  def cdf(self,x=None):
    """
    Function returns the cumulative distribution function at x
    """
    x_sigma = x/self.sigma
    f = 1.0 - math.exp(-0.5*x_sigma*x_sigma)
    return f

  def inv_cdf(self,cdf=None):
    """
    Function returns the inverse cumulative distribution function at cdf
    """
    return math.sqrt(-2.*self.sigma*self.sigma*math.log(1.-cdf))

  def d_cdf_d_sigma(self,x=None):
    """
    Function returns the derivative of the cdf at x with respect to the
    standard deviation
    """
    df = -(x/self.sigma)*self.pdf(x=x)
    return df

  def d_cdf_d_sigma_finite(self,x=None,delta=0.00001):
    """
    Function returns the derivative of the cdf at x with respect to the
    standard deviation
    """
    sigma0 = self.sigma
    self.sigma = sigma0 + delta
    f1 = self.cdf(x=x)
    self.sigma = sigma0 - delta
    f2 = self.cdf(x=x)
    self.sigma = sigma0
    return (f1-f2)/(2.0*delta)

  def cdf_gradients(self,x=None):
    """
    Function returns a flex.double containing all derivatives
    """
    return flex.double([self.d_cdf_d_sigma(x)])


"""
Link: https://leetcode.com/problems/reorder-list/description/

You are given the head of a singly linked-list. The list can be represented as:

L0 → L1 → … → Ln - 1 → Ln
Reorder the list to be on the following form:

L0 → Ln → L1 → Ln - 1 → L2 → Ln - 2 → …
You may not modify the values in the list's nodes. Only nodes themselves may be changed.
"""

# Definition for singly-linked list.
# class ListNode:
#     def __init__(self, val=0, next=None):
#         self.val = val
#         self.next = next
class Solution:
    def reorderList(self, head: Optional[ListNode]) -> None:
        if not head.next:
            return head

        fast, slow = head.next, head
        while fast and fast.next:
            fast = fast.next.next
            slow = slow.next

        cur_head = slow.next
        prev = slow
        while cur_head:
            tmp = cur_head.next
            cur_head.next = prev
            prev = cur_head
            cur_head = tmp
        slow.next = None

        cur_head = head
        while cur_head and prev:
            tmp = cur_head.next
            cur_head.next = prev
            tmp2 = prev.next
            prev.next = tmp
            prev = tmp2
            cur_head = tmp

        return head


# -*- coding:utf-8 -*-
import math

def solve():
    N, K = list(map(int, input().split()))

    ans = 0
    for i in range(1, N+1):
        if i >= K:
            ans += (1/N)
            continue
        # a*2^n >= K より
        # n >= log2(K/a)
        n = math.ceil(math.log2(K/i))
        ans += (1/N) * (1/2)**n

    print(ans)

if __name__ == "__main__":
    solve()