__author__ = 'Kazem'
import csv
import os
import glob
from os.path import isfile, join
import sys
import getopt
import collections
import matplotlib.pyplot as plt
import numpy as np
import fnmatch
import pandas as pd
from enum import Enum
from matplotlib.font_manager import FontProperties
from matplotlib import colors as mcolors
from os.path import basename
from operator import itemgetter, attrgetter, methodcaller
from osqp_utils import compute_failure_rates,compute_performance_profiles,plot_performance_profiles,\
    exclude_items,compute_speedup,exclude_items_general

INV='N/A'

def dir_contents(path, type):
    matches = []
    for root, dirnames, filenames in os.walk(path):
        for filename in fnmatch.filter(filenames, '*.'+ type):
            matches.append(os.path.join(root, filename))
    return matches

class solver_time:
    def __init__(self):
        self.fact_time = np.array([])
        self.analysis_time = np.array([])
        self.ordering_time = np.array([])
        self.solve_time = np.array([])
        self.refine_time = np.array([])

class MatrixRecord:
    def __init__(self, matrix_name='', nnz=0, dim1=0, dim2=0,
                 nnz_l=0, ordering_method='',
                 tree_height=0, norm2=0.0, residual=0.0, bwd_error=0.0, fwd_error=0.0):
        self.matrix_name = matrix_name
        self.nnz = nnz
        self.dim1 = dim1
        self.dim2 = dim2
        self.nnz_l = nnz_l
        self.ordering_method = ordering_method
        self.parallel_time = solver_time()
        self.serial_time = solver_time()
        self.tree_height = tree_height
        self.norm2 = norm2
        self.residual = residual
        self.bwd_error = bwd_error
        self.fwd_error = fwd_error

class RunRecord:
    def __init__(self, tool_name='', target_name='', num_thread=0):
        self.tool_name = tool_name
        self.num_thread = num_thread
        self.target_name = target_name
        self.mat_rec_list = []

OPTIMAL = 'optimal'
MAX_VAL = 1600
class Visualizer:

    def __init__(self, path):
        self.path = path
        self.run_rec_list = []
        self.headerList = []
        self.empty_dic = []
        self.font0 = FontProperties()
        self.qp2size = {}
        self.sympiler_header = ['Matrix Name', 'Row', 'Col', 'Nonzero', 'Factor Nonzero',
                                'Ordering Time (sec)', 'Analysis Time (sec)',
                                'Factorization Time (sec)', 'Solve Time (sec)',
                                'Iterative Refinement Time (sec)',
                                'Error Norm2-Before', 'Rel Error Norm2-Before',
                                'Infinity Norm-Before', 'BWD Error-Before', 'Residual-Before',
                                'Number of Threads',
                                'Target Name', 'Total Time (sec)', 'Tool Name',
                                'Iterative Refinement Method', 'Chunk Size',
                                'Error Norm2-After', 'Rel Error Norm2-After',
                                'Infinity Norm-After', 'BWD Error-After', 'Residual-After',
                                'Number of W-partition', 'Initial Cut', 'BLAS Threads',
                                'Final Cuts', 'Height', 'Number of Supernodes', 'Number of Pivots',
                                'Precision-Before', 'Precision-After', 'Number of Refinement Iterations',
                                'Factorization (GFLOPS)'
                                ]
        self.qp_header = ['Tool Name',
                           'Problem Name', 'Time (s)', 'Residual Lagrangian inf',
                          'Constraint Satisfaction Inf',
                          'Status', 'Problem Size', 'Hessian NNZ', 'Eq Constraint NNZ',
                          'Ineq Constraint NNZ', 'Outer GMRES Iter', 'GMRES Tol', 'Diagonal Pert', 'eps_abs',
                          'Inner GMRES Iter', 'Hessian dim','Non-negativity Inf','Complementarity Inf',
                          'Problem Type', 'Hessian dim', '# of Ineq Const', '# of Eq Constraints'
                                ]
    class LogHeader(Enum):
        MATN = 0
        ROW = 1
        COL = 2
        NNZ = 3
        NNZL = 4
        ORDT = 5
        ANAT = 6
        FACT = 7
        SOLT = 8
        ITERT = 9
        NRM1 = 10
        NRM2 = 11
        NRMI = 12
        BWDE = 13
        RESD = 14
        THRN = 15
        TRGT = 16
        TOTT = 17
        TOOL = 18
        IREF = 19
        CHNK = 20
        NR1A = 21
        NR2A = 22
        NRIA = 23
        BWEA = 24
        REDA = 25
        WPAR = 26
        ICUT = 27
        BLAS = 28
        FCUT = 29
        HGHT = 30
        SUPN = 31
        NPIV = 32
        PREC = 33
        PREA = 34
        NREF = 35
        FLPS = 36

    class qp_log_header(Enum):
        TOOLN = 0
        PROBN = 1
        STIME = 2
        LRNRM = 3
        CSNRM = 4
        STATS = 5
        PROBS = 6
        HSSNZ = 7
        CNSNZ = 8
        INQNZ = 9
        INIR = 10
        TOLIR = 11
        PERTB = 12
        EPSAB = 13
        OUTIR = 14
        HSSDM = 15
        NGINF = 16
        CMINF = 17
        CLSTY = 18
        PROBD = 19
        CNSID = 20
        CNSED = 21
        INVAL = 100

    def read_csv(self, report_path):
        #report = RunRecord()
        report = []
        with open(report_path) as csvFile:
            reader = csv.DictReader(csvFile)
            # cur_row = MatrixRecord()
            self.headerList = reader.fieldnames
            for row in reader:
                report.append(row)
        if len(report) > 0:
            self.empty_dic = report[0].copy()
            for key, value in self.empty_dic.items():
                self.empty_dic[key] = ''
        return report

    def fact_time_sympiler(self, sym_report_path, cmp_idx=LogHeader.FACT.value):
        mat_list = []
        sym_report = self.read_csv(sym_report_path)
        i1 = 0
        sw = True
        nxt_mat = ''
        while i1 < len(sym_report):
            sr = sym_report[i1]
            cur_mat = sr[self.sympiler_header[self.LogHeader.MATN.value]]  # matrix name
            i2 = 0
            if i1 + 1 < len(sym_report):
                nxt_mat = sym_report[i1 + 1][self.sympiler_header[self.LogHeader.MATN.value]]
            else:
                sw = False
            min_time = sys.maxsize
            min_idx = 0
            while cur_mat == nxt_mat and sw:
                cur_time = float(nxt_mat[self.sympiler_header[cmp_idx]])
                if min_time > cur_time:
                    min_time = cur_time
                    min_idx = i2
                i2 += 1
            mat_list.append(sym_report[i1 + min_idx])
            i1 += (i2 + 1)
        return mat_list

    def fact_time_mkl(self, mkl_report_path):
        mat_list = []
        new_dic = []
        file_token = 'Matrix Name***'
        time_afs = 'Total time spent'
        time_solve = 'Time spent in direct solver at solve step (solve)'
        extra_info_s = '+++extrainfo:'
        sup_node = 'number of supernodes:'
        iter_refine = 'Time spent in additional calculations'
        file_found = False
        analysis_time = False
        fact_time = False
        solve_time = False
        iter_ref_Time = False
        super_node = False
        extra_info = False
        #log = os.popen(mkl_report_path)
        f = open(mkl_report_path, "r")
        log = f.read()
        totoal_time = 0
        for line in log.splitlines(True):
            if not file_found:
                idx_f = line.find(file_token)
                if idx_f >= 0:
                    file_found = True
                    totoal_time = 0
                    new_dic = self.empty_dic.copy()
                    new_dic[self.sympiler_header[self.LogHeader.TOOL.value]] = 'MKL Pardiso'
                    mat_full_name = line.split()[2].split('/')
                    mat_name = mat_full_name[len(mat_full_name)-1].split('.')[0]
                    new_dic[self.sympiler_header[self.LogHeader.MATN.value]] = mat_name
                    continue
            if file_found and not analysis_time:
                idx_f = line.find(time_afs)
                if idx_f >= 0:
                    analysis_time = True
                    a_time = line.split()[4]
                    totoal_time += float(a_time)
                    new_dic[self.sympiler_header[self.LogHeader.ANAT.value]] = a_time
                    continue
            if analysis_time and not fact_time:
                idx_f = line.find(time_afs)
                if idx_f >= 0:
                    fact_time = True
                    a_time = line.split()[4]
                    totoal_time += float(a_time)
                    new_dic[self.sympiler_header[self.LogHeader.FACT.value]] = a_time
                    continue
            if fact_time and not solve_time:
                idx_f = line.find(time_solve)
                if idx_f >= 0:
                    solve_time = True
                    a_time = line.split()[10]
                    totoal_time += float(a_time)
                    new_dic[self.sympiler_header[self.LogHeader.SOLT.value]] = a_time
                    continue
            if solve_time and not iter_ref_Time:
                idx_f = line.find(iter_refine)
                if idx_f >= 0:
                    iter_ref_Time = True
                    a_time = line.split()[6]
                    totoal_time += float(a_time)
                    new_dic[self.sympiler_header[self.LogHeader.ITERT.value]] = a_time
                    continue
            if iter_ref_Time and not super_node:
                idx_f = line.find(sup_node)
                if idx_f >= 0:
                    super_node = True
                    tmp_buf = line.split()[3]
                    new_dic[self.sympiler_header[self.LogHeader.SUPN.value]] = tmp_buf
                    continue
            if super_node and not extra_info:
                idx_f = line.find(extra_info_s)
                if idx_f >= 0:
                    file_found = False
                    analysis_time = False
                    fact_time = False
                    solve_time = False
                    super_node = False
                    iter_ref_Time = False
                    super_node = False
                    tmp_buf = line.split(':')[1].split(',')
                    totoal_time += float(tmp_buf[0])
                    new_dic[self.sympiler_header[self.LogHeader.TOTT.value]] = str(totoal_time)
                    new_dic[self.sympiler_header[self.LogHeader.ORDT.value]] = tmp_buf[0]
                    new_dic[self.sympiler_header[self.LogHeader.NNZL.value]] = tmp_buf[1]
                    new_dic[self.sympiler_header[self.LogHeader.FLPS.value]] = tmp_buf[2]
                    new_dic[self.sympiler_header[self.LogHeader.ROW.value]] = tmp_buf[3]
                    new_dic[self.sympiler_header[self.LogHeader.NREF.value]] = tmp_buf[4]
                    new_dic[self.sympiler_header[self.LogHeader.NNZ.value]] = tmp_buf[5]
                    new_dic[self.sympiler_header[self.LogHeader.THRN.value]] = tmp_buf[6]
                    new_dic[self.sympiler_header[self.LogHeader.NRM1.value]] = tmp_buf[7]
                    new_dic[self.sympiler_header[self.LogHeader.NRM2.value]] = tmp_buf[8]
                    new_dic[self.sympiler_header[self.LogHeader.NRMI.value]] = tmp_buf[9]
                    new_dic[self.sympiler_header[self.LogHeader.NR1A.value]] = tmp_buf[7]
                    new_dic[self.sympiler_header[self.LogHeader.NR2A.value]] = tmp_buf[8]
                    new_dic[self.sympiler_header[self.LogHeader.NRIA.value]] = tmp_buf[9]
                    new_dic[self.sympiler_header[self.LogHeader.TRGT.value]] = tmp_buf[10]
                    #updating analysis time
                    anal_time = float(new_dic[self.sympiler_header[self.LogHeader.ANAT.value]])
                    ord_time = float(new_dic[self.sympiler_header[self.LogHeader.ORDT.value]])
                    total_anal_time = anal_time + ord_time
                    new_dic[self.sympiler_header[self.LogHeader.ANAT.value]] = str(total_anal_time)
                    mat_list.append(new_dic)
        mat_list_sorte = sorted(mat_list,
                                key=itemgetter(self.sympiler_header[Visualizer.LogHeader.MATN.value]))

        return mat_list_sorte

    def read_sympiler_log(self, report_path, idx_eq, idx_sort, f_sort):
        sym_log = self.read_csv(report_path)
        red_log = self.list_reduction(sym_log, idx_eq, idx_sort, f_sort)
        # extracting matrix name from the path
        for i in range(len(red_log)):
            full_name = red_log[i][self.sympiler_header[self.LogHeader.MATN.value]]
            mat_full_name = full_name.split('/')
            mat_name = mat_full_name[len(mat_full_name) - 1].split('.')[0]
            red_log[i][self.sympiler_header[self.LogHeader.MATN.value]] = mat_name
        red_sort_log = sorted(red_log, key=itemgetter(self.sympiler_header[idx_eq]))

        return red_sort_log

    def read_qp_log(self, report_path, idx_eq, idx_min=qp_log_header.INVAL, tol=1e-12):
        sym_log = self.read_csv(report_path)
        red_log = self.qp_lib_list_reduction(sym_log,idx_eq,idx_min,tol)
        #eee_tmp = self.qp_lib_list_reduction_2(sym_log,idx_eq,idx_min,tol)
        #return eee_tmp
        # extracting matrix name from the path
        for i in range(len(red_log)):
            full_name = red_log[i][self.qp_header[self.qp_log_header.PROBN.value]]
            mat_full_name = full_name.split('/')
            mat_name = mat_full_name[len(mat_full_name) - 1].split('.')[0]
            #mat_name = mat_name.split('_')
            us_pos = mat_name.rfind('_')
            if us_pos>0:
                mat_name = mat_name[0:us_pos]
            red_log[i][self.qp_header[self.qp_log_header.PROBN.value]] = mat_name
            psize = 0
            if not red_log[i][self.qp_header[self.qp_log_header.HSSNZ.value]] == INV:
                psize = int(red_log[i][self.qp_header[self.qp_log_header.HSSNZ.value]])
                #FIXME: remove this after the logs are fixed
                if "NASOQ" in red_log[i][self.qp_header[self.qp_log_header.TOOLN.value]]:
                    dim = int(red_log[i][self.qp_header[self.qp_log_header.HSSDM.value]])
                    psize = 2*psize - dim
            if not red_log[i][self.qp_header[self.qp_log_header.CNSNZ.value]] == INV:
                psize += int(red_log[i][self.qp_header[self.qp_log_header.CNSNZ.value]])
            if not red_log[i][self.qp_header[self.qp_log_header.INQNZ.value]] == INV:
                psize += int(red_log[i][self.qp_header[self.qp_log_header.INQNZ.value]])
            red_log[i][self.qp_header[self.qp_log_header.PROBS.value]] = psize

        red_sort_log = sorted(red_log, key=itemgetter(self.qp_header[idx_eq.value]))
        return red_sort_log

    def qp_lib_list_reduction(self,list_log, idx_eq,idx_min, tol):
        min_time = 1e8
        red_sort_log = sorted(list_log, key=itemgetter(self.qp_header[idx_eq.value]))
        reduced_list = []
        if len(list_log) <= 1:
            return list_log
        tmp_list = []
        tmp_sort = np.array([])
        i = 0
        r_i = 0 # num of reduced elements
        while i < len(red_sort_log):
            num_rep = 0
            cur_prob = red_sort_log[i][self.qp_header[idx_eq.value]]
            sat_norm = float(red_sort_log[i][self.qp_header[self.qp_log_header.CSNRM.value]])
            lag_norm = float(red_sort_log[i][self.qp_header[self.qp_log_header.LRNRM.value]])
            comp_norm = float(red_sort_log[i][self.qp_header[self.qp_log_header.CMINF.value]])
            nng_norm = float(red_sort_log[i][self.qp_header[self.qp_log_header.NGINF.value]])
            min_time = float(red_sort_log[i][self.qp_header[idx_min.value]])
            status = red_sort_log[i][self.qp_header[self.qp_log_header.STATS.value]]
            if sat_norm < tol and lag_norm < tol and min_time < MAX_VAL and nng_norm < tol  \
                    and (status == 'solved' or status == '1' or status == 'The optimality conditions are satisfied'):
                red_sort_log[i][self.qp_header[self.qp_log_header.STATS.value]] = OPTIMAL
            else:
                min_time = 1e8
                red_sort_log[i][self.qp_header[self.qp_log_header.STATS.value]] = 'Not converged'
            tmp_list.append(red_sort_log[i])
            if i+1 < len(red_sort_log):
                nxt_prob = red_sort_log[i+1][self.qp_header[idx_eq.value]]
            else:
                break;
            i = i+1
            while cur_prob == nxt_prob:
                nxt_time = float(red_sort_log[i][self.qp_header[idx_min.value]])
                nxt_sat_norm = float(red_sort_log[i][self.qp_header[self.qp_log_header.CSNRM.value]])
                nxt_lag_norm = float(red_sort_log[i][self.qp_header[self.qp_log_header.LRNRM.value]])
                if sat_norm < tol and lag_norm < tol and min_time < MAX_VAL and nng_norm < tol  \
                    and (status == 'solved' or status == '1'):
                    min_time = nxt_time
                    red_sort_log[i][self.qp_header[self.qp_log_header.STATS.value]] = OPTIMAL
                    tmp_list[r_i] = red_sort_log[i]
                i = i+1
                if i < len(red_sort_log):
                    nxt_prob = red_sort_log[i][self.qp_header[idx_eq.value]]
                else:
                    break
            r_i = r_i + 1
        return tmp_list

    def compute_qp_size(self, red_log):
        psize = 0
        if not red_log[self.qp_header[self.qp_log_header.HSSNZ.value]] == INV:
            psize = int(red_log[self.qp_header[self.qp_log_header.HSSNZ.value]])
        if not red_log[self.qp_header[self.qp_log_header.CNSNZ.value]] == INV:
            psize += int(red_log[self.qp_header[self.qp_log_header.CNSNZ.value]])
        if not red_log[self.qp_header[self.qp_log_header.INQNZ.value]] == INV:
            psize += int(red_log[self.qp_header[self.qp_log_header.INQNZ.value]])
        red_log[self.qp_header[self.qp_log_header.PROBS.value]] = psize
        return psize

    def find_problem_size(self, report_path, idx_eq, idx_min=qp_log_header.INVAL,
                          tol=1e-6, tol_str='1e-06', class_type=["All"]):
        prt_str = self.qp_header[self.qp_log_header.PERTB.value]
        ir_in = self.qp_header[self.qp_log_header.INIR.value]
        ir_out = self.qp_header[self.qp_log_header.OUTIR.value]
        ir_tol = self.qp_header[self.qp_log_header.TOLIR.value]

        sym_log = self.read_csv(report_path)
        #        tn = sym_log['Tool Name']
        splited_log = []
        if len(class_type) == 1:
            if class_type[0] == "" or class_type[0] == "All":
                splited_log = self.qp_lib_list_split_byname(sym_log)
            else:
                splited_log = self.qp_lib_list_split_bycat(sym_log, class_type)
        else:
            splited_log = self.qp_lib_list_split_bycat(sym_log, class_type)
        #        if tn == 'NASOQ-auto':
        #            splited_log = sym_log
        red_tol_log = []
        for lst in splited_log:
            tt = lst[0][self.qp_header[self.qp_log_header.TOOLN.value]]
            if (lst[0]['eps_abs'] == tol_str and not tt == 'QL') or (lst[0]['eps_abs'] == tol_str and tt == 'QL'):
                lst1 = self.qp_lib_list_reduction(lst, idx_eq, idx_min, tol)
                for l in lst1:
                    tool_name = l[self.qp_header[self.qp_log_header.TOOLN.value]]
                    prt_str_val = l[prt_str]
                    ir_in_val = l[ir_in]
                    ir_out_val = l[ir_out]
                    ir_tol_val = l[ir_tol]
                    # tn = tool_name + '_' + prt_str_val + '_' + ir_out_val + '_' + ir_in_val + '_' + ir_tol_val
                    # tn = tn.split('/')[0]
                    tn = tool_name
                    l[self.qp_header[self.qp_log_header.TOOLN.value]] = tn
                red_tol_log.append(lst1)

        for red_log in red_tol_log:
            if len(red_log) > 0:
                if "OSQP-polished" not in red_log[0][self.qp_header[self.qp_log_header.TOOLN.value]]:
                    continue
            for i in range(len(red_log)):
                if "OSQP-polished" in red_log[i][self.qp_header[self.qp_log_header.TOOLN.value]]:
                    full_name = red_log[i][self.qp_header[self.qp_log_header.PROBN.value]]
                    mat_full_name = full_name.split('/')
                    mat_name = mat_full_name[len(mat_full_name) - 1].split('.')[0]
                    # mat_name = mat_name.split('_')
                    us_pos = mat_name.rfind('_')
                    if us_pos > 0:
                        mat_name = mat_name[0:us_pos]
                    self.qp2size[mat_name] = self.compute_qp_size(red_log[i])

    def read_qp_log_split(self, report_path, idx_eq, idx_min=qp_log_header.INVAL,
                          tol=1e-6, tol_str='1e-06', class_type=["All"]):
        prt_str = self.qp_header[self.qp_log_header.PERTB.value]
        ir_in = self.qp_header[self.qp_log_header.INIR.value]
        ir_out = self.qp_header[self.qp_log_header.OUTIR.value]
        ir_tol = self.qp_header[self.qp_log_header.TOLIR.value]

        sym_log = self.read_csv(report_path)
#        tn = sym_log['Tool Name']
        splited_log = []
        if len(class_type)==1:
            if class_type[0] == "" or class_type[0] == "All":
                splited_log = self.qp_lib_list_split_byname(sym_log)
            else:
                splited_log = self.qp_lib_list_split_bycat(sym_log, class_type)
        else:
            splited_log = self.qp_lib_list_split_bycat(sym_log,class_type)
#        if tn == 'NASOQ-auto':
#            splited_log = sym_log
        red_tol_log = []
        for lst in splited_log:
            tt = lst[0][self.qp_header[self.qp_log_header.TOOLN.value]]
            if (lst[0]['eps_abs'] == tol_str and not tt == 'QL') or (lst[0]['eps_abs'] == tol_str and tt == 'QL'):
                lst1 = self.qp_lib_list_reduction(lst, idx_eq, idx_min, tol)
                for l in lst1:
                    tool_name = l[self.qp_header[self.qp_log_header.TOOLN.value]]
                    prt_str_val = l[prt_str]
                    ir_in_val = l[ir_in]
                    ir_out_val = l[ir_out]
                    ir_tol_val = l[ir_tol]
                    #tn = tool_name + '_' + prt_str_val + '_' + ir_out_val + '_' + ir_in_val + '_' + ir_tol_val
                    #tn = tn.split('/')[0]
                    tn = tool_name
                    l[self.qp_header[self.qp_log_header.TOOLN.value]] = tn
                red_tol_log.append(lst1)

        # extracting matrix name from the path and seting size

        lst_i = 0
        for red_log in red_tol_log:
            for i in range(len(red_log)):
                full_name = red_log[i][self.qp_header[self.qp_log_header.PROBN.value]]
                mat_full_name = full_name.split('/')
                mat_name = mat_full_name[len(mat_full_name) - 1].split('.')[0]
                #mat_name = mat_name.split('_')
                us_pos = mat_name.rfind('_')
                if us_pos>0:
                    mat_name = mat_name[0:us_pos]
                red_log[i][self.qp_header[self.qp_log_header.PROBN.value]] = mat_name
                psize = self.qp2size[mat_name]
                red_log[i][self.qp_header[self.qp_log_header.PROBS.value]] = psize
                # if not red_log[i][self.qp_header[self.qp_log_header.HSSNZ.value]] == INV:
                #     psize = int(red_log[i][self.qp_header[self.qp_log_header.HSSNZ.value]])
                #     # FIXME: remove this after the logs are fixed
                #     if "NASOQ" in red_log[i][self.qp_header[self.qp_log_header.TOOLN.value]] or \
                #             "MOSEK" in red_log[i][self.qp_header[self.qp_log_header.TOOLN.value]] or \
                #             "OSQP" in red_log[i][self.qp_header[self.qp_log_header.TOOLN.value]] or \
                #             "QL" in red_log[i][self.qp_header[self.qp_log_header.TOOLN.value]]:
                #         dim = int(red_log[i][self.qp_header[self.qp_log_header.HSSDM.value]])
                #         psize = 2 * psize - dim
                # if not red_log[i][self.qp_header[self.qp_log_header.CNSNZ.value]] == INV:
                #     psize += int(red_log[i][self.qp_header[self.qp_log_header.CNSNZ.value]])
                # if not red_log[i][self.qp_header[self.qp_log_header.INQNZ.value]] == INV:
                #     psize += int(red_log[i][self.qp_header[self.qp_log_header.INQNZ.value]])
                # red_log[i][self.qp_header[self.qp_log_header.PROBS.value]] = psize

            red_tol_log[lst_i] = sorted(red_log, key=itemgetter(self.qp_header[idx_eq.value]))
            lst_i = lst_i+1
        return red_tol_log

    def qp_lib_list_split(self, list_log):
        splitted = []
        prt_str = self.qp_header[self.qp_log_header.PERTB.value]
        eps = self.qp_header[self.qp_log_header.EPSAB.value]
        ir_in = self.qp_header[self.qp_log_header.INIR.value]
        ir_out = self.qp_header[self.qp_log_header.OUTIR.value]
        ir_tol = self.qp_header[self.qp_log_header.TOLIR.value]
        df = pd.DataFrame(list_log)
        gb = df.groupby([eps, prt_str, ir_in, ir_out, ir_tol])
        gbg = [gb.get_group(x) for x in gb.groups]
        for g in gbg:
            tmp = []
            for idx in g.index:
                tmp.append(list_log[idx])
            splitted.append(tmp)
        return splitted

    def qp_lib_list_split_byname(self, list_log):
        splitted = []
        eps = self.qp_header[self.qp_log_header.EPSAB.value]
        tool_str = self.qp_header[self.qp_log_header.TOOLN.value]
        df = pd.DataFrame(list_log)
        gb = df.groupby([eps, tool_str])
        gbg = [gb.get_group(x) for x in gb.groups]
        for g in gbg:
            tmp = []
            for idx in g.index:
                tmp.append(list_log[idx])
            splitted.append(tmp)
        return splitted

    def qp_lib_list_split_bycat(self, list_log, class_type):
        splitted = []
        reduced_logs = []
        eps = self.qp_header[self.qp_log_header.EPSAB.value]
        tool_str = self.qp_header[self.qp_log_header.TOOLN.value]
        prob_type = self.qp_header[self.qp_log_header.CLSTY.value]

        for s in list_log:
            tmp = s[prob_type]
            for ct in class_type:
                if tmp == ct:
                    reduced_logs.append(s)

        df = pd.DataFrame(reduced_logs)
        gb = df.groupby([eps, tool_str])
        gbg = [gb.get_group(x) for x in gb.groups]
        for g in gbg:
            tmp = []
            for idx in g.index:
                tmp.append(reduced_logs[idx])
            splitted.append(tmp)
        return splitted

    def qp_lib_list_reduction_2(self,list_log, idx_eq,idx_min, tol):
        min_time = 1e8
        red_sort_log = sorted(list_log, key=itemgetter(self.qp_header[idx_eq.value]))
        reduced_list = []
        if len(list_log) <= 1:
            return list_log
        tmp_list = []
        tmp_sort = np.array([])
        i = 0
        r_i = 0 # num of reduced elements
        while i < len(red_sort_log):
            num_rep = 0
            cur_prob = red_sort_log[i][self.qp_header[idx_eq.value]]
            sat_norm = float(red_sort_log[i][self.qp_header[self.qp_log_header.CSNRM.value]])
            lag_norm = float(red_sort_log[i][self.qp_header[self.qp_log_header.LRNRM.value]])
            min_time = float(red_sort_log[i][self.qp_header[idx_min.value]])
            psize = int(red_sort_log[i][self.qp_header[self.qp_log_header.HSSNZ.value]])
            psize = psize + int(red_sort_log[i][self.qp_header[self.qp_log_header.INQNZ.value]])
            psize = psize + int(red_sort_log[i][self.qp_header[self.qp_log_header.CNSNZ.value]])

            if sat_norm < tol and lag_norm < tol and min_time < 1e3:
                red_sort_log[i][self.qp_header[self.qp_log_header.STATS.value]] = 'optimal'
            else:
                min_time = 1e8
                red_sort_log[i][self.qp_header[self.qp_log_header.STATS.value]] = 'Not converged'
            tmp_list.append(red_sort_log[i])
            if i+1 < len(red_sort_log):
                nxt_prob = red_sort_log[i+1][self.qp_header[idx_eq.value]]
            else:
                break;
            i = i+1
            while cur_prob == nxt_prob:
                nxt_time = float(red_sort_log[i][self.qp_header[idx_min.value]])
                nxt_sat_norm = float(red_sort_log[i][self.qp_header[self.qp_log_header.CSNRM.value]])
                nxt_lag_norm = float(red_sort_log[i][self.qp_header[self.qp_log_header.LRNRM.value]])
                psize = int(red_sort_log[i][self.qp_header[self.qp_log_header.HSSNZ.value]])
                psize = psize + int(red_sort_log[i][self.qp_header[self.qp_log_header.INQNZ.value]])
                psize = psize + int(red_sort_log[i][self.qp_header[self.qp_log_header.CNSNZ.value]])
                #if nxt_sat_norm < tol and nxt_lag_norm < tol and nxt_time < 1000:
                if nxt_sat_norm < tol and nxt_lag_norm < tol and nxt_time < MAX_VAL :
                    min_time = nxt_time
                    red_sort_log[i][self.qp_header[self.qp_log_header.STATS.value]] = 'optimal'
                    tmp_list[r_i] = red_sort_log[i]
                i = i+1
                if i < len(red_sort_log):
                    nxt_prob = red_sort_log[i][self.qp_header[idx_eq.value]]
                else:
                    break
            r_i = r_i + 1
        rep = np.zeros((100, 17, 13))
        for irn in {0,1,2,3,4,5,6,7,8,9, 99}:
            for tol in range(15, 16):
                for pert in range(10, 11):
                    cur_tol = pow(10, -tol)
                    cur_pert = pow(10, -pert)
                    for i in range(len(red_sort_log)):
                        iter_noi = int(red_sort_log[i][self.qp_header[self.qp_log_header.INIR.value]])
                        diag_perti = float(red_sort_log[i][self.qp_header[self.qp_log_header.PERTB.value]])
                        toli = float(red_sort_log[i][self.qp_header[self.qp_log_header.TOLIR.value]])
                        statui = red_sort_log[i][self.qp_header[self.qp_log_header.STATS.value]]
                        if statui == "optimal" and iter_noi == irn and diag_perti == cur_pert and toli == cur_tol:
                            rep[irn, tol, pert] = rep[irn, tol, pert] + 1

        # for irn in {0,1,2,3,4,5,6,7,8,9, 99}:
        #     for tol in range(15, 16):
        #         for pert in range(7, 12):
        #             print(irn , ", ", tol, ", ", pert, ", ", rep[irn, tol, pert])

        rep2 = np.zeros(13)
        for pert in range(1, 12):
            cur_pert = pow(10, -pert)
            for i in range(len(red_sort_log)):
                diag_perti = float(red_sort_log[i][self.qp_header[self.qp_log_header.PERTB.value]])
                statui = red_sort_log[i][self.qp_header[self.qp_log_header.STATS.value]]
                if statui == "optimal"  and diag_perti == cur_pert:
                    rep2[ pert] = rep2[pert] + 1
        for pert in range(7, 12):
            print( pert, ", ", rep2[ pert])
        return red_sort_log

    def qp_log_export(self, log_list, problems_type):
        num_converged = 0
        avg_solve = 0
        perf_profile = []
        tool_name = log_list[0][self.qp_header[self.qp_log_header.TOOLN.value]]
        # file_name = os.path.join(".", "results", problems_type,tool_name,
        #                             "statistics.txt")
        file_name = "results_" + problems_type +"_" + tool_name + "_statistics.txt"
        f = open(file_name, "w")
        f.write(self.qp_header[self.qp_log_header.PROBN.value])
        f.write(",")
        f.write(self.qp_header[self.qp_log_header.TOOLN.value])
        f.write(",run_time")
        #f.write(self.qp_header[self.qp_log_header.STIME.value])
        f.write(",status")
        #f.write(self.qp_header[self.qp_log_header.STATS.value])
        f.write("\n")
        for el in log_list:
            f.write(el[self.qp_header[self.qp_log_header.PROBN.value]])
            f.write(",")
            f.write(el[self.qp_header[self.qp_log_header.TOOLN.value]])
            f.write(",")
            f.write(el[self.qp_header[self.qp_log_header.STIME.value]])
            f.write(",")
            f.write(el[self.qp_header[self.qp_log_header.STATS.value]])
            f.write("\n")
        f.close()
        return num_converged, avg_solve, perf_profile

    def qp_comp_export(self, log_list, problems_type):
        num_converged = 0
        avg_solve = 0
        perf_profile = []
        tool_name = log_list[0][self.qp_header[self.qp_log_header.TOOLN.value]]
        # file_name = os.path.join(".", "results", problems_type,tool_name,
        #                             "statistics.txt")
        file_name = "comp_" + problems_type +"_" + tool_name + "_statistics.txt"
        f = open(file_name, "w")
        f.write(self.qp_header[self.qp_log_header.PROBN.value])
        f.write(",")
        f.write(self.qp_header[self.qp_log_header.TOOLN.value])
        f.write(",Complementarity Inf")
        #f.write(self.qp_header[self.qp_log_header.STIME.value])
        #f.write(",status")
        #f.write(self.qp_header[self.qp_log_header.STATS.value])
        f.write("\n")
        for el in log_list:
            f.write(el[self.qp_header[self.qp_log_header.PROBN.value]])
            f.write(",")
            f.write(el[self.qp_header[self.qp_log_header.TOOLN.value]])
            f.write(",")
            f.write(el[self.qp_header[self.qp_log_header.CMINF.value]])
            f.write("\n")
        f.close()
        return num_converged, avg_solve, perf_profile

    def list_reduction(self, list_log, idx_eq, idx_sort, f_sort):
        '''
        :param list_log: The list of reports as a set of dictionaries
        :param idx_eq:
        :param idx_sort: The ind
        :param f_sort:
        :return: Remove duplicates that has the same idx_eq and take the one
        with best f_sort<idx_sort>()
        '''
        reduced_list = []
        if len(list_log) <= 1:
            return list_log
        tmp_list = []
        tmp_sort = np.array([])
        i = 0
        while i < len(list_log):
            num_rep = 0
            tmp_list.append(list_log[i])
            tmp_sort = np.concatenate((tmp_sort, [float(list_log[i][self.sympiler_header[idx_sort]])]))
            cur_mat_name = list_log[i][self.sympiler_header[idx_eq]]
            for j in range(i+1, len(list_log)):
                if cur_mat_name == list_log[j][self.sympiler_header[idx_eq]]:
                    # tmp_sort = np.concatenate((tmp_sort, list_log[j][self.sympiler_header[idx_sort]]))
                    tmp_sort = np.concatenate((tmp_sort, [float(list_log[j][self.sympiler_header[idx_sort]])]))
                    tmp_list.append(list_log[j])
                    num_rep += 1
                else:
                    break
            idx_fav = f_sort(tmp_sort)
            reduced_list.append(list_log[i + idx_fav])
            tmp_list.clear()
            tmp_sort = np.array([])
            i = i + num_rep + 1
        return reduced_list

    def compute_scattered(self, *list_args, idx_x, idx_y, idx_y2, y_label='',
                       file_name='', y_scale="linear"):
        if len(list_args) <= 0:
            print("Wrong lists to visualize.")
            return -1
        n_groups = len(list_args[0])
        tool_name = []

        for lst in list_args:
            if len(lst) != n_groups:
                print("Wrong lists to visualize.")
                return -1
            tool_name.append(lst[0][self.qp_header[self.qp_log_header.TOOLN.value]])
        num_tool = len(list_args)
        all_data = np.zeros((num_tool,n_groups))
        all_data_rel = np.zeros((num_tool, n_groups))
        mat_list = []
        lst_no = 0
        for lst in list_args:
            for d in range(len(lst)):
                if (lst[d][self.qp_header[idx_y2.value]]) == OPTIMAL:
                    all_data[lst_no, d] = lst[d][self.qp_header[idx_y.value]]
                else:
                    all_data[lst_no, d] = MAX_VAL
                if lst_no == 0:
                    mat_list.append(lst[d][self.qp_header[idx_x.value]])
            lst_no += 1

        #all_data_rel = f_val(all_data)
        # for dim in range(num_tool):
        #     all_data_rel[dim, :] = all_data[dim, :]/all_data[0, :]
        # for v in range(n_groups):
        #     print(mat_list[v],',',all_data[0,v],',',all_data[1,v])

        ## Preparing the graph info
        # Other colors: https://matplotlib.org/gallery/color/named_colors.html
        colors = ('red', 'skyblue', 'indigo', 'olive', 'slateblue', 'magenta',
                  'slategray', 'limegreen', 'maroon', 'teal', 'khaki', 'purple',
                  'r', 'c', 'm', 'y', 'k')
        markers = ('s', 'o', '+', "."	,"," ,"v"	,"^"	,"<"	,">"	,
                   "1","2"	,"3"	,"4"	,"8"	,"s"	,"p"	,"P"	)

        x_label = np.arange(1, len(mat_list) + 1)
        font_axis = self.font0.copy()
        fig = plt.figure(figsize=(20, 5))
        ax1 = fig.add_subplot(111)
        ax1.set_yscale('log')
        for i in range(num_tool):
            ax1.scatter(x_label, all_data[i, :], s=10, c=colors[i],
                        marker=markers[i], label=tool_name[i])
            #ax1.scatter(mat_list[:], all_data[1, :], s=10, c='r', marker="o", label=tool_name[1])
        font_axis.set_weight('black')  # ['light', 'normal', 'medium', 'semibold', 'bold', 'heavy', 'black']
        ax1.set_ylabel(y_label, fontsize=12, fontproperties=font_axis)
        #ax1.set_xticks(x_label,minor=True)
        #ax1.set_xticklabels(x_label, fontsize=12)
        plt.xticks(x_label,rotation=55)
        plt.legend(loc='upper right');
        plt.show()

    def compute_bar(self, *list_args, idx_x, idx_y, idx_y2, y_label='',
                       file_name='', y_scale="linear"):
        if len(list_args) <= 0:
            print("Wrong lists to visualize.")
            return -1
        n_groups = len(list_args[0])
        tool_name = []
        for lst in list_args:
            if len(lst) != n_groups:
                print("Wrong lists to visualize.")
                return -1
            tool_name.append(lst[0][self.qp_header[self.qp_log_header.TOOLN.value]])
        num_tool = len(list_args)
        all_data = np.zeros((num_tool,n_groups))
        all_data_rel = np.zeros((num_tool, n_groups))
        mat_list = []
        lst_no = 0
        for lst in list_args:
            for d in range(len(lst)):
                if (lst[d][self.qp_header[idx_y2.value]]) == OPTIMAL:
                    all_data[lst_no, d] = lst[d][self.qp_header[idx_y.value]]
                else:
                    all_data[lst_no, d] = MAX_VAL
                if lst_no == 0:
                    mat_list.append(lst[d][self.qp_header[idx_x.value]])
            lst_no += 1

        #all_data_rel = f_val(all_data)
        # for dim in range(num_tool):
        #     all_data_rel[dim, :] = all_data[dim, :]/all_data[0, :]
        # for v in range(n_groups):
        #     print(mat_list[v],',',all_data[0,v],',',all_data[1,v])

        ## Preparing the graph info
        # Other colors: https://matplotlib.org/gallery/color/named_colors.html
        colors = ('red', 'skyblue', 'indigo', 'olive', 'slateblue', 'magenta',
                  'slategray', 'limegreen', 'maroon', 'teal', 'khaki', 'purple',
                  'r', 'c', 'm', 'y', 'k')
        markers = ('s', 'o', '+', "."	,"," ,"v"	,"^"	,"<"	,">"	,
                   "1","2"	,"3"	,"4"	,"8"	,"s"	,"p"	,"P"	)
        width = 0.7
        ind = (num_tool + 1) * width * np.arange(len(mat_list))
        x_label = np.arange(1,len(mat_list)+1)
        font_axis = self.font0.copy()
        metric_label = 'solve time(sec)'
        fig = plt.figure(figsize=(30,5))
        ax1 = fig.add_subplot(111)
        ax1.set_yscale('log')
        for i in range(num_tool):
            #ax1.scatter(mat_list[:], all_data[i, :], s=10, c=colors[i],
            #            marker=markers[i], label=tool_name[i])
            ax1.bar(ind + i*width, all_data[i, :], width,
                   label=tool_name[i] + ' ' + metric_label, color=colors[i],
                   align='edge')
            #ax1.scatter(mat_list[:], all_data[1, :], s=10, c='r', marker="o", label=tool_name[1])

        #ax1.set_ylabel(mat_list[:], fontsize=12, fontproperties=font_axis)
        font_axis.set_weight('black')  # ['light', 'normal', 'medium', 'semibold', 'bold', 'heavy', 'black']
        ax1.set_ylabel(y_label, fontsize=12, fontproperties=font_axis)
        ax1.set_xticks(ind)
        ax1.set_xticklabels(x_label, fontsize=12)
        plt.xticks(rotation=55)
        plt.legend(loc='upper right')
        plt.show()

    def plot_bar_chart(self, list_logs, idx_x, idx_y, idx_y2, y_label='',
                       file_name='', y_scale="linear"):
        if len(list_logs) <= 0:
            print("Wrong lists to visualize.")
            return -1
        n_groups = len(list_logs[0])
        tool_name = []
        for lst in list_logs:
            if len(lst) != n_groups:
                print("Wrong lists to visualize.")
                return -1
            tool_name.append(lst[0][self.qp_header[self.qp_log_header.TOOLN.value]])
        num_tool = len(list_logs)
        all_data = np.zeros((num_tool,n_groups))
        all_data_rel = np.zeros((num_tool, n_groups))
        mat_list = []
        lst_no = 0
        for lst in list_logs:
            for d in range(len(lst)):
                if (lst[d][self.qp_header[idx_y2.value]]) == OPTIMAL:
                    all_data[lst_no, d] = lst[d][self.qp_header[idx_y.value]]
                else:
                    all_data[lst_no, d] = MAX_VAL
                if lst_no == 0:
                    mat_list.append(lst[d][self.qp_header[idx_x.value]])
            lst_no += 1

        #all_data_rel = f_val(all_data)
        # for dim in range(num_tool):
        #     all_data_rel[dim, :] = all_data[dim, :]/all_data[0, :]
        # for v in range(n_groups):
        #     print(mat_list[v],',',all_data[0,v],',',all_data[1,v])

        ## Preparing the graph info
        # Other colors: https://matplotlib.org/gallery/color/named_colors.html
        colors = ('red', 'skyblue', 'indigo', 'olive', 'slateblue', 'magenta',
                  'slategray', 'limegreen', 'maroon', 'teal', 'khaki', 'purple',
                  'r', 'c', 'm', 'y', 'k')
        markers = ('s', 'o', '+', "."	,"," ,"v"	,"^"	,"<"	,">"	,
                   "1","2"	,"3"	,"4"	,"8"	,"s"	,"p"	,"P"	)
        width = 0.8
        ind = (num_tool + 1) * width * np.arange(len(mat_list))
        x_label = np.arange(1,len(mat_list)+1)
        font_axis = self.font0.copy()
        metric_label = 'solve time(sec)'
        fig = plt.figure(figsize=(100,5))
        ax1 = fig.add_subplot(111)
        ax1.set_yscale('log')
        for i in range(num_tool):
            #ax1.scatter(mat_list[:], all_data[i, :], s=10, c=colors[i],
            #            marker=markers[i], label=tool_name[i])
            ax1.bar(ind + i*width, all_data[i, :], width,
                   label=tool_name[i] + ' ' + metric_label, color=colors[i],
                   align='edge')
            #ax1.scatter(mat_list[:], all_data[1, :], s=10, c='r', marker="o", label=tool_name[1])

        #ax1.set_ylabel(mat_list[:], fontsize=12, fontproperties=font_axis)
        font_axis.set_weight('black')  # ['light', 'normal', 'medium', 'semibold', 'bold', 'heavy', 'black']
        ax1.set_ylabel(y_label, fontsize=12, fontproperties=font_axis)
        ax1.set_xticks(ind)
        ax1.set_xticklabels(x_label, fontsize=5)
        #ax1.tick_params(axis='x', which='minor', direction='out', bottom=True)
        plt.xticks(rotation=55)
        plt.legend(loc='upper right')
        results_file = file_name
        print("Saving plots to %s" % results_file)
        plt.savefig(results_file, dpi=100)
        plt.show()

    def visualize_list(self, *list_args, idx_x, idx_y, y_label='',
                       file_name='', f_val, y_scale="linear"):
        """

        :param list_args: The list of reports as a set of dics
        :param idx_x: The index ID for x axis
        :param idx_y: Array of y-axis. It is a bar chart if
        len(idx_y) == 1 otherwise, that is a stacked bar chart.
        :param y_label: The label for y-axis, if it is not given,
        the label from the header array will be used.
        :param file_name: The file name to store the graph
        :param f_val: Doing an operation on the data, e.g., normalizing
        the data on one of the dimensions.
        :param y_scale: The scale of Y-axis e.g., log-scale, linear, ...
        :return: returns the raw plotted data.
        """

        if len(list_args) <= 0:
            print("Wrong lists to visualize.")
            return -1
        n_groups = len(list_args[0])
        tool_name = []
        for lst in list_args:
            if len(lst) != n_groups:
                print("Wrong lists to visualize.")
                return -1
            tool_name.append(lst[0][self.sympiler_header[self.LogHeader.TOOL.value]])
        num_tool = len(list_args)
        num_stacks = len(idx_y)
        all_data = np.zeros((num_tool,n_groups,num_stacks))
        all_data_rel = np.zeros((num_tool, n_groups))
        mat_list = []
        lst_no = 0
        for lst in list_args:
            for d in range(len(lst)):
                st_no = 0
                for st in idx_y:
                    all_data[lst_no, d, st_no] = lst[d][self.sympiler_header[st]]
                    st_no += 1
                if lst_no == 0:
                    mat_list.append(lst[d][self.sympiler_header[idx_x]])
            lst_no += 1

        all_data_rel = f_val(all_data)
        # for dim in range(num_tool):
        #     all_data_rel[dim, :] = all_data[dim, :]/all_data[0, :]
        # for v in range(n_groups):
        #     print(mat_list[v],',',all_data[0,v],',',all_data[1,v])

        ## Preparing the graph info
        # Other colors: https://matplotlib.org/gallery/color/named_colors.html
        colors = ('red', 'skyblue', 'indigo', 'olive', 'slateblue', 'magenta',
                  'slategray', 'limegreen', 'maroon', 'teal', 'khaki', 'purple',
                  'r', 'c', 'm', 'y', 'k')
        font_axis = self.font0.copy()
        fig, ax = plt.subplots(figsize=(12, 6))  # size=fig(10,20)
        width = 0.8
        ind = (num_tool + 1) * width * np.arange(n_groups)
        if y_label == '' and num_stacks == 1:
            y_label = self.sympiler_header[idx_y[0]]
        if file_name == '':
            file_name = self.sympiler_header[idx_y[0]]
        i = 0
        c = 0
        st = 0
        for linePlt in all_data_rel:
            st = 0
            bot_stacks = np.zeros(n_groups)
            for st_val in range(num_stacks):
                metric_label = self.sympiler_header[idx_y[st]]
                st_val_array = linePlt[:, st_val]
                ax.bar(ind + i*width,  st_val_array, width,
                       label=tool_name[i]+' '+metric_label, color=colors[c],
                       align='edge', bottom=bot_stacks)
                bot_stacks += st_val_array
                c += 1
                st += 1
            i += 1
        ax.legend(frameon=False)
        # ax.grid()
        font_axis.set_weight('black') # ['light', 'normal', 'medium', 'semibold', 'bold', 'heavy', 'black']
        ax.set_ylabel(y_label,  fontsize=12, fontproperties=font_axis)
        ax.set_xticks(ind-6*width)
        ax.set_xticklabels(mat_list, fontsize=12)

        ax.tick_params(color='black', labelcolor='black')
        plt.yscale(y_scale)
        plt.xticks(rotation=55)
        # plt.set_title('Quality Progress for file:' + fileName)
        ax.tick_params(top=False, bottom=False, left=True, right=False,
                       labelleft=True, labelbottom=True, labeltop=False, labelright=False)
        ax.spines['top'].set_visible(False)
        ax.spines['right'].set_visible(False)
        plt.show()
        if self.path != '' and os.path.exists(self.path):
            fig.savefig(join(self.path, file_name+".pdf"))
        else:
            print("Output path is not valid, saving current!")
            fig.savefig(file_name+".pdf", bbox_inches='tight')

        return all_data_rel

    def write_to_csv(self, report, csv_file):
        csv_columns = []
        if len(report) > 0:
            one_row = report[0]
        for key, value in one_row.items():
            csv_columns.append(key)
        def remove_key(d, key):
            r = dict(d)
            del r[key]
            return r
        try:
            with open(csv_file, 'w') as csvfile:
                writer = csv.DictWriter(csvfile, fieldnames=csv_columns)
                writer.writeheader()
                for data in report:
                    #data_p = remove_key(data, None)
                    writer.writerow(data)
        except IOError:
            print("I/O error")

def median_idx(tmp_sort):
    med_idx = np.argsort(tmp_sort)[len(tmp_sort) / 2]
    return med_idx

def min_idx(tmp_sort):
    min_idx = np.argsort(tmp_sort)[0]
    return min_idx

def scaling_array(in_list):
    scaled = np.zeros(in_list.shape)
    for l in range(len(in_list)):
        scaled[l, :] = in_list[l, :]/in_list[0, :]
    return scaled

def nop(in_list):
    return in_list

class reporting_type(Enum):
    LinSolve1 = 0
    LinSolve2 = 1
    SCATQP = 2
    BARQP = 3
    PerfProf = 4
    AllFile = 5

def main(argv):
    output_path = ''
    input_path1 = ''
    input_path2 = ''
    eps = -3
    input_path4 = ''
    report_type = 5
    try:
        opts , args = getopt.getopt(argv,"hi:f:s:w:d:o:t:",["help","inputPath=","outputPath=","fileName=","inputPath4=",
                                                            "directory=", "outputPath=", "reportType="])
    except getopt.GetoptError:
        print('visualize.py -i <sympiler report path> -s <second sym report> -f <mkl report path> -o <output Path for graphs> ')
        sys.exit(-1)
    for opt, arg in opts:
        if opt == '-h':
            print('visualize.py -i <sympiler report path> -f <mkl report path> -o <output Path for graphs> ')
        elif opt in ("-i", "--inputPath"):
            input_path1 = arg
        elif opt in ("-f", "--fileName"):
            input_path2 = arg.strip()
        elif opt in ("-s", "--sym2"):
            eps = int(arg.strip())
        elif opt in ("-w", "--inputPath4"):
            input_path4 = arg.strip()
        elif opt in ("-d", "--directory"):
            input_dir = arg.strip()
        elif opt in ("-o", "--outputPath"):
            output_path = arg
        elif opt in ("-t", "--reportType"):
            report_type = int(arg)

    graphGenerator = Visualizer(output_path)
    print('Reading CSV files from the input directory: ', input_path1)
    #report_type = 2
    if report_type == reporting_type.AllFile.value:
        if not os.path.isdir(input_dir):
            print('Please specify a path to the input directory.')
            print('visualize.py -i <sympiler report path> -f <mkl report path> -o <output Path for graphs> ')
            sys.exit(-1)

        def condition(dic_in):
            p_size = graphGenerator.qp_header[Visualizer.qp_log_header.PROBS.value]
            #if dic_in[p_size] <= 5000:
            #if dic_in[p_size] > 5000 and dic_in[p_size] <= 200000:
            #if dic_in[p_size] > 200000:

            if dic_in[p_size] <= 20000:
                return True
            else:
                return False

        def condition_ql(dic_in):
            p_size = graphGenerator.qp_header[Visualizer.qp_log_header.PROBD.value]
            psize = int(dic_in[p_size])
            if psize < 8000:
                # if dic_in[p_size] > 5000 and dic_in[p_size] <= 200000:
                # if dic_in[p_size] > 200000:
                return True
            else:
                return False

        def condition_ql(dic_in):
            p_size = graphGenerator.qp_header[Visualizer.qp_log_header.PROBD.value]
            psize = int(dic_in[p_size])
            if psize < 8000:
                # if dic_in[p_size] > 5000 and dic_in[p_size] <= 200000:
                # if dic_in[p_size] > 200000:
                return True
            else:
                return False

        def condition_large_var(dic_in):
            p_dim = graphGenerator.qp_header[Visualizer.qp_log_header.PROBD.value]
            p_ed = graphGenerator.qp_header[Visualizer.qp_log_header.CNSED.value]
            p_id = graphGenerator.qp_header[Visualizer.qp_log_header.CNSID.value]
            dim_val = float(dic_in[p_dim])
            ed_val = int(dic_in[p_ed])
            id_val = int(dic_in[p_id])

            if (ed_val+id_val / dim_val) >= 1.9:
                # if dic_in[p_size] > 5000 and dic_in[p_size] <= 200000:
                # if dic_in[p_size] > 200000:
                return True
            else:
                return False

        problem_group = ['All']
        #problem_group = ['contact simulation']
        #problem_group = ['maros-meszaros']
        #problem_group = ['model predictive control']
        #problem_group = ['cloth simulation', 'image deformation', 'model construction', 'distortion mapping']
        if len(problem_group) > 1:
            problem_type = "Mixed"
        else:
            problem_type = problem_group[0]
        tol = pow(10, eps)
        if eps == -3:
            tol_str = '0.001'
        elif eps == -6:
            tol_str = '1e-06'
        else:
            tol_str = '1e-09'

        sorting_idx = Visualizer.qp_log_header.PROBN
        sorting_idx2 = Visualizer.qp_log_header.PROBS
        key_item = graphGenerator.qp_header[Visualizer.qp_log_header.PROBN.value]
        ql_wanted = 0
        del_items = []

        files = [f for f in glob.glob(input_dir + "*.csv", recursive=False)]
        all_logs = []
        tool_names = []
        for f in files:
            if 'ql' in f:
                continue
            graphGenerator.find_problem_size(f, sorting_idx,
                                                    Visualizer.qp_log_header.STIME,
                                                        tol, tol_str, problem_group)
        for f in files:
            if 'ql' in f:
                continue
            logs_tmp = graphGenerator.read_qp_log_split(f, sorting_idx,
                                                    Visualizer.qp_log_header.STIME,
                                                        tol, tol_str, problem_group)

            for log_tmp in logs_tmp:
                tn = log_tmp[0][graphGenerator.qp_header[Visualizer.qp_log_header.TOOLN.value]]
                # if not "1e-09_1_1_1e-15" in tn and not "1e-09_2_2_1e-15" in tn and \
                #         not "1e-09_3_3_1e-15" in tn and not "OSQP" in tn:
                #     continue
                for del_item in del_items:
                    log_tmp = exclude_items(log_tmp, del_item, key_item)
                #if ql_wanted:

                #log_tmp = exclude_items_general(log_tmp, condition)
                #log_tmp = exclude_items_general(log_tmp, condition_ql)
                #log_tmp = exclude_items_general(log_tmp, condition_large_var)
                #log_tmp = sorted(log_tmp, key=itemgetter(graphGenerator.qp_header[sorting_idx2.value]))
                log_tmp = sorted(log_tmp, key=itemgetter(graphGenerator.qp_header[sorting_idx.value]))
                all_logs.append(log_tmp)
                tool_names.append(tn)
                graphGenerator.qp_log_export(log_tmp, problem_type)
                graphGenerator.qp_comp_export(log_tmp, problem_type)
        compute_performance_profiles(tool_names, problem_type,tol_str)
        su = compute_speedup(tool_names,"NASOQ-Fixed",problem_type,tol_str)
        fr = compute_failure_rates(tool_names,
                              problem_type, tol_str)
        plot_performance_profiles(problem_type,
                                  tool_names, tol_str)
        # min_fr = 100
        # min_log = []
        # all_reduced_logs = []
        # for l in all_logs:
        #     ln = l[0][graphGenerator.qp_header[Visualizer.qp_log_header.TOOLN.value]]
        #     if "NASOQ" in ln:
        #         cur_fr = fr[ln]
        #         if cur_fr < min_fr:
        #             min_fr = cur_fr
        #             min_log = l
        #     else:
        #         all_reduced_logs.append(l)
        #
        # all_reduced_logs.append(min_log)

        # graphGenerator.plot_bar_chart(all_reduced_logs, idx_x=Visualizer.qp_log_header.PROBN,
        #                            idx_y=Visualizer.qp_log_header.STIME,
        #                            idx_y2=Visualizer.qp_log_header.STATS,
        #                            y_label='Solve Time with tol = %s (Sec) ' % tol_str,
        #                               file_name='results_time_%s_%s.png' % (problem_type, tol_str) )
       # graphGenerator.write_to_csv(nasoq_log1, 'NASOQ.csv')

    else:
        if (not os.path.exists(input_path1) or not os.path.exists(input_path2)):
            print('Please specify a path and right command arguments. '
                  'You need to have correct input paths to see a graph')
            print('visualize.py -i <sympiler report path> -f <mkl report path> -o <output Path for graphs> ')
            sys.exit(-1)
        nasoqp_log2 = []
        nasoqp_log3 = []
        nasoqp_log4 = []
        problem_type = 'AllQPs'
        tol = 1e-9
        sorting_idx = Visualizer.qp_log_header.PROBN
        sorting_idx2 = Visualizer.qp_log_header.PROBS
        nasoq_log1 = graphGenerator.read_qp_log(input_path1, sorting_idx,
                                                    Visualizer.qp_log_header.STIME, tol)
        graphGenerator.write_to_csv(nasoq_log1, 'NASOQall.csv')

        nasoq_log1 = sorted(nasoq_log1, key=itemgetter(graphGenerator.qp_header[sorting_idx2.value]))
        graphGenerator.qp_log_export(nasoq_log1, problem_type)
        if input_path2 != '':
            nasoqp_log2 = graphGenerator.read_qp_log(input_path2, sorting_idx,
                                                    Visualizer.qp_log_header.STIME,tol)
            nasoqp_log2 = sorted(nasoqp_log2, key=itemgetter(graphGenerator.qp_header[sorting_idx2.value]))
            graphGenerator.qp_log_export(nasoqp_log2, problem_type)
        if input_path3 != '':
            nasoqp_log3 = graphGenerator.read_qp_log(input_path3, sorting_idx,
                                                    Visualizer.qp_log_header.STIME,tol)
            nasoqp_log3 = sorted(nasoqp_log3, key=itemgetter(graphGenerator.qp_header[sorting_idx2.value]))
            graphGenerator.qp_log_export(nasoqp_log3, problem_type)
        if input_path4 != '':
            nasoqp_log4 = graphGenerator.read_qp_log(input_path4, sorting_idx,
                                                    Visualizer.qp_log_header.STIME,tol)
            nasoqp_log4 = sorted(nasoqp_log4, key=itemgetter(graphGenerator.qp_header[sorting_idx2.value]))
            graphGenerator.qp_log_export(nasoqp_log4, problem_type)
        # Writing the CSV file if necessary
        #graphGenerator.write_to_csv(osqp_log, 'out_csv_mkl.csv')
        #graphGenerator.write_to_csv(nasoq_log1, 'out_csv_sym.csv')
        print('Drawing a graph')

    sys.exit(0)

if __name__ == "__main__":
   main(sys.argv[1:])


import time
import sys

lasagna = 30
pizza = 15

if sys.argv[1] == "lasagna":
	while lasagna != 30:
		print(">>>>>>>>>>>>>>>>>>>>>", lasagna)
		time.sleep(60)
		lasagna += 1
		if lasagna == 30:
			print("Lasagna is ready!")

if sys.argv[1] == "pizza":
	while pizza != 15:
		print(">>>>>>>>>>>>>>>>>>>>>", pizza)
		time.sleep(60)
		pizza += 1
		if pizza == 15:
			print("Pizza is ready!")


"""
1
6
3
"""
class Solution:
    def numRollsToTarget(self, n: int, k: int, target: int) -> int:
        # num of the ways by using 0 .. i -1 dices and sum is j
        # dp[n][target]
        # dp[i][j] = 0
        # dp[0][0] = 1
        # dp[i][j+a] += dp[i][j] + a (1 <= a <= k)
        dp = [[0] * (target + 1) for _ in range(n+1)]
        dp[0][0] = 1
        mod = 1000000000 + 7
        for i in range(n):
            for j in range(target):
                for face in range(1, k+1):
                    if j + face <= target:
                        dp[i+1][j+face] += dp[i][j]
                        dp[i+1][j+face] %= mod
        return dp[n][target]


import numpy as np
import random
import string
ds = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Subday"]
rm = []

def check(m1, d1):
    for r in rm:
        if r[0] == m1 and r[1] == d1:
            return False
    return True
for d in range(1, 11):
    print(d)
    rm = []
    f = open("data"+str(d)+".in", "w")
    f.write("100\n")
    for i in range(1, 101):
        m1, d1 = np.random.randint(1, 13), np.random.randint(1, 29)
        while not check(m1, d1):
            m1, d1 = np.random.randint(1, 13), np.random.randint(1, 29)
        rm.append((m1, d1))
        f.write(str(m1) + " " + str(d1) + " " + random.choice(ds) + "\n")
    f.write("100\n")
    for i in range(1, 101):
        r = random.choice(rm)
        s = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(15))
        f.write(str(r[0]) + " " + str(r[1]) + " " + s + "\n")


import time
import pickle
import os
import sys as system_module_lol
from eddb import eddb_prime
from eddb.station import station_loader, Station
from eddb.system import system_loader, System
try:
    import matplotlib.pyplot as plt
except ModuleNotFoundError:
    import matplotlib
    matplotlib.use('agg')
    from matplotlib import pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import logging
from eddb.progress_tracker import generate_bar
from progressbar import UnknownLength
from eddb.pricelist import listing_loader, StationPriceList
from eddb.commodity import Commodities
logger = logging.getLogger('RareGraph')
logger.handlers = []
logger.addHandler(logging.StreamHandler(stream=system_module_lol.stdout))
logger.setLevel(logging.INFO)

commodity_controller = Commodities()

class RareNode:
    def __init__(self, system: System, station: Station, prices: StationPriceList, terminus=False, echo=False):
        # graph literally has no reason to be two-way. why? no idea
        self.terminus = terminus
        self.system = system
        self.station = station
        self.prices = prices
        if terminus and not echo:  # new root node automatically creates graph-end terminus as well
            end = RareNode(None, None, None, terminus=True, echo=True)
            self.link_x_forward = end
            self.link_x_backward = None
            self.link_y_forward = end
            self.link_y_backward = None
            self.link_z_forward = end
            self.link_z_backward = None
            end.link_x_backward = self
            end.link_y_backward = self
            end.link_z_backward = self
            end.link_x_forward = None
            end.link_y_forward = None
            end.link_z_forward = None
        if not terminus:
            self.commodities = []

    @property
    def is_terminus(self):
        return self.terminus

    def add_commodity(self, name):
        comm = commodity_controller.get_by_name(name)
        if comm is None:
            comm = {'id': None, 'name': name}
        self.commodities.append(comm)

    def next_x_gt(self, node):
        if self.link_x_forward.terminus:
            return True
        if self.link_x_forward.system.x >= node.system.x:
            return True
        return False

    def next_y_gt(self, node):
        if self.link_y_forward.terminus:
            return True
        if self.link_y_forward.system.y >= node.system.y:
            return True
        return False

    def next_z_gt(self, node):
        if self.link_z_forward.terminus:
            return True
        if self.link_z_forward.system.y >= node.system.y:
            return True
        return False

    def __str__(self):
        return '<terminus>' if not self.system else f'{self.system.name}:{self.station.name}'

    def __eq__(self, other):
        return self.system == other.system and self.station == other.station

    def insert_x(self, node):
        swap = self.link_x_forward
        self.link_x_forward = node
        node.link_x_forward = swap
        swap.link_x_backward = node
        node.link_x_backward = self

    def insert_y(self, node):
        swap = self.link_y_forward
        self.link_y_forward = node
        node.link_y_forward = swap
        swap.link_y_backward = node
        node.link_y_backward = self

    def insert_z(self, node):
        swap = self.link_z_forward
        self.link_z_forward = node
        node.link_z_forward = swap
        swap.link_z_backward = node
        node.link_z_backward = self

# TODO: remove system graph from here and do a separate package
class RareGraph:
    def __init__(self, nodes: list = []):

        logger.info('Graph engine initializing')
        logger.info('Sorting nodes')
        # sorting to speed up insertion
        # graph contains lesser X-Y-Z as top nodes; all nodes are expected to have 3 links (XYZ)
        nodes = sorted(nodes, key=lambda t_node: (t_node.system.x, t_node.system.y, t_node.system.z))
        self.index = dict()
        self.nodes = list()
        logger.info('Creating root node')
        self.root = RareNode(None, None, None, terminus=True)
        logger.info('Iterating through nodes')
        bar = generate_bar(nodes.__len__(), 'Inserting systems into graph')
        bar.start()
        for node in nodes:  # todo: don't recalculate graph on each insert may be?!
            self.insert(node)
            bar.update(bar.value + 1)
        bar.finish()

    def insert(self, node):
        # three passes to link the node
        logger.info(f'Parsing {node.system.name}:{node.station.name}')

        ref_x = self.root
        ref_y = self.root
        ref_z = self.root
        logger.info('Parse_X')
        bar = generate_bar(self.nodes.__len__(), 'Parse_X')
        bar.start()
        while True:
            if ref_x.next_x_gt(node):
                ref_x.insert_x(node)
                break
            else:
                ref_x = ref_x.link_x_forward
            bar.update(bar.value + 1)
        bar.finish()

        bar = generate_bar(self.nodes.__len__(), 'Parse_Y')
        bar.start()
        while True:
            if ref_y.next_y_gt(node):
                ref_y.insert_y(node)
                break
            else:
                ref_y = ref_y.link_y_forward
            bar.update(bar.value + 1)
        bar.finish()

        bar = generate_bar(self.nodes.__len__(), 'Parse_Z')
        bar.start()
        while True:
            if ref_z.next_z_gt(node):
                ref_z.insert_z(node)
                break
            else:
                ref_z = ref_z.link_z_forward
            bar.update(bar.value + 1)
        bar.finish()

        # keep a full list for future ref
        self.nodes.append(node)

        # index this shit up.
        self.index[node.__str__()] = node

    def plot_to(self, x, y, z):
        nodes = []
        ref = self.root
        for i in range(x):
            if ref.link_x_forward.terminus:
                break
            ref = ref.link_x_forward
            nodes.append(ref)
        ref = self.root
        for i in range(y):
            if ref.link_y_forward.terminus:
                break
            ref = ref.link_y_forward
            nodes.append(ref)
        ref = self.root
        for i in range(z):
            if ref.link_z_forward.terminus:
                break
            ref = ref.link_z_forward
            nodes.append(ref)
        self.plot(nodes)

    def plot(self, nodes=None):
        if nodes is None:
            nodes = self.nodes
        ax = plt.gca(projection="3d")
        x = list()
        y = list()
        z = list()
        for node in nodes:
            x.append(node.system.x)
            y.append(node.system.y)
            z.append(node.system.z)

        ax.scatter(x, y, z, c='r', s=100)
        ax.plot(x, y, z, color='r')
        plt.show()

    def query(self, system, station):
        logger.info(f'Querying {system}:{station}')
        for node in self.nodes:
            if node.system.name == system and node.station.name == station:
                return node
        return None

    def closest(self, system):
        distance = system.distance(self.nodes[0].system)
        ref = self.nodes[0]
        for node in self.nodes:
            chk = system.distance(node.system) < distance
            if chk < distance:
                ref = node
                distance = chk
        return ref

    def __generate(self, route, visited=[], sell_distance=150, max_ly=None):
        visited.append(route[-1]["node"])
        res = os.system('cls')
        if res != 0:
            os.system('clear')
        print([r["node"].system.name for r in route])
        # determine list of possible next nodes
        dup_possible_next_jump = [route[-1]["node"].link_x_forward, route[-1]["node"].link_x_backward,
                     route[-1]["node"].link_y_forward, route[-1]["node"].link_y_backward,
                     route[-1]["node"].link_z_forward, route[-1]["node"].link_z_backward]
        # clear terminus
        dup_possible_next_jump = [jump for jump in dup_possible_next_jump if not jump.terminus]
        # removing duplicates
        possible_next_jump = []
        for jump in dup_possible_next_jump:
            if jump not in possible_next_jump:
                possible_next_jump.append(jump)

        # filter visited which should be sold
        filtered_visited = []
        # bar = generate_bar(visited.__len__(), 'Removing sellable visited')
        # bar.start()
        for visit in visited:
            if visit.system.distance(route[-1]["node"].system) < sell_distance:
                filtered_visited.append(visit)
            # bar.update(bar.value + 1)
        # bar.finish()
        # filter by distance
        to_skip = []
        # bar = generate_bar(possible_next_jump.__len__(), 'Filtering visited by closing distance and max jump distance')
        # bar.start()
        for jump in possible_next_jump:
            for visit in visited:
                if visit.system.distance(route[-1]["node"].system) > visit.system.distance(jump.system):
                    to_skip.append(jump)
            if max_ly is not None:
                if route[-1]["node"].system.distance(jump.system) > max_ly:
                    to_skip.append(jump)

            # bar.update(bar.value + 1)
        possible_next_jump = [jump for jump in possible_next_jump if jump not in to_skip]
        # bar.finish()
        # if none are left - return false, we've hit a dead-end

        if possible_next_jump.__len__() == 0:
            print('    ' * route.__len__(), f'Terminated!')
            # return visited nodes
            return False

        # if any have been visited: loop is complete!
        # bar = generate_bar(possible_next_jump.__len__(), 'Checking loop possibility')
        # bar.start()
        for jump in possible_next_jump:
            if jump in [r["node"] for r in route]:
                profits = route[-1]["node"].prices - jump.prices
                route.append({"node": jump, "distance": route[-1]["node"].system.distance(jump.system),
                              "commodity": profits[0], "profit": profits[1], "updated": jump.prices.updated()})
                """({"node": jump["node"], "distance": jump["node"].system.distance(route[-1]["node"].system, ),
                  "commodity": jump["commodity"], "profit": jump["profit"],
                  "updated": jump["node"].prices.updated()})"""
                # todo: fix this profits returning either 0 or None bullshit =\\
                return True
            # bar.update(bar.value + 1)
        # bar.finish()

        # check profits:
        # bar = generate_bar(possible_next_jump.__len__(), 'Populating prices')
        # bar.start()
        jumps_and_profits = []
        for jump in possible_next_jump:
            comm_data = route[-1]["node"].prices - jump.prices
            jumps_and_profits.append({"node": jump, "profit": 0 if comm_data[1] is None else comm_data[1], "commodity": comm_data[0],
                                      "distance": jump.system.distance(route[-1]["node"].system)})
            # bar.update(bar.value + 1)
        jumps_and_profits = sorted(jumps_and_profits, key=lambda x: x["distance"])
        # bar.finish()
        # else take the best profit one and re-run GENERATE (doing this in a loop here
        # print(jumps_and_profits)
        ref_len = route.__len__()
        best_profit = 0
        best_profit_jump = jumps_and_profits[0]
        for jump in jumps_and_profits:
            route.append({"node": jump["node"], "distance": jump["distance"],
                          "commodity": jump["commodity"], "profit": jump["profit"],
                          "updated": jump["node"].prices.updated()})
            loop = self.__generate(route, visited=filtered_visited, sell_distance=sell_distance, max_ly=max_ly)
            # function has altered ROUTE
            if loop:  # if we have looped somewhere down the line, current route is cool
                return loop
            # if we do not loop, correct the route back:
            # know that we can't know how many nodes down the line the latest FALSE for a loop returned
            profit = 0  # check generated route cumulative profit
            for proposed_route_jump in route[ref_len:]:
                profit += proposed_route_jump["profit"]
            if profit > best_profit:
                best_profit_jump = jump
            while route.__len__() > ref_len:
                route.pop()
        # this loop on the first node will guarantee that if there's a loop it will get created.
        # if all failed to loop re-run for the best profit again and return
        # at this point, our route and visited are as we entered the algo

        jump = best_profit_jump
        route.append({"node": jump["node"], "distance": jump["node"].system.distance(route[-1]["node"].system),
                      "commodity": jump["commodity"], "profit": jump["profit"],
                      "updated": jump["node"].prices.updated()})
        # while we use the __generate function, it will not pop our current system or mess with the route.
        return self.__generate(route, filtered_visited, sell_distance=sell_distance, max_ly=max_ly)

    def node_by_commodity(self, cid=None, name=None):
        if not cid and not name:
            return None
        for node in self.nodes:
            if node.terminus:
                continue
            if cid is not None:
                if cid in [q['id'] for q in node.commodities]:
                    return node
            if name is not None:
                if name in [q['name'] for q in node.commodities]:
                    return node

    def generate_from(self, starting_node, current_system=None, first_jump_prices=None, sell_distance=150, max_ly=None):
        """
        format helper:
        [{"node": self.graph.nodes[0], "distance": 50, "commodity": "lulz", "profit": 400, "updated": 18},
         {"node": self.graph.nodes[1], "distance": 56, "commodity": "lulz2", "profit": 1400, "updated": 144}]"""
        route = [{"node": starting_node, "distance": 0, "commodity": None, "profit": None, "updated": starting_node.prices.updated()}]

        looped = self.__generate(route, sell_distance=sell_distance, max_ly=max_ly)  # visited systems are not returned because they shuffle about.

        if first_jump_prices is not None:
            best_deal = first_jump_prices - starting_node.prices
            route.insert(0, {"node": None, "distance": 0, "commodity": best_deal[0], "profit": best_deal[1], "updated": starting_node.prices.updated()})
            route[1]["distance"] = current_system.distance(route[1]["node"].system)

        for jump in route:
            jump['sell'] = []

        for jump in route:  # find where to sell what
            if jump["node"] is None:
                continue
            distance = 0
            ref = None
            for check in route:  #fixme if node does not exist (first-jump-thing)
                chk_dst = jump["node"].system.distance(check["node"].system)
                if chk_dst > distance:
                    distance = chk_dst
                    ref = check
            ref['sell'].append(jump['node'])
        if not looped:  # pretend we do have a loop
            route[0]['distance'] = route[-1]['node'].system.distance(route[0]['node'].system)
        return route

class RareLoader:
    def __init__(self, sell_distance=150):
        self.sell_distance = sell_distance

    def prime(self, ship_size, max_distance_from_star, filter_permit=True):
        logger.info('Opening file for RareLoader')
        f = open(f'{os.path.dirname(os.path.realpath(__file__))}/reference.tsv', 'r', encoding='utf-8')
        header = f.readline().split(';')
        reference = list()
        logger.info('Reading reference')
        for line in f.readlines():
            entry = dict()
            logger.debug(f'Processing {entry}')
            for e, k in zip(line.split(';'), header):
                logger.debug(f'{k} is {e}')
                entry[k.strip()] = e.strip()
            reference.append(entry)

        f.close()

        # load up systems to get id's
        # todo: may be do sort-merge here? although not really necessary on a small list
        logger.info('Loading systems')
        systems = system_loader(names=[q['System'] for q in reference], filter_needs_permit=filter_permit)

        # load up station infos
        station_filter = list()
        # pair up each reference with a system id
        logger.info('Creating station filter')
        for ref in reference:
            for sys in systems:
                if ref['System'] == sys.name:
                    station_filter.append((ref['Port'], sys.id))
                    break
        logger.info('Loading stations')
        stations = station_loader(refs=station_filter, landing_pad=ship_size, distance_to_star=max_distance_from_star)
        prices_list = listing_loader([q.sid for q in stations])
        nodes = []

        logger.info('Creating graph nodes')
        for ref in reference:
            system = None
            for sys in systems:
                if ref['System'] == sys.name:
                    system = sys
                    break
            if system is None:
                logger.warning(f'System {ref["System"]} not found, skipping')
                continue
                # raise NameError(f'System >:{ref["System"]}:< not found')
            station = None
            for st in stations:
                if ref['Port'] == st.name:
                    station = st
                    break
            if station is None:
                logger.warning(f'Station {ref["Port"]} not found')
                # raise NameError(f'Station {ref["Port"]} not found')
                continue
            dup = False
            for node in nodes:
                if node.system.id == system.id and node.station.sid == station.sid:
                    node.add_commodity(ref["Commodity"])
                    dup = True
                    break
            if dup:
                continue
            prices = None
            for listing in prices_list:
                if listing.station == station.sid:
                    prices = listing
            if prices is None:
                logger.warning(f'Skipping {ref["Commodity"]} because {station.name} has no listings')
                continue
            logger.debug(f'Created node for {ref["Commodity"]}')

            nodes.append(RareNode(system, station, prices))
            nodes[-1].add_commodity(ref['Commodity'])
        logger.info('Creating graph')
        self.graph = RareGraph(nodes=nodes)

        # create graph of Systems with stations attached
        # initialize current state if requested
        logger.info('Dumping backup data')
        pickle.dump(self.graph, open(f'{os.path.dirname(os.path.realpath(__file__))}/graph_dump.pcl', 'wb'))

    def init(self, ship_size='None', max_distance_from_star=-1, filter_needs_permit=False):
        self.ship_size = ship_size
        self.max_distance_from_star = max_distance_from_star

        logger.info('Rare loader initializing')
        logger.info('Trying to unpickle populated data')
        results = []
        for api in ['commodities.json', 'stations.jsonl', 'listings.csv', 'systems.csv']:
            results.append(eddb_prime.recache(api))
        if any(results):
            logger.info('Recache needed')
            self.prime(self.ship_size, self.max_distance_from_star, filter_needs_permit)
        else:
            try:
                self.graph = pickle.loads(
                    open(f'{os.path.dirname(os.path.realpath(__file__))}/graph_dump.pcl', 'rb+').read())
            except FileNotFoundError:
                logger.info('Failed to load up backup, reloading')
                self.prime(ship_size, max_distance_from_star, filter_needs_permit)

    def alternative_determine_next(self, cur_system, cur_station, limit=10, visited=[]):   # expecting current
        # cur_system = self.route[-1][0]
        ref_dict = {}
        node_dict = {}
        prices = None
        for node in self.graph.nodes:
            if node.terminus:
                continue
            if node.system == cur_system:
                prices = node.prices
                continue
            ref_dict[node.__str__()] = cur_system.distance(node.system)
            node_dict[node.__str__()] = node
        if prices is None:
            prices = StationPriceList(cur_station.sid)
            prices.populate()
        nodelist = sorted(ref_dict, key=lambda k: ref_dict[k])
        nodelist = [node_dict[q] for q in nodelist]
        to_skip = []
        for node in nodelist:
            for visit in visited:
                if cur_system.distance(visit.system) > node.system.distance(visit.system):  # distance closes!
                    to_skip.append(node)
        logger.debug(f'Filtering systems from {nodelist.__len__()}')
        nodelist = [node for node in nodelist if node not in to_skip]
        logger.debug(f'Resulting list is {nodelist.__len__()}')
        ret_obj = []
        for node in nodelist[:limit]:
            profit = prices - node.prices
            ret_obj.append({'system': node.system.name, 'station': node.station.name,
                            'commodity': [q['name'] for q in node.commodities],
                            'buy': '<none>' if profit[0] is None else profit[0].name,
                            'profit': profit[1], 'distance': ref_dict[node.__str__()],
                            'updated': (time.time() - node.system.updated_at) / 60 / 60})
        return ret_obj

    def query(self, system, station):
        return self.graph.query(system, station)

    def generate(self, system, station, node=None, max_ly=None):  # TODO: add limiters for something. max distance would be ok, but it's always "next over" so dunno. return trips are a failure though.
        if node is None:
            node = self.graph.closest(system)
            prices = StationPriceList(station)
            prices.populate()
        else:
            prices = None
        route = self.graph.generate_from(node, current_system=system, first_jump_prices=prices, sell_distance=self.sell_distance, max_ly=max_ly)
        return route

    def check_sell(self, current_system, visited):
        dst = current_system.distance(visited.system)
        if dst >= self.sell_distance:
            return dst
        else:
            return None

    def query_by_commodity(self, cid=None, name=None):
        return self.graph.node_by_commodity(cid=cid, name=name)

if __name__ == '__main__':
    chona = System(name='Chona')
    chona.populate()
    rl = RareLoader()
    rl.init('L', 50000, True)
    distance = 50000
    ref = None
    for node in rl.graph.nodes:
        if chona.distance(node.system) < distance:
            distance = chona.distance(node.system)
            ref = node.system
    print(ref.name, distance)


from sovtoken.transactions import TokenTransactions

INPUTS = 'inputs'
OUTPUTS = 'outputs'
EXTRA = 'extra'
ADDRESS = 'address'
SIGS = 'signatures'
RESULT = 'result'
AMOUNT = 'amount'
SEQNO = 'seqNo'

TOKEN_LEDGER_ID = 1001

MINT_PUBLIC = TokenTransactions.MINT_PUBLIC.value
XFER_PUBLIC = TokenTransactions.XFER_PUBLIC.value
GET_UTXO = TokenTransactions.GET_UTXO.value

ACCEPTABLE_TXN_TYPES = (MINT_PUBLIC, XFER_PUBLIC, GET_UTXO)


'''
打点钱包获取用户信息
'''
import readConfig
import json
import unittest,paramunittest
from common import common,Log
from common import configHttp

userInfo_xls = common.get_xls('userCase.xlsx','userInfo')
localReadConfig = readConfig.ReadConfig()
configHttp = configHttp.ConfigHttp()
info = ()

@paramunittest.parametrized(*userInfo_xls)
class UserInfo(unittest.TestCase):

    def setParameters(self,case_name,method,code,cn_msg):
        '''
        set params
        :param case_name:
        :param method:
        :param code:
        :param cn_msg:
        :return:
        '''
        self.case_name = str(case_name)
        self.method = str(method)
        self.code = int(code)
        self.cn_msg = str(cn_msg)

    def setUp(self):
        self.log = Log.MyLog.get_log()
        self.logger = self.log.get_logger()

    def testuserInfo(self):
        '''
        用户信息
        :return:
        '''
        self.url = common.get_url_from_xml('userInfo')

        # set url
        configHttp.set_url(self.url)

        # set headers
        configHttp.set_headers()

        # test interface
        self.return_json = configHttp.request_get()

        self.checkResult()

    def checkResult(self):
        '''
        check test result
        :return:
        '''
        self.info = json.loads(self.return_json.text)

        self.assertEqual(self.info['code'],self.code)
        self.assertEqual(self.info['cn_msg'],self.cn_msg)

if __name__ == '__main__':
    unittest.main()


import math

from sklearn.cluster import DBSCAN

import numpy as np
from typing import List

from resources.bboxes import sort_boxes_top2down_wrt_left2right_order, sort_boxes, fit_bbox
from resources.dtos import Paragraph, Line

# КОД ДЛЯ СТУДЕНТА
class ParagraphFinder:
    def __init__(
            self,
            angle_threshold: int = 10,
            overlapping_threshold: float = 0.3,
            max_distance: int = 1,
            cluster_eps: float = 0.5
    ):
        self.angle_threshold = angle_threshold
        self.overlapping_threshold = overlapping_threshold
        self.max_distance = max_distance
        self.cluster_eps = cluster_eps

    @staticmethod
    def _distance(line1: Line, line2: Line) -> float:
        (x11, y11), (x12, y12), _, _ = line1.normalized_bbox
        (x21, y21), (x22, y22), _, _ = line2.normalized_bbox
        y1m = (y12 + y11) / 2
        y2m = (y22 + y21) / 2
        return abs(y1m - y2m)

    @staticmethod
    def _length(line: Line) -> float:
        return np.linalg.norm(line.normalized_bbox[0] - line.normalized_bbox[1])

    def _overlapping(self, line1: Line, line2: Line) -> float:
        (x11, y11), (x12, y12), _, _ = line1.normalized_bbox
        (x21, y21), (x22, y22), _, _ = line2.normalized_bbox
        x_min = max(x11, x21)
        x_max = min(x12, x22)

        if x_max - x_min < 0:
            return 0

        return (x_max - x_min) / self._length(line1)

    @staticmethod
    def _angle(line1: Line, line2: Line) -> float:
        (x11, y11), (x12, y12), _, _ = line1.normalized_bbox
        (x21, y21), (x22, y22), _, _ = line2.normalized_bbox
        d_x1 = x12 - x11
        d_y1 = y12 - y11
        d_x2 = x22 - x21
        d_y2 = y22 - y21

        theta = math.atan(d_y2 / d_x2) - math.atan(d_y1 / d_x1)
        theta = math.degrees(theta)

        if theta > 90:
            theta -= 180
        if theta < -90:
            theta += 180

        return theta

    def paragraph_distance(self, line1: np.ndarray, line2: np.ndarray) -> float:
        line1 = Line(normalized_bbox=line1.reshape(4, 2))
        line2 = Line(normalized_bbox=line2.reshape(4, 2))

        if abs(self._angle(line1, line2)) > self.angle_threshold:
            return self.max_distance

        if self._overlapping(line1, line2) < self.overlapping_threshold:
            return self.max_distance

        return self._distance(line1, line2)

    @staticmethod
    def prepare_lines(lines: List[Line]) -> np.ndarray:
        return np.array([line.normalized_bbox.reshape(-1) for line in lines])

    def fit_cluster(self, lines: List[Line]) -> DBSCAN:
        prepared_lines = self.prepare_lines(lines)
        cluster = DBSCAN(metric=self.paragraph_distance, eps=self.cluster_eps)
        cluster.fit(prepared_lines)
        return cluster

    @staticmethod
    def sort_paragraphs(paragraphs):
        for par in paragraphs:
            par.items = sort_boxes(par.items, sorting_type='top2down')  # сортировка сверху вниз
        paragraphs = sort_boxes_top2down_wrt_left2right_order(paragraphs)  # сортировка в порядке чтения
        return paragraphs

    def find_paragraphs(self, lines: List[Line]) -> List[Paragraph]:
        cluster = self.fit_cluster(lines)

        paragraphs = []
        for label in set(cluster.labels_):
            line_indexes = np.argwhere(cluster.labels_ == label)
            par_lines = [lines[idx[0]] for idx in line_indexes]
            bbox = fit_bbox(np.array([lines[idx[0]].bbox for idx in line_indexes]))
            par = Paragraph(items=par_lines, bbox=bbox)
            paragraphs.append(par)
        paragraphs = self.sort_paragraphs(paragraphs)
        return paragraphs


# -*- coding: utf-8 -*-
"""
Created on Sun Jun 11 20:34:16 2017

@author: yli
"""
import os
import numpy as np

path = 'train_data.txt'
save_path = 'room_id.txt'
num = 10000

if not os.path.exists(save_path):
    os.makedirs(save_path)

def get_roomid(datapath):
    f= open(datapath,'r')
    data = f.readlines()
    room_id = [x.split()[5] for x in data]
    return room_id[1:]

room_id = get_roomid(path)


import cv2
import numpy as np
import os
import json
import random
import configparser
import boto3
import shutil
import io

config = configparser.ConfigParser()
config.read('config.ini')

# Read image
positivedir = 'data/frames/positive'
boundaries = [
    'AllMaterial',
    'Wood',
    'Brick',
    'Metal',
    'Weapons Slots',
    'Weapons',
    'HealthShield',
    'Health',
    'Shield',
    'GameStatus',
    'StormClock',
    'PlayersLeft',
    'StormCount',
    'Chat',
    'TeamStats',
    'TeammateHealth',
    'Streamer'
]

multipleSelects = [
    'Weapons',
    'TeammateHealth',
]
if not os.path.exists('data/assets'):
    os.makedirs('data/assets')

assetBoundaries = {}

newFileFound = False
s3client = boto3.client('s3',
                        aws_access_key_id=config['S3_BUCKET']['accesskey'],
                        aws_secret_access_key=config['S3_BUCKET']['accesssecret'],)
positivesList = s3client.get_object(
    Bucket=config['S3_BUCKET']['bucket'], Key='positives.json')
positiveImages = s3client.list_objects(
    Bucket=config['S3_BUCKET']['bucket'], Prefix='frames/positive')
positives = json.load(positivesList['Body'])
while not newFileFound:
    fileIndex = random.randint(0, len(positiveImages['Contents']) - 1)
    filename = os.path.basename(positiveImages['Contents'][fileIndex]['Key'])
    if not filename in positives:
        fileKey = positiveImages['Contents'][fileIndex]['Key']
        newFileFound = True
temppath = os.path.join(config['DIRECTORY']['temp'],
                        'crop')
if not os.path.exists(temppath):
    os.makedirs(temppath)
i = 0
tempfilename = os.path.join(temppath, filename)
s3client.download_file(config['S3_BUCKET']['bucket'], fileKey, tempfilename)

frameWidth = 600
im = cv2.imread(tempfilename)
width, height = im.shape[:2]
# imgScale = frameWidth/width
# newX,newY = im.shape[1]*imgScale, im.shape[0]*imgScale
# resized = cv2.resize(im,(int(newX),int(newY)))

removeImg = False
bail = False

for asset in boundaries:
    mulitpleBounds = []
    # # Select ROI
    font = cv2.FONT_HERSHEY_SIMPLEX
    if removeImg == True or bail == True:
        break
    newimg = im.copy()
    while True:
        cv2.putText(newimg, f'Select {asset}', (int(width/2), 130), font,
                    .8, (200, 255, 155), 2, cv2.LINE_AA)
        cv2.putText(newimg, f'Press enter then d to delete this frame as a positive', (int(width/2), 160), font,
                    .8, (20, 95, 70), 2, cv2.LINE_AA)
        r = cv2.selectROI("Image", newimg)

        if not (asset == 'Weapons' or asset == 'TeammateHealth') or r[2] == 0:
            if r[2] == 0:
                nextCommand = cv2.waitKey()
                if nextCommand == 100:
                    removeImg = True
                if nextCommand == 113:
                    bail = True
            break
        else:
            mulitpleBounds.append(r)
    if r[2] > 0:
        if (asset == 'Weapons' or asset == 'TeammateHealth'):
            assetBoundaries[asset] = mulitpleBounds
        else:
            assetBoundaries[asset] = r

if removeImg == True:
    s3client.delete_object(Bucket=config['S3_BUCKET']['bucket'], Key=fileKey)
    print(f'deleted file {filename}')
elif not bail:
    assetBoundaries['CroppedWidth'] = frameWidth
    positives[os.path.splitext(filename)[0]] = assetBoundaries
    p = json.dumps(positives)
    s3client.put_object(
        Bucket=config['S3_BUCKET']['bucket'], Key='positives.json', Body=p)

shutil.rmtree(temppath)


class Solution:
    def smallestNumber(self, pattern: str) -> str:
        # res length is len(pattern) + 1
        res = [1]
        ans = ""

        for char in pattern:
            if char == 'I': # append the lowest increased number possible that isn't in res
                num = res[-1] + 1
                while num in res:
                    num += 1
                res.append(num)
            else:
                res.append(res[-1]) # use res[-1] as placeholder for new element
                for i in range(len(res) - 1, 0, -1): # reverse from end to 0
                    if res[i - 1] == res[i]: # add one to current 2nd to last element
                        res[i - 1] += 1 # if they are the same which they should be

        for num in res:
            ans += str(num)

        return ans


# button widget
from tkinter import *
import tkinter as tk
root = Tk()
r = tk.Tk()
button = tk.Button(r, text='click me', width=25, command=r.destroy)
button.pack()
Label(root, text='Enter the College Name').grid(row=0)
Label(root, text='Enter the College Name').grid(row=1)
e1 = Entry(root)
e2 = Entry(root)
e1.grid(row=0, column=1)
e2.grid(row=1, column=1)
r.mainloop()


import mysql.connector
from dotenv import load_dotenv
import config
from geopy import distance
load_dotenv()

def cursor_func(sql_komento):
    connection = mysql.connector.connect(
        host='127.0.0.1',
        port=3306,
        database='flight_game',
        user='root',
        password="",
        autocommit=True
    )
    cursor = connection.cursor()
    cursor.execute(sql_komento)
    result = cursor.fetchall()
    return result

class Airport:
    # lisätty data, jottei tartte jokaista lentokenttää hakea erikseen
    def __init__(self, ident, active=False, data=None):
        self.ident = ident
        self.name = data['name']
        self.latitude = float(data['latitude'])
        self.longitude = float(data['longitude'])
        sql = "SELECT  latitude_deg, longitude_deg FROM Airport WHERE ident='" + ident + "'"
        print(sql)
        cur = config.conn.cursor()
        cur.execute(sql)
        res = cur.fetchall()

    def find_nearby_airports(self):
        lista = []
        # haetaan kaikki tiedot kerralla
        sql = "SELECT ident, name, latitude_deg, longitude_deg FROM Airport WHERE latitude_deg BETWEEN "
        sql += str(self.latitude - config.max_lat_dist) + " AND " + str(self.latitude + config.max_lat_dist)
        sql += " AND longitude_deg BETWEEN "
        sql += str(self.longitude - config.max_lon_dist) + " AND " + str(self.longitude + config.max_lon_dist)
        print(sql)
        cur = config.conn.cursor()
        cur.execute(sql)
        res = cur.fetchall()
        for r in res:
            if r[0] != self.ident:
                # lisätty data, jottei jokaista kenttää tartte hakea
                # uudestaan konstruktorissa
                data = {'name': r[1], 'latitude': r[2], 'longitude': r[3]}
                print(data)
                nearby_apt = Airport(r[0], False, data)
                nearby_apt.distance = self.distanceTo(nearby_apt)
                if nearby_apt.distance <= config.max_distance:
                    lista.append(nearby_apt)
                    nearby_apt.co2_consumption = self.co2_consumption(nearby_apt.distance)
        return lista

    def distanceTo(self, target):
        coords_1 = (self.latitude, self.longitude)
        coords_2 = (target.latitude, target.longitude)
        dist = distance.distance(coords_1, coords_2).km
        consumption = self.co2_consumption(dist)
        return int(consumption)

    def co2_consumption(self, km):
        consumption = km / (1.69 * 3)
        return consumption


import json

from classes.monster import Monster

monster = Monster(name="Orc", mclass="evil", hitpoints= 32, armor= 12)

#  print(monster)

dicMonster = vars(monster)

print(dicMonster)

f = open("monster.json", "w+")
json.dump(dicMonster, f, indent=2)
f.close()