import json
import urllib
from textwrap import dedent

import netlib.http
from netlib.utils import parse_content_type

def curl_command(flow):
    data = "curl "

    for k, v in flow.request.headers.fields:
        data += "-H '%s:%s' " % (k, v)

    if flow.request.method != "GET":
        data += "-X %s " % flow.request.method

    full_url = flow.request.scheme + "://" + flow.request.host + flow.request.path
    data += "'%s'" % full_url

    if flow.request.content:
        data += " --data-binary '%s'" % flow.request.content

    return data

def python_code(flow):
    code = dedent("""
        import requests

        url = '{url}'
        {headers}{params}{data}
        response = requests.request(
            method='{method}',
            url=url,{args}
        )

        print(response.text)
    """).strip()

    components = map(lambda x: urllib.quote(x, safe=""), flow.request.path_components)
    url = flow.request.scheme + "://" + flow.request.host + "/" + "/".join(components)

    args = ""
    headers = ""
    if flow.request.headers:
        lines = ["    '%s': '%s',\n" % (k, v) for k, v in flow.request.headers.fields]
        headers += "\nheaders = {\n%s}\n" % "".join(lines)
        args += "\n    headers=headers,"

    params = ""
    if flow.request.query:
        lines = ["    '%s': '%s',\n" % (k, v) for k, v in flow.request.query]
        params = "\nparams = {\n%s}\n" % "".join(lines)
        args += "\n    params=params,"

    data = ""
    if flow.request.body:
        json_obj = is_json(flow.request.headers, flow.request.body)
        if json_obj:
            # Without the separators field json.dumps() produces
            # trailing white spaces: https://bugs.python.org/issue16333
            data = json.dumps(json_obj, indent=4, separators=(',', ': '))
            data = "\njson = %s\n" % data
            args += "\n    json=json,"
        else:
            data = "\ndata = '''%s'''\n" % flow.request.body
            args += "\n    data=data,"

    code = code.format(
        url=url,
        headers=headers,
        params=params,
        data=data,
        method=flow.request.method,
        args=args,
    )

    return code

def raw_request(flow):
    data = netlib.http.http1.assemble_request(flow.request)
    return data

def is_json(headers, content):
    if headers:
        ct = parse_content_type(headers.get("content-type", ""))
        if ct and "%s/%s" % (ct[0], ct[1]) == "application/json":
            try:
                return json.loads(content)
            except ValueError:
                return False
    return False


from pathlib import Path

import pandas as pd
import prefect
from prefect import Flow, Parameter, case, task
from prefect.executors import LocalDaskExecutor

from config import (
    CONTROLS_STATISTICS_CSV_RESOURCE_ID,
    CONTROLS_STATISTICS_CSV_RESOURCE_TITLE,
    CONTROLS_STATISTICS_DATASET_ID,
    FLEET_SEGMENTS_CSV_RESOURCE_ID,
    FLEET_SEGMENTS_CSV_RESOURCE_TITLE,
    IS_INTEGRATION,
)
from src.pipeline.generic_tasks import extract
from src.pipeline.processing import prepare_df_for_loading
from src.pipeline.shared_tasks.control_flow import check_flow_not_running
from src.pipeline.shared_tasks.datagouv import get_csv_file_object, update_resource

@task(checkpoint=False)
def extract_controls_open_data() -> pd.DataFrame:
    return extract(
        "monitorfish_remote",
        "monitorfish/controls_open_data.sql",
        dtypes={
            "control_year": int,
            "control_month": int,
            "number_controls": int,
            "number_controls_ytd": int,
        },
    )

@task(checkpoint=False)
def extract_fleet_segments_open_data() -> pd.DataFrame:
    return extract(
        "monitorfish_remote",
        "monitorfish/fleet_segments_open_data.sql",
    )

@task(checkpoint=False)
def transform_controls_open_data(controls) -> pd.DataFrame:
    logger = prefect.context.get("logger")
    controls = prepare_df_for_loading(
        controls,
        logger=logger,
        nullable_integer_columns=["target_number_of_controls_year"],
    )
    return controls

@task(checkpoint=False)
def transform_fleet_segments_open_data(fleet_segments) -> pd.DataFrame:
    logger = prefect.context.get("logger")
    fleet_segments = prepare_df_for_loading(
        fleet_segments,
        logger=logger,
        nullable_integer_columns=["year"],
        pg_array_columns=["gears", "fao_areas", "species"],
    )
    return fleet_segments

with Flow("Controls open data", executor=LocalDaskExecutor()) as flow:

    flow_not_running = check_flow_not_running()
    with case(flow_not_running, True):

        dataset_id = Parameter("dataset_id", default=CONTROLS_STATISTICS_DATASET_ID)
        controls_resource_id = Parameter(
            "controls_resource_id", default=CONTROLS_STATISTICS_CSV_RESOURCE_ID
        )
        controls_resource_title = Parameter(
            "controls_resource_title", default=CONTROLS_STATISTICS_CSV_RESOURCE_TITLE
        )

        fleet_segments_resource_id = Parameter(
            "fleet_segments_resource_id", default=FLEET_SEGMENTS_CSV_RESOURCE_ID
        )
        fleet_segments_resource_title = Parameter(
            "fleet_segments_resource_title", default=FLEET_SEGMENTS_CSV_RESOURCE_TITLE
        )

        is_integration = Parameter("is_integration", default=IS_INTEGRATION)

        controls = extract_controls_open_data()
        controls = transform_controls_open_data(controls)
        controls_csv_file = get_csv_file_object(controls)

        fleet_segments = extract_fleet_segments_open_data()
        fleet_segments = transform_fleet_segments_open_data(fleet_segments)
        fleet_segments_csv_file = get_csv_file_object(fleet_segments)

        update_resource(
            dataset_id=dataset_id,
            resource_id=controls_resource_id,
            resource_title=controls_resource_title,
            resource=controls_csv_file,
            mock_update=is_integration,
        )

        update_resource(
            dataset_id=dataset_id,
            resource_id=fleet_segments_resource_id,
            resource_title=fleet_segments_resource_title,
            resource=fleet_segments_csv_file,
            mock_update=is_integration,
        )

flow.file_name = Path(__file__).name


from collections import deque
def BFS(s):
    D = [0] * (V + 1)
    visit = [False] * (V + 1)
    Q = deque()
    Q.append(s); D[s] = 0

    while Q:
        u = Q.popleft()
        for v in G[u]:
            if D[v] > D[u] + w:
                D[v] += D[u] + w
                Q.append(v)

python/알고리즘 응용/그래프/최단경로.py


import pandas as pd
import numpy as np

Rf = 1.56
GME_data = pd.read_csv('GME.csv')
GME_data["Returns"] = ""
GME_data["From Mean"] = ""

SP_data = pd.read_csv('S&P.csv')
SP_data["Returns"] = ""
SP_data["From Mean"] = ""

class Details:
    def __init__(self, data, mean, varience, covarience, beta, yearly_mean, CAPM):
        self.data = data
        self.mean = mean
        self.varience = varience
        self.covarience = covarience
        self.beta = beta
        self.yearly_mean = yearly_mean
        self.CAPM = CAPM

def varience(mean, data):
    count = 0
    avg_sum2 = 0
    for point in range(1, len(data)):
        count += 1
        avg_sum2 += (mean - data[point]) ** 2

    varience = avg_sum2 / count
    return(varience)

def yearly_mean(mean):
    result = ((1 + (mean/100)) ** 330) - 1

    return(result)

def ind_return(data):
    stuff = []
    ranger = len(data["Adj Close"])
    mean_sum = 0
    count = 0

    for Adj in range(1, ranger):
        first = (data["Adj Close"][Adj - 1])
        sec = (data["Adj Close"][Adj])
        if isinstance(first, str):
            first = first.replace(",", "").replace(".", "")
            sec = sec.replace(".", "").replace(",", "")
        first = int(first)
        sec = int(sec)
        r = sec - first
        pr = r / first
        mean_sum += float(round(pr, 8) * 100)
        count += 1

        data["Returns"][Adj] = float(round(pr, 8) * 100)
    mean = mean_sum / count
    stuff.append(mean)
    thing = varience(mean, data["Returns"])
    stuff.append(thing)

    for Adj in range(1, ranger):
        num = (data["Returns"][Adj])
        data["From Mean"][Adj] = num - mean

    return(stuff)

def covarience(set1, set2):
    summy = 0
    count = -1
    set1 = set1["From Mean"]
    set2 = set2["From Mean"]
    for i in range(1, len(set1)):
        product = float(set1[i]) * float(set2[i])
        summy += product
        count += 1
    covarience = summy / count
    return(covarience)

GME = Details(GME_data, ind_return(GME_data)[0], ind_return(GME_data)[1], "null", "null", "null", "null")

SP = Details(SP_data, ind_return(SP_data)[0], ind_return(SP_data)[1], "null", "null", "null", "null")

cov1 = covarience(GME_data, SP_data)

GME.covarience = cov1

GME.beta = cov1 / SP.varience

GME.yearly_mean = yearly_mean(GME.mean)

SP.yearly_mean = yearly_mean(SP.mean)

def CAPM(stock):
    result = Rf + (stock.beta * (stock.mean - Rf))
    return result
GME.CAPM = CAPM(GME)
print(GME.CAPM)


from app.models.db import db, Group, UserGroup, Expense, Transaction, Comment
from faker import Faker

faker = Faker()

def seed_expenses():

    expenses1 = Expense(description="Electric", amount=240, date=faker.date(
        pattern='%m-%d-%Y', end_datetime=None), notes="Electric bill for January", group_id=1, user_id=2)
    expenses2 = Expense(description="Flights", amount=400, date=faker.date(
        pattern='%m-%d-%Y', end_datetime=None), notes="Flights to Arubaaaaa", group_id=2, user_id=1)
    expenses3 = Expense(description="Internet", amount=120, date=faker.date(
        pattern='%m-%d-%Y', end_datetime=None), notes="Comcast", group_id=3, user_id=5)
    expenses4 = Expense(description="Groceries", amount=200, date=faker.date(
        pattern='%m-%d-%Y', end_datetime=None), notes="Trip to Trader Joes", group_id=4, user_id=1)
    expenses5 = Expense(description="Uber", amount=30, date=faker.date(
        pattern='%m-%d-%Y', end_datetime=None), notes="Uber to bar", group_id=1, user_id=3)

    db.session.add(expenses1)
    db.session.add(expenses2)
    db.session.add(expenses3)
    db.session.add(expenses4)
    db.session.add(expenses5)

    db.session.commit()

def undo_expenses():
    db.session.execute('TRUNCATE expenses;')
    db.session.commit()


#!/usr/bin/env python

import rospy
import numpy as np
import argparse

from UserInputMapper import UserInputMapper
from RobotState import RobotState
import input_handlers
from input_handlers import KinovaJoystickListener, HydraListener, MouseJoystickListener

from AdaTeleopHandler import *

VIEWER_DEFAULT = 'InteractiveMarker'
SIMULATE_DEFAULT = False

def Initialize_Adapy(args):
    """ Initializes robot and environment through adapy, using the specified environment path

    @param env_path path to OpenRAVE environment
    @return environment, robot
    """

    #env_path = '/environments/tablewithobjects_assisttest.env.xml'
    adapy_args = {'sim':args.sim,
                  'attach_viewer':args.viewer,
                  #'env_path':env_path
                  }
    openravepy.RaveInitialize(True, level=openravepy.DebugLevel.Debug)
    #openravepy.misc.InitOpenRAVELogging();
    env, robot = adapy.initialize(**adapy_args)
    Init_Robot(robot)
    return env,robot

def Init_Robot(robot):
    robot.SetActiveDOFs(range(6))
    #self.robot.arm.hand.OpenHand()
    #if (self.sim):
    robot_pose = np.array([[1, 0, 0, 0.409],[0, 1, 0, 0.338],[0, 0, 1, 0.795],[0, 0, 0, 1]])
    robot.SetTransform(robot_pose)
    if (robot.simulated):
      #servo simulator params
      robot.arm.servo_simulator.period=1./200.

def Reset_Robot(robot):
  if robot.simulated:
    num_hand_dofs = len(robot.arm.hand.GetDOFValues())
    inds, pos = robot.configurations.get_configuration('home')
    with robot.GetEnv():
      robot.SetDOFValues(pos, inds)
      robot.arm.hand.SetDOFValues(np.ones(num_hand_dofs)*0.1)
  #else:
    #robot.arm.hand.OpenHand()
  #  robot.arm.PlanToNamedConfiguration('home', execute=True)

if __name__ == "__main__":
    #parser.add_argument('-num', '--mouse-num', help='mouse number given by X in /dev/input/mouseX', type=str)
    parser = argparse.ArgumentParser(description="Direct Teleoperation for Ada")

    parser.add_argument('-s', '--sim', action='store_true', default=SIMULATE_DEFAULT,
                        help='simulation mode')
    parser.add_argument('-v', '--viewer', nargs='?', const=True, default=VIEWER_DEFAULT,
                        help='attach a viewer of the specified type')
    #parser.add_argument('--env-xml', type=str,
                        #help='environment XML file; defaults to an empty environment')
    parser.add_argument('--debug', action='store_true',
                        help='enable debug logging')
    parser.add_argument('-input', '--input-interface-name', help='name of the input interface. Possible choices: ' + str(possible_teleop_interface_names), type=str)
    parser.add_argument('-joy_dofs', '--num-input-dofs', help='number of dofs of input, either 2 or 3', type=int, default=2)
    args = parser.parse_args()

    rospy.init_node('ada_teleoperation', anonymous = True)

    env,robot = Initialize_Adapy(args)
    Reset_Robot(robot)
    ada_teleop = AdaTeleopHandler(env, robot, args.input_interface_name, args.num_input_dofs)
    ada_teleop.ExecuteDirectTeleop()


#!/usr/bin/env python

"""
marginal_distributions_temp.py
"""
import numpy as np
import numpy.ma as ma
import matplotlib.pyplot as plt
import sys
import _pickle as cPickle
from matplotlib import rcParams
rcParams['legend.fontsize']='small'

cpath_errorstats = '/Volumes/Backup Plus/python/fcst_stats/'

clead = sys.argv[1]

# Reference std
stdrefs = {"Jan-Feb-Mar":1.0,   "Apr-May-Jun":1.0,\
    "Jul-Aug-Sep":1.0, "Oct-Nov-Dec":1.0}

models = ['raw', 'dav', 'qmap', 'mos', 'mvmos']

# --- read land-surface mask, lons, lats

infilename = 'lsmask_0p5.cPick'
print (infilename)
inf = open(infilename, 'rb')
lsmask = cPickle.load(inf)
lats = cPickle.load(inf)
lons = cPickle.load(inf)
inf.close()

# ---- populate the histogram of correlations, std devs

xstd = np.arange(0.5,1.501,0.01)
xcorr = np.arange(0.0,1.001,0.01)
nxstd = len(xstd)
nxcorr = len(xcorr)
corr_winter_hist = ma.zeros((nxcorr,5), dtype=np.float64)
corr_spring_hist = ma.zeros((nxcorr,5), dtype=np.float64)
corr_summer_hist = ma.zeros((nxcorr,5), dtype=np.float64)
corr_fall_hist   = ma.zeros((nxcorr,5), dtype=np.float64)
std_winter_hist = ma.zeros((nxstd,5), dtype=np.float64)
std_spring_hist = ma.zeros((nxstd,5), dtype=np.float64)
std_summer_hist = ma.zeros((nxstd,5), dtype=np.float64)
std_fall_hist   = ma.zeros((nxstd,5), dtype=np.float64)

for imodel, model in enumerate(models):

    # ---- read in the standardized anomaly and correlation from files
    #      prepared for Taylor diagram generation

    if model == 'raw':
        ftype = 'raw_forecast_errorstats_2019_'
        title = 'Raw\nmodel output\n+'+clead+' h'
        color = 'Red'
    elif model ==  'dav':
        ftype = 'decayavg_forecast_errorstats_2019_'
        title = 'Decaying-average\nbias correction\n+'+clead+' h'
        color = 'Peru'
    elif model == 'qmap':
        ftype = 'quantile_mapping_errorstats_2019_'
        title = 'Quantile\nmapping\n+'+clead+' h'
        color = 'Blue'
    elif model == 'mos':
        ftype = 'MOS_GEFSv12_2019_'
        title = 'Univariate\nMOS\n+'+clead+' h'
        color = 'Green'
    elif model == 'mvmos':
        ftype = 'MOS_mvr_GEFSv12_2019_'
        title = 'Multi-variate\nMOS\n+'+clead+' h'
        color = 'Gray'

    statsfile = cpath_errorstats + ftype + clead+'h_taylor.cPick'
    inf = open(statsfile, 'rb')
    corr_winter = cPickle.load(inf)
    corr_spring = cPickle.load(inf)
    corr_summer = cPickle.load(inf)
    corr_fall = cPickle.load(inf)
    std_normalized_winter = cPickle.load(inf)
    std_normalized_spring = cPickle.load(inf)
    std_normalized_summer = cPickle.load(inf)
    std_normalized_fall = cPickle.load(inf)
    std_obs_winter = cPickle.load(inf)
    std_obs_spring = cPickle.load(inf)
    std_obs_summer = cPickle.load(inf)
    std_obs_fall = cPickle.load(inf)
    inf.close()
    nlats, nlons = np.shape(corr_fall)

    for jy in range(nlats):
        for ix in range(nlons):
            if lsmask[jy,ix] == 1:
                corr_idx = np.int(corr_winter[jy,ix]*100.0)
                corr_winter_hist[corr_idx,imodel] = corr_winter_hist[corr_idx,imodel] + 1.0
                corr_idx = np.int(corr_spring[jy,ix]*100.0)
                corr_spring_hist[corr_idx,imodel] = corr_spring_hist[corr_idx,imodel] + 1.0
                corr_idx = np.int(corr_summer[jy,ix]*100.0)
                corr_summer_hist[corr_idx,imodel] = corr_summer_hist[corr_idx,imodel] + 1.0
                corr_idx = np.int(corr_winter[jy,ix]*100.0)
                corr_fall_hist[corr_idx,imodel] = corr_fall_hist[corr_idx,imodel] + 1.0

                std_idx = np.amin([nxstd-1, np.int(std_normalized_winter[jy,ix]*100.0 - 50.0)])
                std_winter_hist[std_idx,imodel] = std_winter_hist[std_idx,imodel] + 1.0
                std_idx = np.amin([nxstd-1, np.int(std_normalized_spring[jy,ix]*100.0 - 50.0)])
                std_spring_hist[std_idx,imodel] = std_spring_hist[std_idx,imodel] + 1.0
                std_idx = np.amin([nxstd-1, np.int(std_normalized_summer[jy,ix]*100.0 - 50.0)])
                std_summer_hist[std_idx,imodel] = std_summer_hist[std_idx,imodel] + 1.0
                std_idx = np.amin([nxstd-1, np.int(std_normalized_fall[jy,ix]*100.0 - 50.0)])
                std_fall_hist[std_idx,imodel] = std_fall_hist[std_idx,imodel] + 1.0

fig1 = plt.figure(figsize=(8.5,8))
fig1.suptitle('Histograms of forecast vs. analyzed correlation, +'+clead+' h', size='xx-large')

# ---- plot the histograms in fig1

for iseason, season in enumerate(['Jan-Feb-Mar','Apr-May-Jun','Jul-Aug-Sep','Oct-Nov-Dec']):
    for imodel, model in enumerate(models):

        if season == 'Jan-Feb-Mar':
            corrcoef = corr_winter_hist[:,imodel]
            cletter = '(a) '
            axlocs = [0.08,0.54,0.4,0.36]
        elif season == 'Apr-May-Jun':
            corrcoef = corr_spring_hist[:,imodel]
            cletter = '(b) '
            axlocs = [0.57,0.54,0.4,0.36]
        elif season == 'Jul-Aug-Sep':
            corrcoef = corr_summer_hist[:,imodel]
            cletter = '(c) '
            axlocs = [0.08,0.06,0.4,0.36]
        elif season == 'Oct-Nov-Dec':
            corrcoef = corr_fall_hist[:,imodel]
            cletter = '(d) '
            axlocs = [0.57,0.06,0.4,0.36]
            print ('iseason, imodel, corr = ', iseason, imodel, corrcoef)

        if model == 'raw':
            ftype = 'raw_forecast_errorstats_2019_'
            label = 'Raw model output'
            color = 'Red'
            linestyle = '--'
        elif model ==  'dav':
            ftype = 'decayavg_forecast_errorstats_2019_'
            label = 'Decaying-average'
            color = 'Peru'
            linestyle = '-'
        elif model == 'qmap':
            ftype = 'quantile_mapping_errorstats_2019_'
            label = 'Quantile mapping'
            color = 'Blue'
            linestyle = '-.'
        elif model == 'mos':
            ftype = 'MOS_GEFSv12_2019_'
            label = 'Univariate MOS'
            color = 'Green'
            linestyle = '-'
        elif model == 'mvmos':
            ftype = 'MOS_mvr_GEFSv12_2019_'
            label = 'Multi-variate MOS'
            color = 'Gray'
            linestyle = ':'

        # --- add to correlation plots

        corrcoef = corrcoef / np.sum(corrcoef)
        corrcoef = ma.masked_where(corrcoef == 0.0, corrcoef)
        if imodel == 0: ax1 = fig1.add_axes(axlocs)
        if imodel == 0: ax1.set_title(cletter+season, fontsize=15)
        if season == 'Jul-Aug-Sep':
            ax1.plot(xcorr, corrcoef, color=color,linestyle=linestyle,lw=1.5,label=label)
        else:
            ax1.plot(xcorr, corrcoef, color=color,linestyle=linestyle,lw=1.5)
        if imodel == 0: ax1.set_xlim(0.7,1.0)
        if imodel == 0: ax1.set_xticks([0.7,0.75,0.8,0.85,0.9,0.95,1.0])
        #if imodel == 0: ax1.set_ylim(0.0,0.2)
        if imodel == 0: ax1.set_xlabel('Correlation',fontsize=12)
        if imodel == 0: ax1.set_ylabel('Relative frequency',fontsize=12)
        #if imodel == 0: ax1.set_yticks([0.001,0.005, 0.01,0.05, 0.1, 0.5])
        #if imodel == 0: ax1.set_yticks([0.0,0.04,0.08,0.12,0.16,0.2])
        if imodel == 0: ax1.grid(True, lw=0.25)

        if season == 'Jul-Aug-Sep' and model == 'mvmos':
            ax1.legend(loc=0)

# ---- set plot titles and write to files

outfile1 = 'corr_histogram_4panel_'+clead+'h.pdf'
print ('saving to ', outfile1)
plt.savefig(outfile1)

fig2 = plt.figure(figsize=(8.5,8))
fig2.suptitle('Histograms of forecast vs. analyzed normalized standard deviation, +'+clead+' h', size='x-large')

# ---- plot the histograms in fig1

for iseason, season in enumerate(['Jan-Feb-Mar','Apr-May-Jun','Jul-Aug-Sep','Oct-Nov-Dec']):

    for imodel, model in enumerate(models):

        if season == 'Jan-Feb-Mar':
            stddev = std_winter_hist[:,imodel]
            cletter = '(a) '
            axlocs = [0.08,0.54,0.4,0.37]
        elif season == 'Apr-May-Jun':
            stddev = std_spring_hist[:,imodel]
            cletter = '(b) '
            axlocs = [0.58,0.54,0.4,0.37]
        elif season == 'Jul-Aug-Sep':
            stddev = std_summer_hist[:,imodel]
            cletter = '(c) '
            axlocs = [0.08,0.06,0.4,0.37]
        elif season == 'Oct-Nov-Dec':
            stddev = std_fall_hist[:,imodel]
            cletter = '(d) '
            axlocs = [0.58,0.06,0.4,0.37]

        if model == 'raw':
            ftype = 'raw_forecast_errorstats_2019_'
            label = 'Raw model output +'+clead+' h'
            color = 'Red'
            linestyle = '--'
        elif model ==  'dav':
            ftype = 'decayavg_forecast_errorstats_2019_'
            label = 'Decaying-average'
            color = 'Peru'
            linestyle = '-'
        elif model == 'qmap':
            ftype = 'quantile_mapping_errorstats_2019_'
            label = 'Quantile mapping '+clead+' h'
            color = 'Blue'
            linestyle = '-.'
        elif model == 'mos':
            ftype = 'MOS_GEFSv12_2019_'
            label = 'Univariate MOS '+clead+' h'
            color = 'Green'
            linestyle = '-'
        elif model == 'mvmos':
            ftype = 'MOS_mvr_GEFSv12_2019_'
            label = 'Multi-variate MOS '+clead+' h'
            color = 'Gray'
            linestyle = ':'

        # --- add to stddev plots

        stddev = stddev / np.sum(stddev)
        stddev = ma.masked_where(stddev == 0.0, stddev)
        if imodel == 0: ax2 = fig2.add_axes(axlocs)
        if imodel == 0: ax2.set_title(cletter+season, fontsize=14)
        if season == 'Jan-Feb-Mar':
            ax2.semilogy(xstd, stddev, color=color,linestyle=linestyle,lw=1.5,label=label)
        else:
            ax2.semilogy(xstd, stddev, color=color,linestyle=linestyle,lw=1.5)
        if imodel == 0: ax2.set_xlim(0.5,1.5)
        if imodel == 0: ax2.set_ylim(0.001,0.1)
        if imodel == 0: ax2.set_xlabel(r'$\sigma_f$ / $\sigma_a$',fontsize=12)
        if imodel == 0: ax2.set_ylabel('Relative frequency',fontsize=12)
        if imodel == 0: ax2.set_xticks([0.5,0.75,1.0,1.25,1.5])
        if imodel == 0: ax2.set_yticks([0.001,0.005, 0.01,0.05, 0.1])
        if imodel == 0: ax2.grid(True, lw=0.25)

        if season == 'Jan-Feb-Mar' and model == 'mvmos':
            ax2.legend(loc=3)

outfile2 = 'stddev_histogram_4panel_'+clead+'h.pdf'
print ('saving to ', outfile2)
plt.savefig(outfile2)


# Equal Sum Arrays With Minimum Number of Operations
"""
You are given two arrays of integers nums1 and nums2, possibly of different lengths. The values in the arrays are between 1 and 6, inclusive.

In one operation, you can change any integer's value in any of the arrays to any value between 1 and 6, inclusive.

Return the minimum number of operations required to make the sum of values in nums1 equal to the sum of values in nums2. Return -1 if it is not possible to make the sum of the two arrays equal.
"""
class Solution:
    def minOperations(self, nums1: List[int], nums2: List[int]) -> int:

		# step1. determine the larger array and the smaller array, and get the difference on sum
        sum1 = sum(nums1)
        sum2 = sum(nums2)

        if sum1==sum2:
            return 0
        elif sum1>sum2:
            larger_sum_nums = nums1
            smaller_sum_nums = nums2
        else:
            larger_sum_nums = nums2
            smaller_sum_nums = nums1

        sum_diff = abs(sum1-sum2)

        # step2. calculate the max "gain" at each position (how much difference we can reduce if operating on that position)
        gains_in_larger_array = [num-1 for num in larger_sum_nums]
        gains_in_smaller_array = [6-num for num in smaller_sum_nums]

		# step3. sort the "gain" and check the least number of steps to reduce the difference to 0
        gains = gains_in_larger_array + gains_in_smaller_array
        gains.sort(reverse = True)

        count = 0
        target_diff = sum_diff

        for i in range(len(gains)):
            target_diff -= gains[i]
            count += 1

            if target_diff <= 0:
                return count

		# return -1 if the difference still cannot be reduced to 0 even after operating on all positions
        return -1


import json
import requests

new_paste_url = 'https://hastebin.com/documents'

def paste(paste_content):
    try:
        response = requests.post(new_paste_url, data=paste_content, timeout=1000).json()
    except json.decoder.JSONDecodeError:
        return {'status': 0, 'message': 'Application Error'}
    if response.get('key'):
        return {'status': 1, 'result': response['key']}
    else:
        return {'status': 0, 'message': response['message']}


import copy
import logging
import numpy as np
import torch
from detectron2.structures import BoxMode, Instances, Boxes, PolygonMasks, polygons_to_bitmask, BitMasks
from pycocotools.mask import encode, decode
import albumentations as A

import detectron2.data.transforms as T
from detectron2.data import detection_utils

from utils.bbox_conversion import pascal_voc_bboxes_to_albumentations, albumentations_bboxes_to_pascal_voc

"""
This file contains the mapping with Albumentations augmentation.
"""

class AlbumentationsMapper:
    """
    A callable which takes a dataset dict in Detectron2 Dataset format,
    apply augmentations, and map it into a format used by the model.

    The callable does the following:
    1. Read the image from "file_name"
    2. Use detectron tools to resize and crop the image (ResizeShortestEdge and RandomCrop)
    3. Apply augmentations/transforms to the image and annotations with Albumentations
    4. Prepare data and annotations and convert them to Tensor and :class:`Instances`
    """

    def __init__(self, cfg, is_train: bool = True):
        """
        Args:
            cfg: configuration
            is_train: whether it's used in training or inference
        """
        self.is_train = is_train
        self.instance_mask_format = cfg.INPUT.MASK_FORMAT

        # Define transforms (Detectron2)
        transforms = detection_utils.build_augmentation(cfg, is_train)
        if cfg.INPUT.CROP.ENABLED and is_train:
            transforms.insert(0, T.RandomCrop(cfg.INPUT.CROP.TYPE, cfg.INPUT.CROP.SIZE))
            self.recompute_boxes = cfg.MODEL.MASK_ON
        else:
            self.recompute_boxes = False
        self.transforms = T.AugmentationList(transforms)

        logger = logging.getLogger("detectron2")
        mode = "training" if is_train else "inference"
        logger.info("############# TRANSFORMS #################")
        logger.info(f"Transforms used in {mode}:")
        for transform in transforms:
            logger.info(transform)
        logger.info("##############################################")

        # Define augmentations (Albumentations) (only training)
        if is_train:
            augmentations = get_augmentations(cfg)
            self.augmentations = A.Compose(
                augmentations,
                bbox_params=A.BboxParams(format='albumentations', label_fields=['class_labels', 'bbox_ids']))
            logger.info("############# AUGMENTATIONS #################")
            logger.info("Augmentations used in training:")
            for aug in augmentations:
                logger.info(aug)
            logger.info("##############################################")

    def __call__(self, dataset_dict):
        """
        Args:
            dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format.

        Returns:
            dict: a format that builtin models in detectron2 accept
        """
        dataset_dict = copy.deepcopy(dataset_dict)

        image = detection_utils.read_image(dataset_dict["file_name"], format="BGR")
        detection_utils.check_image_size(dataset_dict, image)

        # Apply Albumentations during training
        if self.is_train:
            # BGR to RGB format required by albumentations
            image = image[:, :, ::-1]

            bboxes = [anno["bbox"] for anno in dataset_dict["annotations"]]
            bbox_mode = dataset_dict["annotations"][0]["bbox_mode"]
            masks = [decode(anno["segmentation"]) for anno in dataset_dict["annotations"]]
            class_labels = [anno["category_id"] for anno in dataset_dict["annotations"]]

            # Convert bboxes from the pascal_voc to the albumentations format
            bboxes = pascal_voc_bboxes_to_albumentations(bboxes, height=image.shape[0], width=image.shape[1])

            # if self.is_train:
            # Apply augmentations
            augmented = self.augmentations(
                image=image,
                bboxes=bboxes,
                masks=masks,
                class_labels=class_labels,
                bbox_ids=np.arange(len(bboxes))
            )
            image = augmented["image"]
            bboxes = augmented["bboxes"]
            masks = augmented["masks"]
            class_labels = augmented["class_labels"]
            bbox_ids = augmented["bbox_ids"]

            # Filter the masks that don't have a corresponding bbox anymore
            # and convert uint8 masks of 0s and 1s into dicts in COCO’s compressed RLE format
            masks = [encode(np.asarray(masks[i], order="F")) for i in bbox_ids]

            # Convert bboxes from the albumentations format to the pascal_voc format
            bboxes = albumentations_bboxes_to_pascal_voc(bboxes, height=image.shape[0], width=image.shape[1])

            assert len(bboxes) == len(class_labels), \
                "The number of bounding boxes should be equal to the number of class labels"
            assert len(bboxes) == len(masks), \
                "The number of bounding boxes should be equal to the number of masks"

            dataset_dict["annotations"] = [
                {
                    "bbox": bboxes[i],
                    "bbox_mode": bbox_mode,
                    "segmentation": masks[i],
                    "category_id": class_labels[i]
                }
                for i in range(len(bboxes))
            ]

            # RGB to BGR required by the model
            image = image[:, :, ::-1]

        # Apply transforms
        aug_input = T.AugInput(image)
        transforms = self.transforms(aug_input)
        image = aug_input.image

        annotations = [obj for obj in dataset_dict.pop("annotations") if obj.get("iscrowd", 0) == 0]
        # Keep original annotations for later evaluation
        dataset_dict["annotations"] = copy.deepcopy(annotations)

        # Transform annotations
        annotations = [detection_utils.transform_instance_annotations(obj, transforms, image.shape[:2])
                       for obj in annotations]
        # Convert annotations to instances
        instances = annotations_to_instances(annotations, image.shape[:2])

        # If cropping is applied, the bounding box may no longer tightly bound the object
        # if self.recompute_boxes:
        #     instances.gt_boxes = instances.gt_masks.get_bounding_boxes()
        dataset_dict["instances"] = detection_utils.filter_empty_instances(instances)

        # Convert H,W,C image to C,H,W tensor
        dataset_dict["image"] = torch.as_tensor(np.ascontiguousarray(image.transpose((2, 0, 1))))

        return dataset_dict

def annotations_to_instances(annos, image_size):

    boxes = [BoxMode.convert(obj["bbox"], obj["bbox_mode"], BoxMode.XYXY_ABS) for obj in annos]
    target = Instances(image_size)
    target.gt_boxes = Boxes(boxes)

    classes = [int(obj["category_id"]) for obj in annos]
    classes = torch.tensor(classes, dtype=torch.int64)
    target.gt_classes = classes

    if "segmentation" in annos[0]:
        masks = [obj["segmentation"] for obj in annos]
        masks = BitMasks(torch.stack([torch.from_numpy(np.copy(x)) for x in masks]))
        target.gt_masks = masks

    return target

def pixel_dropout(image, p_min, p_max, **kwargs):
    """
    Set a fraction of pixels in images to zero.
    :param image:
    :param p_min: minimum pixel dropout probability
    :param p_max: maximum pixel dropout probability
    :param kwargs:
    :return: augmented image
    """
    assert p_min <= p_max, (
        f"Expected p_min <= p_max. Got {p_min:.4f} and {p_max:.4f}.")
    assert 0 <= p_min <= 1.0 and 0 <= p_max <= 1.0, (
        f"Expected values in the interval [0.0, 1.0], got {p_min:.4f} and {p_max:.4f}.")

    height = image.shape[0]
    width = image.shape[1]
    # Dropout probability
    p = np.random.uniform(p_min, p_max)
    # Pixels to dropout
    dropouts = np.random.choice([0, 1], size=(height, width), p=[p, 1.0 - p]).astype('uint8')
    image = image * dropouts[:, :, np.newaxis]
    return image

def get_augmentations(cfg):
    augmentations = []

    # Pixel Dropout
    if cfg.ALBUMENTATIONS.PIXEL_DROPOUT.ENABLED:
        augmentations.append(A.Lambda(
            name="pixel_dropout",
            image=lambda image, **kwargs: pixel_dropout(
                image,
                p_min=cfg.ALBUMENTATIONS.PIXEL_DROPOUT.DROPOUT_MIN,
                p_max=cfg.ALBUMENTATIONS.PIXEL_DROPOUT.DROPOUT_MAX),
            p=0.5))

    # Gaussian Noise
    if cfg.ALBUMENTATIONS.GAUSSIAN_NOISE.ENABLED:
        augmentations.append(A.GaussNoise())

    # Random Brightness Contrast
    if cfg.ALBUMENTATIONS.RANDOM_BRIGHTNESS_CONTRAST.ENABLED:
        augmentations.append(A.RandomBrightnessContrast())

    # Gaussian Blur
    if cfg.ALBUMENTATIONS.GAUSSIAN_BLUR.ENABLED:
        augmentations.append(A.GaussianBlur())

    return augmentations


import time
import collections
import settings
import re
import codecs

def is_english(s):
    try:
        s.encode('ascii')
        return True
    except UnicodeEncodeError:
        return False

def trim_str(s, stopwords):
    s = s.replace('\n', ' ').replace('\r', ' ')
    s = re.sub(r"http\S+", "", s)
    s = re.sub(r"\d+", "", s)
    s = re.sub(r'[`\-=~!@#$%^&*()_+\[\]{};\'\\:"|<,./>?]', ' ', s)
    wordlist = [w for w in s.lower().split() if
                (w not in stopwords) & (len(w) > 1) & (len(w) < 16)]
    return wordlist

def fetch_and_save_feature(desc_file, feature_file, connection, extends_dict, selector):
    """
    从数据库features_t中读取特征和文本并形成文件
    :param desc_file:
    :param feature_file:
    :param connection:
    :param extends_dict:
    :param selector:
    :return:
    """
    type_dict = {}
    next_type = 0
    cursor = connection.cursor()
    cursor.execute("select feedid, streamid, doc, location_type, created from features_t", connection)

    with codecs.open(settings.DEFINITIONS_ROOT + '/resource/stopwords.txt', 'r') as f:
        stoplist = set(f.read().split())

    epoch = int(time.mktime(time.strptime('2008.01.01', '%Y.%m.%d')))
    result_dict = {}
    regular_word = collections.defaultdict(int)
    for feed_id, streamid, doc, l_type, create_time in cursor.fetchall():

        # 生成文当前对doc进行补全
        if extends_dict is not None and (feed_id, streamid) in extends_dict:
            doc = doc + '. ' + extends_dict[(feed_id, streamid)]

        # 时间
        time_from = time.mktime(create_time.timetuple())
        lb_time = str(int((time_from - epoch) / (3600 * 24 * 30 * 6)))

        # 地点
        if l_type not in type_dict:
            type_dict[l_type] = next_type
            print('l_type=' + l_type + '\ttype_id=' + str(next_type))
            next_type += 1

        # 词
        wordlist = trim_str(doc, stoplist) if is_english(doc) else []
        for w in wordlist:
            regular_word[w] += 1

        result_dict[feed_id] = (lb_time, type_dict[l_type], wordlist)

    # 检查词频是否满足要求，分别存放到text.in和features.in中
    feed_id_list = []
    with codecs.open(desc_file, 'w') as ft:
        with codecs.open(feature_file, 'w') as ff:
            for feed_id, (lb_time, l_type, wordlist) in result_dict.items():
                final_words = " ".join([w for w in wordlist if regular_word[w] > 2])
                if len(final_words) > 1:
                    feed_id_list.append(feed_id)
                    ft.write(final_words + "\n")
                    if selector is not None:
                        if selector == 0:
                            ff.write("f=0\n")  # 默认feature
                        elif selector == 1:
                            ff.write("f_time=" + lb_time + "\n")
                        elif selector == 2:
                            ff.write("f_loc_" + str(l_type) + "\n")
                        elif selector == 3:
                            ff.write("f_loc_" + str(l_type) + " f_time=" + lb_time + "\n")
                    else:
                        ff.write("f_loc_" + str(l_type) + " f_time=" + lb_time + "\n")

    regular_set = set([w for w, c in regular_word.items() if c > 2])  # word set
    return feed_id_list, regular_set


import os

def read_ids(ids_file):
    file = open(ids_file, 'r', encoding='utf8')
    ids_list = list()
    for line in file.readlines():
        params = line.strip('\n').split('\t')
        assert len(params) == 2
        id = int(params[0])
        ids_list.append(id)
    return ids_list

def embedding2file(embeddings, embeddings_out_file):
    print("Embedding:", embeddings.shape)
    fw = open(embeddings_out_file, 'w', encoding='utf8')
    for i in range(embeddings.shape[0]):
        line = ''
        for j in range(embeddings.shape[1]):
            line = line + str(embeddings[i, j]) + '\t'
        fw.write(line.strip() + '\n')
    fw.close()

def print_time(t):
    print('time:{:.3f} s'.format(t))

def radio_2file(radio, folder):
    path = folder + str(radio).replace('.', '_')
    if not os.path.exists(path):
        os.makedirs(path)
    return path + '/'

def read_pairs(file_path):
    file = open(file_path, 'r', encoding='utf8')
    pairs = list()
    for line in file.readlines():
        params = line.strip('\n').split('\t')
        assert len(params) == 2
        pairs.append((params[0], params[1]))
    file.close()
    return pairs

def read_pair_ids(file_path):
    file = open(file_path, 'r', encoding='utf8')
    pairs = list()
    for line in file.readlines():
        params = line.strip('\n').split('\t')
        assert len(params) == 2
        pairs.append((int(params[0]), int(params[1])))
    file.close()
    return pairs

def pair_2set(pairs):
    s1, s2 = set(), set()
    for pair in pairs:
        s1.add(pair[0])
        s2.add(pair[1])
    return s1, s2

def read_triples_ids(file_path):
    triples = list()
    file = open(file_path, 'r', encoding='utf8')
    for line in file.readlines():
        params = line.strip('\n').split('\t')
        assert len(params) == 3
        h = int(params[0])
        r = int(params[1])
        t = int(params[2])
        triples.append((h, r, t))
    return triples

def read_ref(file_path):
    refs = list()
    reft = list()
    file = open(file_path, 'r', encoding='utf8')
    for line in file.readlines():
        params = line.strip('\n').split('\t')
        assert len(params) == 2
        e1 = int(params[0])
        e2 = int(params[1])
        refs.append(e1)
        reft.append(e2)
    assert len(refs) == len(reft)
    return refs, reft

def read_ents_by_order(ids_file):
    file = open(ids_file, 'r', encoding='utf8')
    ids_list = list()
    for line in file.readlines():
        params = line.strip('\n').split('\t')
        assert len(params) == 2
        ids_list.append(int(params[0]))
    return ids_list

def pair_2_rev_dict(pairs):
    d = dict()
    for pair in pairs:
        if pair[1] not in d:
            d[pair[1]] = pair[0]
        else:
            print("Error")
    return d

def pair_2int_set(pairs):
    s1, s2 = set(), set()
    for pair in pairs:
        s1.add(int(pair[0]))
        s2.add(int(pair[1]))
    return s1, s2

def div_list(ls, n):
    ls_len = len(ls)
    if n <= 0 or 0 == ls_len:
        return []
    if n > ls_len:
        return []
    elif n == ls_len:
        return [[i] for i in ls]
    else:
        j = ls_len // n
        k = ls_len % n
        ls_return = []
        for i in range(0, (n - 1) * j, j):
            ls_return.append(ls[i:i + j])
        ls_return.append(ls[(n - 1) * j:])
        return ls_return


# imports
import os
from rich import print
import time
from random import seed
from random import randint

# color class

# seed random number generator
seed(1000)

# main
loop = True

while loop == True:
    ask = int(input("Enter first number to generate (type 'q' to quit): "))
    ask2 = int(input('Enter second number to generate: '))
    value = randint(ask, ask2)
    print(value)
    time.sleep(1)
    if ask == 'q':
        loop == False
    else:
        loop == True


# 다음과 같이 두 개의 숫자 N, M이 주어질 때, N의 M 거듭제곱 값을 구하는 프로그램을 재귀호출을 이용하여 구현해 보아라.
# 2 5 = 2 X 2 X 2 X 2 X 2 = 32
# 3 6 = 3 X 3 X 3 X 3 X 3 X 3 = 729

# [제약 사항]
# 총 10개의 테스트 케이스가 주어진다.
# 결과 값은 Integer 범위를 넘어가지 않는다.

# [입력]
# 각 테스트 케이스의 첫 줄에는 테스트 케이스의 번호가 주어지고 그 다음 줄에는 두 개의 숫자가 주어진다.

# [출력]
# #부호와 함께 테스트 케이스의 번호를 출력하고, 공백 문자 후 테스트 케이스에 대한 답을 출력한다.

import sys
sys.stdin = open('230213 4일차-거듭제곱.txt', 'r')

# [문제풀이]

# T = int(input())
# for testcase in range(1, T+1):
#     N, M = map(int, input().split())                    # 숫자 N, 거듭제곱횟수 M
#     K = N
#     for i in range(M-1):
#         N = N*K
#     print(N)

# T = 10
# for testcase in range(1, T+1):
#     A = int(input())
#     N, M = map(int, input().split())                    # 숫자 N, 거듭제곱횟수 M
#     def multimulti(N, M):
#         if M == 0:
#             return 1
#         elif M == 1:
#             return N
#         else:
#             return N*multimulti(N, M-1)
#     print(f'#{testcase } {multimulti(N,M)}')

T = 10
for testcase in range(1, T+1):
    A = int(input())
    N, M = map(int, input().split())                    # 숫자 N, 거듭제곱횟수 M
    def multimulti(N, M):
        if M == 2:
            return N*N
        else:
            return N*multimulti(N, M-1)
    print(f'#{testcase } {multimulti(N,M)}')


from __future__ import absolute_import

import io
import json
import os

from typing import (
    cast,
    TYPE_CHECKING,
)

from urllib import parse

from . import (
    AbstractStatefulFetcher,
    DocumentedProtocolFetcher,
    FetcherException,
    ProtocolFetcherReturn,
)
from .http import fetchClassicURL

from ..common import (
    ContentKind,
    URIWithMetadata,
)

if TYPE_CHECKING:
    from typing import (
        Mapping,
        MutableSequence,
        Optional,
        Sequence,
    )

    from ..common import (
        AbsPath,
        SecurityContextConfig,
        URIType,
    )

INTERNAL_TRS_SCHEME_PREFIX = "wfexs.trs.files"
TRS_SCHEME_PREFIX = "trs"

TRS_FILES_SUFFIX = "/files"
TRS_DESCRIPTOR_INFIX = "/descriptor/"

def fetchTRSFiles(
    remote_file: "URIType",
    cachedFilename: "AbsPath",
    secContext: "Optional[SecurityContextConfig]" = None,
) -> "ProtocolFetcherReturn":
    """
    Method to download contents from TRS files related to a tool

    :param remote_file:
    :param cachedFilename: Destination filename for the fetched content
    :param secContext: The security context containing the credentials
    """

    parsedInputURL = parse.urlparse(remote_file)
    path_steps: "Sequence[str]" = parsedInputURL.path.split("/")
    embedded_remote_file = parsedInputURL.path

    metadata_array: "MutableSequence[URIWithMetadata]" = []
    if parsedInputURL.scheme == INTERNAL_TRS_SCHEME_PREFIX:
        # TODO: Improve this code
        if not embedded_remote_file.endswith(TRS_FILES_SUFFIX):
            metadata_url = cast("URIType", embedded_remote_file + TRS_FILES_SUFFIX)
            descriptor_base_url = embedded_remote_file + TRS_DESCRIPTOR_INFIX
        else:
            metadata_url = cast("URIType", embedded_remote_file)
            descriptor_base_url = (
                embedded_remote_file[0 : -len(TRS_FILES_SUFFIX)] + TRS_DESCRIPTOR_INFIX
            )
    elif parsedInputURL.scheme == TRS_SCHEME_PREFIX:
        # TRS official scheme
        if len(path_steps) < 3 or path_steps[0] != "":
            raise FetcherException(
                f"Ill-formed TRS CURIE {remote_file}. It should be in the format of {TRS_SCHEME_PREFIX}://id/version or {TRS_SCHEME_PREFIX}://prefix-with-slashes/id/version"
            )

        version_steps = cast("MutableSequence[str]", path_steps[0:-2])
        version_steps.extend(
            ["ga4gh", "trs", "v2", "tools", path_steps[-2], "versions", path_steps[-1]]
        )
        version_metadata_url = cast(
            "URIType",
            parse.urlunparse(
                parse.ParseResult(
                    scheme="https",
                    netloc=parsedInputURL.netloc,
                    path="/".join(version_steps),
                    params="",
                    query="",
                    fragment="",
                )
            ),
        )
        version_meta = {
            "fetched": version_metadata_url,
            "payload": None,
        }
        metadata_array.append(URIWithMetadata(remote_file, version_meta))
        try:
            metaio = io.BytesIO()
            _, metametaio, _ = fetchClassicURL(version_metadata_url, metaio)
            version_metadata = json.loads(metaio.getvalue().decode("utf-8"))
            version_meta["payload"] = version_metadata
            metadata_array.extend(metametaio)

        except FetcherException as fe:
            raise FetcherException(
                f"Error fetching or processing TRS version metadata for {remote_file} : {fe.code} {fe.reason}"
            ) from fe

        # At last, we can finish building the URL
        new_path_steps = [
            *version_steps,
            version_metadata["descriptor_type"][0],
            "files",
        ]

        metadata_url = cast(
            "URIType",
            parse.urlunparse(
                parse.ParseResult(
                    scheme="https",
                    netloc=parsedInputURL.netloc,
                    path="/".join(new_path_steps),
                    params="",
                    query="",
                    fragment="",
                )
            ),
        )

        descriptor_steps = [
            *version_steps,
            version_metadata["descriptor_type"][0],
            "descriptor",
        ]
        descriptor_base_url = parse.urlunparse(
            parse.ParseResult(
                scheme="https",
                netloc=parsedInputURL.netloc,
                path="/".join(descriptor_steps) + "/",
                params="",
                query="",
                fragment="",
            )
        )

    topMeta = {
        "fetched": metadata_url,
        "payload": None,
        "workflow_entrypoint": None,
        "remote_workflow_entrypoint": None,
    }
    metadata_array = [URIWithMetadata(remote_file, topMeta)]
    try:
        metaio = io.BytesIO()
        _, metametaio, _ = fetchClassicURL(metadata_url, metaio)
        metadata = json.loads(metaio.getvalue().decode("utf-8"))
        topMeta["payload"] = metadata
        metadata_array.extend(metametaio)
    except FetcherException as fe:
        raise FetcherException(
            "Error fetching or processing TRS files metadata for {} : {} {}".format(
                remote_file, fe.code, fe.reason
            )
        ) from fe

    os.makedirs(cachedFilename, exist_ok=True)
    absdirs = set()
    emptyWorkflow = True
    # First pass, identify primary descriptor / workflow entrypoint
    # and learn whether the destination paths should be sanitized
    deepest_file_rel = 1
    for file_desc in metadata:
        file_rel_path = file_desc.get("path")
        if file_rel_path is not None:
            frp_parsed = parse.urlparse(file_rel_path)
            if frp_parsed.scheme in ("http", "https", "ftp"):
                # An absolute URL, like in the case of DDBJ TRS implementation
                # A mixure of resource might be catastrophic, the code is doing
                # its best effort
                file_rel_path = os.path.join(frp_parsed.netloc, frp_parsed.params)

            # BEWARE! The relpath could contain references to parent directories
            # escaping from the URL to be built and from the download "sandbox"
            # Avoid absolute paths corner case before splitting
            file_rel_path_steps = file_rel_path.lstrip("/").split("/")

            file_rel_depth = (
                len(file_rel_path_steps)
                - file_rel_path_steps.count(".")
                - file_rel_path_steps.count("")
                - 2 * file_rel_path_steps.count("..")
            )
            if file_rel_depth < deepest_file_rel:
                deepest_file_rel = file_rel_depth

    # We have to create anonymous directories to avoid leaving the download "sandbox"
    abs_download_dir = cachedFilename
    if deepest_file_rel < 1:
        for depth in range(deepest_file_rel, 1):
            abs_download_dir = cast(
                "AbsPath", os.path.join(abs_download_dir, f"unnamed{depth}")
            )

    # Second pass, fetching the contents, sanitizing the destination paths
    for file_desc in metadata:
        file_rel_path = file_desc.get("path")
        if file_rel_path is not None:
            emptyWorkflow = False

            # BEWARE! The relpath could contain references to parent directories
            # escaping from the URL to be built and from the download "sandbox"
            frp_parsed = parse.urlparse(file_rel_path)
            is_abs_url = frp_parsed.scheme in ("http", "https", "ftp")
            if is_abs_url:
                # An absolute URL, like in the case of DDBJ TRS implementation
                file_url = cast("URIType", file_rel_path)
                absfile = cast(
                    "AbsPath",
                    os.path.join(
                        abs_download_dir, frp_parsed.netloc, frp_parsed.path.lstrip("/")
                    ),
                )
            else:
                file_url = cast(
                    "URIType",
                    descriptor_base_url + parse.quote(file_rel_path, safe="/"),
                )
                absfile = cast(
                    "AbsPath", os.path.join(abs_download_dir, file_rel_path.lstrip("/"))
                )

            # Intermediate path creation
            absdir = os.path.dirname(absfile)
            if absdir not in absdirs:
                absdirs.add(absdir)
                os.makedirs(absdir, exist_ok=True)
            real_rel_path = os.path.relpath(os.path.normpath(absfile), cachedFilename)

            # When it is the primary descriptor, it is fetched twice
            if file_desc.get("file_type") == "PRIMARY_DESCRIPTOR":
                topMeta["workflow_entrypoint"] = cast("URIType", real_rel_path)
                if is_abs_url:
                    topMeta["remote_workflow_entrypoint"] = file_url
                else:
                    descriptorMeta = io.BytesIO()
                    _, metaprimary, _ = fetchClassicURL(file_url, descriptorMeta)
                    metadata_array.extend(metaprimary)

                    # This metadata can help a lot to get the workflow repo
                    metadataPD = json.loads(descriptorMeta.getvalue().decode("utf-8"))
                    topMeta["remote_workflow_entrypoint"] = metadataPD.get("url")

                    del descriptorMeta
                    del metadataPD

            accept_val = "*/*" if is_abs_url else "text/plain"
            # Getting the raw content
            _, metaelem, _ = fetchClassicURL(
                file_url, absfile, {"headers": {"Accept": accept_val}}
            )
            metadata_array.extend(metaelem)

    if emptyWorkflow:
        raise FetcherException(
            "Error processing TRS files for {} : no file was found.\n{}".format(
                remote_file, metadata
            )
        )

    return ProtocolFetcherReturn(
        kind_or_resolved=ContentKind.Directory,
        metadata_array=metadata_array,
    )

# These are schemes from identifiers.org
SCHEME_HANDLERS: "Mapping[str, DocumentedProtocolFetcher]" = {
    INTERNAL_TRS_SCHEME_PREFIX: DocumentedProtocolFetcher(
        fetcher=fetchTRSFiles,
        description="WfExS internal pseudo-scheme used to materialize files from pure TRS servers",
    ),
    TRS_SCHEME_PREFIX: DocumentedProtocolFetcher(
        fetcher=fetchTRSFiles,
        description="GA4GH TRS metadata is fetched using the APIs described at https://ga4gh.github.io/tool-registry-service-schemas/. Contents are downloaded delegating their associated URIs to other fetchers",
    ),
}


# System (Default)
import sys
#   Add access if it is not in the system path.
if '../' + 'src' not in sys.path:
    sys.path.append('../..')
# OS (Operating system interfaces)
import os
# Custom Lib.:
#   ../Lib/Parameters/Robot
import Lib.Parameters.Robot as Parameters
#   ../Lib/PyBullet/Core
import Lib.PyBullet.Core
#   ../Lib/Kinematics/Core
import Lib.Kinematics.Core as Kinematics

"""
Description:
    Initialization of constants.
"""
# Set the structure of the main parameters of the robot.
CONST_ROBOT_TYPE = Parameters.ABB_IRB_120_Str
# Locate the path to the project folder.
CONST_PROJECT_FOLDER = os.getcwd().split('PyBullet_Template_Industrial_Robotics')[0] + 'PyBullet_Template_Industrial_Robotics'
# The properties of the PyBullet environment.
#   Note:
#      ABB_IRB_14000_{L, R}_Str:
#       'External_Base': f'{CONST_PROJECT_FOLDER}/URDFs/Robots/ABB_IRB_14000_Base/ABB_IRB_14000_Base.urdf'
CONST_PYBULLET_ENV_PROPERTIES = {'Enable_GUI': True, 'fps': 100,
                                 'External_Base': None,
                                 'Camera': {'Yaw': 70.0, 'Pitch': -32.0, 'Distance':1.3,
                                            'Position': [0.05, -0.10, 0.06]}}

def main():
    """
    Description:
        A program that demonstrates simple trajectory generation in the PyBullet environment using a designed class.
    """

    # Initialization of the structure of the main parameters of the robot.
    Robot_Str = CONST_ROBOT_TYPE

    # Obtain the desired absolute position of the robot joints.
    theta = Robot_Str.Theta.Home

    # Initialization of the class to work with a robotic arm object in a PyBullet environment.
    PyBullet_Robot_Cls = Lib.PyBullet.Core.Robot_Cls(Robot_Str, f'{CONST_PROJECT_FOLDER}/URDFs/Robots/{Robot_Str.Name}/{Robot_Str.Name}.urdf',
                                                     CONST_PYBULLET_ENV_PROPERTIES)

    # Reset the absolute position of the robot joints to the 'Zero'.
    PyBullet_Robot_Cls.Reset('Zero')

    # The physical simulation is in progress.
    while PyBullet_Robot_Cls.is_connected == True:
        # Get the homogeneous transformation matrix of the robot end-effector.
        T = Kinematics.Forward_Kinematics(theta, 'Fast', Robot_Str)[1]

        # Add a viewpoint (+ sphere) with the correct transformation to the end-effector of the structure.
        PyBullet_Robot_Cls.Add_External_Object(f'{CONST_PROJECT_FOLDER}/URDFs/Primitives/Sphere/Sphere.urdf', T, [0.0, 1.0, 0.0, 0.25],
                                               0.025, True, False)
        PyBullet_Robot_Cls.Add_External_Object(f'{CONST_PROJECT_FOLDER}/URDFs/Viewpoint/Viewpoint.urdf', T, None,
                                               0.5, True, False)

        # Set the absolute position of the robot joints.
        in_position = PyBullet_Robot_Cls.Set_Absolute_Joint_Position(theta, 100.0, 0.0, 3.0)

        if in_position == True:
            # Change the desired absolute position of the robot joint.
            theta = Robot_Str.Theta.Home if (theta == Robot_Str.Theta.Zero).all() else Robot_Str.Theta.Zero

            # Remove all models from the PyBullet environment.
            PyBullet_Robot_Cls.Remove_All_External_Objects()
            in_position = False

    # Disconnect the created environment from a physical server.
    PyBullet_Robot_Cls.Disconnect()

if __name__ == '__main__':
    main()


from django.shortcuts import render
from django.core.mail import send_mail
from django.conf import settings
from django.contrib import messages

# Create your views here.
def contact(request):
    if request.method == "POST":

        message_name = request.POST.get('name')
        message_subject = request.POST.get('subject')
        message_email = request.POST.get('email')
        message_message = request.POST.get('message')

        data = {
            'name': message_name,
            'subject': message_subject,
            'email': message_email,
            'message': message_message
        }

        message = """
        From: {}
        New message: {}
        Email: {}
        """.format(data['name'], data['message'], data['email'])
        # send an email
        send_mail(data['subject'], message, '', ['mpt.craig@gmail.com'])
        messages.success(request, f'Your email was send Successfully!')

        return render(request, 'contact/contact.html', {})
    else:
        return render(request, 'contact/contact.html', {})


import unittest

from models.league_models.LeagueModel import LeagueModel
from models.league_models.MatchupModel import MatchupModel
from models.league_models.TeamModel import TeamModel
from models.league_models.WeekModel import WeekModel
from models.league_models.YearModel import YearModel
from packages.StatCalculators.SmartCalculator import SmartCalculator

class TestSmartCalculator(unittest.TestCase):

    def test_getSmartWinsOfScore(self):
        team1 = TeamModel(1, "team1")
        team2 = TeamModel(2, "team2")
        team3 = TeamModel(3, "team3")
        team4 = TeamModel(4, "team4")
        team5 = TeamModel(5, "team5")
        team6 = TeamModel(6, "team6")
        teamList = [team1, team2, team3, team4, team5, team6]
        matchup1 = MatchupModel(1, team1, team2, 100, 100.5)
        matchup2 = MatchupModel(2, team3, team4, 0.0, 100)
        matchup3 = MatchupModel(3, team5, team6, 104, 105)
        matchupList = [matchup1, matchup2, matchup3]
        week1 = WeekModel(1, matchupList)
        matchup1 = MatchupModel(1, team1, team2, 100, 100.5)
        matchup2 = MatchupModel(2, team3, team4, 0.0, 100)
        matchup3 = MatchupModel(3, team5, team6, 104, 105)
        matchupList = [matchup1, matchup2, matchup3]
        week2 = WeekModel(2, matchupList)
        weekList = [week1, week2]
        year2020 = YearModel(2020, teamList, weekList)
        matchup1 = MatchupModel(1, team1, team2, 100, 100.5)
        matchup2 = MatchupModel(2, team3, team4, 0.0, 100)
        matchup3 = MatchupModel(3, team5, team6, 104, 105)
        matchupList = [matchup1, matchup2, matchup3]
        week1 = WeekModel(1, matchupList)
        matchup1 = MatchupModel(1, team1, team2, 100.6, 100.5)
        matchup2 = MatchupModel(2, team3, team4, 100.1, 100)
        matchup3 = MatchupModel(3, team5, team6, 104, 105)
        matchupList = [matchup1, matchup2, matchup3]
        week2 = WeekModel(2, matchupList)
        weekList = [week1, week2]
        year2021 = YearModel(2021, teamList, weekList)
        yearDict = {"2020": year2020, "2021": year2021}
        leagueModel = LeagueModel(123456, "test", 6, yearDict)
        smartCalculator2020 = SmartCalculator(leagueModel, [2020])
        smartCalculator2021 = SmartCalculator(leagueModel, [2021])
        smartCalculatorBothYears = SmartCalculator(leagueModel, [2020, 2021])
        smartWins1 = smartCalculator2020.getSmartWinsOfScore(100)
        smartWins2 = smartCalculator2020.getSmartWinsOfScore(100)
        smartWinsDefault = smartCalculator2020.getSmartWinsOfScore(100)
        smartWins_2021 = smartCalculator2021.getSmartWinsOfScore(100)
        smartWinsBothYears = smartCalculatorBothYears.getSmartWinsOfScore(100)
        self.assertIsInstance(smartWins1, float)
        self.assertEqual(0.32, smartWins1)
        self.assertEqual(0.32, smartWins2)
        self.assertEqual(0.32, smartWinsDefault)
        self.assertEqual(0.18, smartWins_2021)
        self.assertEqual(0.26, smartWinsBothYears)

    def test_getSmartWinsOfScoresList(self):
        team1 = TeamModel(1, "team1")
        team2 = TeamModel(2, "team2")
        team3 = TeamModel(3, "team3")
        team4 = TeamModel(4, "team4")
        team5 = TeamModel(5, "team5")
        team6 = TeamModel(6, "team6")
        teamList = [team1, team2, team3, team4, team5, team6]
        matchup1 = MatchupModel(1, team1, team2, 100, 100.5)
        matchup2 = MatchupModel(2, team3, team4, 0.0, 100)
        matchup3 = MatchupModel(3, team5, team6, 104, 105)
        matchupList = [matchup1, matchup2, matchup3]
        week1 = WeekModel(1, matchupList)
        matchup1 = MatchupModel(1, team1, team2, 100, 100.5)
        matchup2 = MatchupModel(2, team3, team4, 0.0, 100)
        matchup3 = MatchupModel(3, team5, team6, 104, 105)
        matchupList = [matchup1, matchup2, matchup3]
        week2 = WeekModel(2, matchupList)
        weekList = [week1, week2]
        year = YearModel(2020, teamList, weekList)
        yearDict = {"2020": year}
        leagueModel = LeagueModel(123456, "test", 6, yearDict)
        smartCalculator = SmartCalculator(leagueModel, [2020])
        smartWins2 = smartCalculator.getSmartWinsOfScoresList([100, 100])
        smartWinsDefault = smartCalculator.getSmartWinsOfScoresList([100, 100])
        self.assertIsInstance(smartWins2, float)
        self.assertEqual(0.64, smartWins2)
        self.assertEqual(0.64, smartWinsDefault)

    def test_getSmartWinsAdjustmentOfScores(self):
        team1 = TeamModel(1, "team1")
        team2 = TeamModel(2, "team2")
        team3 = TeamModel(3, "team3")
        team4 = TeamModel(4, "team4")
        team5 = TeamModel(5, "team5")
        team6 = TeamModel(6, "team6")
        teamList = [team1, team2, team3, team4, team5, team6]
        matchup1 = MatchupModel(1, team1, team2, 100, 100.5)
        matchup2 = MatchupModel(2, team3, team4, 0.0, 100)
        matchup3 = MatchupModel(3, team5, team6, 104, 105)
        matchupList = [matchup1, matchup2, matchup3]
        week1 = WeekModel(1, matchupList)
        matchup1 = MatchupModel(1, team1, team2, 100, 99)
        matchup2 = MatchupModel(2, team3, team4, 0.0, 100)
        matchup3 = MatchupModel(3, team5, team6, 104, 105)
        matchupList = [matchup1, matchup2, matchup3]
        week2 = WeekModel(2, matchupList)
        weekList = [week1, week2]
        year = YearModel(2020, teamList, weekList)
        yearDict = {"2020": year}
        leagueModel = LeagueModel(123456, "test", 6, yearDict)
        smartCalculator = SmartCalculator(leagueModel, [2020])
        swa2 = smartCalculator.getSmartWinsAdjustmentOfScores([100, 100], 1)
        swaDefault = smartCalculator.getSmartWinsAdjustmentOfScores([100, 100], 1)
        self.assertIsInstance(swa2, float)
        self.assertEqual(-0.18, swa2)
        self.assertEqual(-0.18, swaDefault)


'''
Description: Product Generator for the FFW and FFS products.

@author Tracy.L.Hansen@noaa.gov
@version 1.0
'''
import collections
import HydroGenerator

class Product(HydroGenerator.Product):

    def __init__(self) :
        ''' Hazard Types covered
             ('FF.W.Convective',     'Flood'),
             ('FF.W.NonConvective',  'Flood'),
             ('FF.W.BurnScar',       'Flood'),
        '''
        super(Product, self).__init__()

        # Used by the VTECEngineWrapper to access the productGeneratorTable
        self._productGeneratorName = 'FFW_FFS_ProductGenerator'

    def _initialize(self) :
        super(Product, self)._initialize()
        self._productID = 'FFW'
        self._productCategory = 'FFW_FFS'
        self._productName = 'Flash Flood Warning'
        self._purgeHours = -1.0
        self._FFW_ProductName = 'Flash Flood Warning'
        self._FFS_ProductName = 'Flash Flood Statement'
        # Polygon-based, so locations listed will be limited to within the polygon rather than county area
        self._polygonBased = True

    def defineScriptMetadata(self):
        metadata = collections.OrderedDict()
        metadata['author'] = 'Raytheon'
        metadata['description'] = 'Content generator for flash flood warning.'
        metadata['version'] = '1.0'
        return metadata

    def defineDialog(self, eventSet):
        return {}

    def executeFrom(self, dataList, eventSet, productParts=None):
        if isinstance(productParts, list) and len(productParts) > 0:
            dataList = self.correctProduct(dataList, eventSet, productParts, True)
        else:
            self.updateExpireTimes(dataList)
        return dataList

    def execute(self, eventSet, dialogInputMap):
        self._initialize()

        self.logger.info('Start ProductGeneratorTemplate:execute FFW_FFS')

        # Extract information for execution
        self._getVariables(eventSet, dialogInputMap)
        eventSetAttributes = eventSet.getAttributes()

        if not self._inputHazardEvents:
            return []

        productDicts, hazardEvents = self._makeProducts_FromHazardEvents(self._inputHazardEvents, eventSetAttributes)

        return productDicts, hazardEvents

    def _preProcessHazardEvents(self, hazardEvents):
        '''
        Add expirationTime to each event and set
        immediate cause for FF.W.NonConvective prior to VTEC processing
        '''
        for hazardEvent in hazardEvents:
            expirationTime = self._tpc.round(hazardEvent.getEndTime())
            hazardEvent.setExpirationTime(expirationTime)
            if hazardEvent.getHazardType() == 'FF.W.NonConvective':
                immediateCause = self.hydrologicCauseMapping(hazardEvent.get('hydrologicCause'))
                hazardEvent.set('immediateCause', immediateCause)

    def _groupSegments(self, segments):
        '''
         Group the segments into the products
            return a list of productSegmentGroup dictionaries

         Check the pil
          IF FFW -- make a new FFW -- there can only be one segment per FFW
          Group the segments into FFS products with same ETN
        '''
        # For short fused areal products,
        #   we can safely make the assumption of only one hazard/action per segment.
        productSegmentGroups = []
        for segment in segments:
            vtecRecords = self.getVtecRecords(segment)
            for vtecRecord in vtecRecords:  # NOTE there is only one vtecRecord / hazard to process
                pil = vtecRecord.get('pil')
                etn = vtecRecord.get('etn')
                if pil == 'FFW':
                    # Create new FFW
                    productSegmentGroup = self.createProductSegmentGroup(pil, self._FFW_ProductName, 'area', self._vtecEngine, 'counties', False,
                                            [self.createProductSegment(segment, vtecRecords)], etn=etn, formatPolygon=True)
                    productSegmentGroups.append(productSegmentGroup)
                else:  # FFS
                    # See if this record matches the ETN of an existing FFS
                    found = False
                    for productSegmentGroup in productSegmentGroups:
                        if productSegmentGroup.productID == 'FFS' and productSegmentGroup.etn == etn:
                            productSegmentGroup.addProductSegment(self.createProductSegment(segment, vtecRecords))
                            found = True
                    if not found:
                        # Make a new FFS productSegmentGroup
                       productSegmentGroup = self.createProductSegmentGroup(pil, self._FFS_ProductName, 'area', self._vtecEngine, 'counties', True,
                                                [self.createProductSegment(segment, vtecRecords)], etn=etn, formatPolygon=True)
                       productSegmentGroups.append(productSegmentGroup)
        for productSegmentGroup in productSegmentGroups:
            self._addProductParts(productSegmentGroup)
        return productSegmentGroups

    def _addProductParts(self, productSegmentGroup):
        productID = productSegmentGroup.productID
        productSegments = productSegmentGroup.productSegments
        if productID == 'FFW':
            productPartsDict = self._hydroProductParts.productParts_FFW(productSegments)
        elif productID == 'FFS':
            productPartsDict = self._hydroProductParts.productParts_FFS(productSegments)
        productParts = self.createProductParts(productPartsDict)
        productSegmentGroup.setProductParts(productParts)


from Tracker import Tracker
import torch
import torch.nn as nn
import torch.optim as optim
from Tracker import Tracker
from flask import Flask, render_template
from flask import jsonify
import sys
app = Flask(__name__, static_url_path='/static')
class incidenceGuesser(nn.Module):

    def __init__(self):
        super(incidenceGuesser, self).__init__()
        self.rnn = nn.LSTM(60, 7, 1)
        self.ReLu = nn.ReLU()
        self.hn = torch.rand(1,1,7)
        self.cn = torch.rand(1,1,7)

    def forward(self, x):
        x = x.view(-1,1,60)
        output, (hn, cn) = self.rnn(x, (self.hn, self.cn))
        hn = hn.detach()
        cn = cn.detach()
        self.hn = hn
        self.cn = cn
        return self.ReLu(output.view(-1,7))

PATH = './model/model-safed/mk2/cnn.pth'

model = incidenceGuesser()
model.load_state_dict(torch.load(PATH))
model.eval()

def estimate(id, latest = True):
    tracker = Tracker()
    ags = id.split("-")[0]
    tracker.track(id = ags)
    f = open("MinMax.txt", "r")
    MinMax = f.read().split(",")
    f = open(f"data/{id}.csv", "r")
    csv = f.read().split("\n")
    dates = []
    values = []
    MinMax[0] = float(MinMax[0])
    MinMax[1] = float(MinMax[1])
    for x, i in enumerate(csv):
        if i != '':
            dates.append(i.split(",")[1])
            values.append(i.split(",")[0])
    for x, i in enumerate(values):
        if i == '':
            values.pop(x)
        else:
            values[x] = (float(i) - MinMax[0]) / (MinMax[1] - MinMax[0])
    values.reverse()
    latest = values[0:60]
    dates.reverse()
    for x, i in enumerate(dates):
        dates[x] = i.split("T")[0]
    dates = dates[0:60]
    latest.reverse()
    dates.reverse()
    est = model(torch.tensor(latest))
    est = est.tolist()
    print(est, file=sys.stderr)
    latest.extend(est[0]) #remove [0] for non LSTM models readd after deletion
    for x, i in enumerate(latest):
        latest[x] = i * (MinMax[1] - MinMax[0]) + MinMax[0]
    return latest, dates

@app.route('/estimate/latest/<id>', methods = ['GET'])
def EstimateLatest(id = None):
    res = estimate(id)
    return jsonify(res)

@app.route('/track/all', methods = ['POST'])
def trackAlll():
    tracker = Tracker()
    tracker.track()
    return

@app.route("/view/main", methods = ['GET'])
def ViewMain():
    return render_template('main.html')

@app.route('/static/<path:path>')
def send_js(path):
    return app.send_static_file(path)


import sys
from utils.print_color import print_color

from character import Character
def starting_location():
    pc = Character()
    print_color('\nWelcome to the land of mystery what is your name: ', 'green')
    name = raw_input()
    pc.start_pc(name)
    print_color('\nWelcome ' + pc.name, 'green')
    command = raw_input('\nWhat do you want to do? ')
    while 'quit' not in command:
        pc.process_command(command)
        print_color('\n' + pc.name, 'cyan')
        print_color('=: ', 'green')
        command = raw_input()
    raw_input('Thanks for playing, press enter to exit.')

starting_location()


import requests
import heapq
from astar import astar

def colinear(p1,p2,p3):
  if p1[0]==p2[0] or p1[0]==p3[0]:
    return p2[0]==p3[0]
  return (p2[1]-p1[1])/(p2[0]-p1[0])==(p3[1]-p1[1])/(p3[0]-p1[0])

def path_plan(start,finish,weight=0.001,limit=8):
  x1 = start[0]
  y1 = start[1]
  x2 = finish[0]
  y2 = finish[1]
  max_y = 37.821
  min_y = 37.707
  max_x = -122.365
  min_x = -122.513
  rows = 57
  cols = 75

  pts = astar(start,finish,(min_x,max_x),(min_y,max_y),(cols,rows),weight)

  #path simplification algorithm
  short = [(x1,y1)] #assuming first node not returned by A*
  for i in range(1,len(pts)):
    if not colinear(short[len(short)-1],pts[i],pts[i-1]):
      short.append(pts[i])
  short.append((x2,y2)) #assuming final node not returned by A*
  if len(short)<limit: #limit on waypoints
    return short
  q = (len(short)-1)/limit
  r = (len(short)-1)%limit
  out = []
  count = 0
  for i in range(limit+1):
    out.append(short[count])
    count+=q
    if i<r:
      count+=1
  return out


# -*- encoding: utf-8 -*-
from __future__ import print_function

import datetime
import logging
import os

try:
    import settings
    (settings.DEBUG, settings.CURRENT_PATH)
except:
    print('''add settings.py with:

DEBUG = True / False
CURRENT_PATH = os.getcwd()
''')
    import sys
    sys.exit()

__all__ = ('get_logger', )

def get_logger(name, level=logging.ERROR):
    utc_now = datetime.datetime.utcnow()
    path = os.path.join(
        settings.CURRENT_PATH,
        utc_now.strftime('log/%Y/%m/%d/'),
    )
    os.path.exists(path) or os.makedirs(path)
    level = logging.DEBUG if settings.DEBUG else level
    formatter = logging.Formatter(
        fmt='%(asctime)s %(levelname)-8s %(message)s',
        datefmt='%Y-%m-%d %H:%M:%S',
    )

    logger = logging.getLogger('{}-logger'.format(name))
    logger.setLevel(level)

    file_handler = logging.FileHandler(
        os.path.join(path, '{}.log'.format(name)),
    )
    file_handler.setLevel(level)
    file_handler.setFormatter(formatter)
    logger.addHandler(file_handler)

    if settings.DEBUG:
        console_handler = logging.StreamHandler()
        console_handler.setLevel(level)
        console_handler.setFormatter(formatter)
        logger.addHandler(console_handler)

    return logger


import pathlib

import nbformat
import pytest
from nbconvert.preprocessors import CellExecutionError
from nbconvert.preprocessors import ExecutePreprocessor

root = pathlib.Path(__file__).parent / "../examples"
fpaths = sorted(root.glob("*.ipynb"))

def idfn(val):
    if isinstance(val, pathlib.Path):
        return val.name

@pytest.mark.parametrize("fpath", fpaths, ids=idfn)
def test_notebook(fpath):
    """Execute each notebook."""
    # Information on running notebooks is found here
    # https://nbconvert.readthedocs.io/en/latest/execute_api.html
    with fpath.open() as fp:
        nb = nbformat.read(fp, as_version=4)

    ep = ExecutePreprocessor(timeout=600, kernel_name="python3")

    try:
        ep.preprocess(nb, {"metadata": {"path": fpath.parent}})
    except CellExecutionError:
        msg = 'Error executing the notebook "%s".\n\n' % fpath
        msg += 'See notebook "%s" for the traceback.' % fpath
        print(msg)
        raise
    finally:
        with fpath.open(mode="w", encoding="utf-8") as fp:
            nbformat.write(nb, fp)


#-*-encoding=utf-8-*-
from lxml import etree
import async_http
from spider import log_with_time
from spider import jsonp_json
from spider import format_price
import pdb
import re
import json
import time

burl = "http://www.zm7.cn/"
def cats_parser(url, res, rule):
    content = res['text']
    t = etree.HTML(content)
    ret = [burl + i for i in t.xpath(rule)]

    return ret

def pager(task, rule):
    pad = lambda n: "min0-max0-attr0-%d-sort_order-ASC.html" % n
    burl = task['url'][:-5] + '-'
    t = etree.HTML(task['text'])
    pagetxt = t.xpath(rule)

    ret = []

    if not pagetxt:
        log_with_time("bad response: %r" % task['url'])
        return ret

    pagetxt = pagetxt[0].text
    pagenum = re.findall("\d+", pagetxt)
    pagenum = int(pagenum[-1])
    for i in range(1, pagenum+1):
        ret.append(burl+pad(i))
    return ret

re_price = re.compile("\d+\.\d+")
def list_parser(task, rule):
    t = etree.HTML(task['text'])
    nodes = t.xpath(rule['nodes'])
    ret = []
    for node in nodes:
        gid = node.xpath(rule['gid'])
        price = node.xpath(rule['price'])
        if not gid or not price:
            log_with_time("bad response: %r" % task['url'])
            continue
        gid = burl + gid[0]
        price = price[0].text
        price = re_price.search(price).group()
        ret.append((gid, price))
    return ret

def stock_parser(task, rule):
    ret = []
    ostock = re.search("暂时无货", task['text'])

    gid = task["url"]
    price = task["price"]

    if ostock:
        stock = 0
    else:
        stock = 1

    ret = [(gid, price, stock)]
    fret = format_price(ret)
    dps = {}
    for i in fret:
        dps[i[1]] = int(time.time())
    return {"result":fret, "dps":dps}


from random import randint, shuffle
import fire
import json
from typing import NamedTuple
from tqdm import tqdm

import torch
import torch.nn as nn
from tensorboardX import SummaryWriter
from transformers import ElectraForPreTraining, ElectraForMaskedLM
from QDElectra_model import DistillElectraForPreTraining, QuantizedElectraForPreTraining
import torch.nn.functional as F
from transformers import ElectraConfig

import tokenization
import optim
import train

from utils import set_seeds, get_device, truncate_tokens_pair, check_dirs_exist

# Input file format :
# 1. One sentence per line. These should ideally be actual sentences,
#    not entire paragraphs or arbitrary spans of text. (Because we use
#    the sentence boundaries for the "next sentence prediction" task).
# 2. Blank lines between documents. Document boundaries are needed
#    so that the "next sentence prediction" task doesn't span between documents.

class SentPairDataLoader():
    """ Load sentence pair (sequential or random order) from corpus """
    def __init__(self, file, batch_size, tokenize, max_len, short_sampling_prob=0.1, pipeline=[]):
        super().__init__()
        self.file = open(file, "r", encoding='utf-8', errors='ignore')
        self.tokenize = tokenize # tokenize function
        self.max_len = max_len # maximum length of tokens
        self.short_sampling_prob = short_sampling_prob
        self.pipeline = pipeline
        self.batch_size = batch_size

    def read_tokens(self, f, length, discard_last_and_restart=True):
        """ Read tokens from file pointer with limited length """
        tokens = []
        while len(tokens) < length:
            line = f.readline()
            if not line: # end of file
                return None
            if not line.strip(): # blank line (delimiter of documents)
                if discard_last_and_restart:
                    tokens = [] # throw all and restart
                    continue
                else:
                    return tokens # return last tokens in the document
            tokens.extend(self.tokenize(line.strip()))
        return tokens

    def __iter__(self): # iterator to load data
        while True:
            batch = []
            instance = None
            for i in range(self.batch_size):
                # sampling length of each tokens_a and tokens_b
                # sometimes sample a short sentence to match between train and test sequences
                # len_tokens = randint(1, int(self.max_len / 2)) \
                #     if rand() < self.short_sampling_prob \
                #     else int(self.max_len / 2)

                tokens_a = self.read_tokens(self.file, self.max_len, True)

                if tokens_a is None: # end of file
                    self.file.seek(0, 0) # reset file pointer
                    return

                for proc in self.pipeline:
                    instance = proc(tokens_a)

                batch.append(instance)

            # To Tensor
            batch_tensors = [torch.tensor(x, dtype=torch.long) for x in zip(*batch)]
            yield batch_tensors

class Pipeline():
    """ Pre-process Pipeline Class : callable """
    def __init__(self):
        super().__init__()

    def __call__(self, instance):
        raise NotImplementedError

class Preprocess4Pretrain(Pipeline):
    """ Pre-processing steps for pretraining transformer """
    def __init__(self, max_pred, mask_prob, vocab_words, indexer, max_len=512):
        super().__init__()
        self.max_pred = max_pred # max tokens of prediction
        self.mask_prob = mask_prob # masking probability
        self.vocab_words = vocab_words # vocabulary (sub)words
        self.indexer = indexer # function from token to token index
        self.max_len = max_len

    def __call__(self, tokens_a):
        # -2  for special tokens [CLS], [SEP]
        truncate_tokens_pair(tokens_a, [], self.max_len - 2)

        # Add Special Tokens
        tokens = ['[CLS]'] + tokens_a + ['[SEP]']
        token_type_ids = [0] * self.max_len
        attention_mask = [1] * len(tokens)
        original_attention_mask = attention_mask.copy()

        # Get ElectraGenerator label. "-100" means the corresponding token is unmasked, else means the masked token ids
        g_label = [-100] * self.max_len

        # Get original input ids as ElectraDiscriminator labels
        original_input_ids = self.indexer(tokens)

        # For masked Language Models
        # The number of prediction is sometimes less than max_pred when sequence is short
        n_pred = min(self.max_pred, max(1, int(round(len(tokens) * self.mask_prob))))
        # candidate positions of masked tokens
        cand_pos = [i for i, token in enumerate(tokens)
                    if token != '[CLS]' and token != '[SEP]']
        shuffle(cand_pos)
        for pos in cand_pos[:n_pred]:
            attention_mask[pos] = 0
            g_label[pos] = self.indexer(tokens[pos])[0]  # get the only one element from list
            tokens[pos] = '[MASK]'

        # Token Indexing
        input_ids = self.indexer(tokens)

        # Zero Padding
        n_pad = self.max_len - len(input_ids)
        input_ids.extend([0] * n_pad)
        attention_mask.extend([0] * n_pad)

        return input_ids, attention_mask, token_type_ids, g_label, original_input_ids, original_attention_mask

class QuantizedDistillElectraTrainer(train.Trainer):
    def __init__(self, writer, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.writer = writer
        self.bceLoss = nn.BCELoss()
        self.mseLoss = nn.MSELoss()

    def get_loss(self, model, batch, global_step):
        g_outputs, t_d_outputs, s_d_outputs, s2t_hidden_states = model(*batch)

        # Get original electra loss
        g_outputs.loss = g_outputs.loss.mean()
        t_d_outputs.loss = t_d_outputs.loss.mean()
        s_d_outputs.loss = s_d_outputs.loss.mean()
        t_d_outputs.loss *= self.train_cfg.lambda_
        s_d_outputs.loss *= self.train_cfg.lambda_
        electra_loss = g_outputs.loss + s_d_outputs.loss + t_d_outputs.loss

        # -----------------------
        # Get distillation loss
        # 1. teacher and student logits (cross-entropy + temperature)
        # 2. embedding layer loss + hidden losses (MSE)
        # 3. attention matrices loss (MSE)
        #       We only consider :
        #       3-1. Teacher layer numbers equal to student layer numbers
        #       3-2. Teacher head numbers are divisible by Student head numbers
        # -----------------------
        soft_logits_loss = self.bceLoss(
            F.sigmoid(s_d_outputs.logits / self.train_cfg.temperature),
            F.sigmoid(t_d_outputs.logits.detach() / self.train_cfg.temperature),
        ) * self.train_cfg.temperature * self.train_cfg.temperature

        hidden_layers_loss = 0
        for t_hidden, s_hidden in zip(t_d_outputs.hidden_states, s2t_hidden_states):
            hidden_layers_loss += self.mseLoss(s_hidden, t_hidden.detach())

        # -----------------------
        # teacher attention shape per layer : (batch_size, t_n_heads, max_seq_len, max_seq_len)
        # student attention shape per layer : (batch_size, s_n_heads, max_seq_len, max_seq_len)
        # -----------------------
        atten_layers_loss = 0
        split_sections = [self.model_cfg.s_n_heads] * (self.model_cfg.t_n_heads // self.model_cfg.s_n_heads)
        for t_atten, s_atten in zip(t_d_outputs.attentions, s_d_outputs.attentions):
            split_t_attens = torch.split(t_atten.detach(), split_sections, dim=1)
            for i, split_t_atten in enumerate(split_t_attens):
                atten_layers_loss += self.mseLoss(s_atten[:, i, :, :], torch.mean(split_t_atten, dim=1))

        total_loss = electra_loss + soft_logits_loss + hidden_layers_loss + atten_layers_loss

        self.writer.add_scalars(
            'data/scalar_group', {
                'generator_loss': g_outputs.loss.item(),
                't_discriminator_loss': t_d_outputs.loss.item(),
                's_discriminator_loss': s_d_outputs.loss.item(),
                'soft_logits_loss': soft_logits_loss.item(),
                'hidden_layers_loss': hidden_layers_loss.item(),
                'attention_loss': atten_layers_loss.item(),
                'total_loss': total_loss.item(),
                'lr': self.optimizer.get_lr()[0]
            }, global_step
        )

        print(f'\tGenerator Loss {g_outputs.loss.item():.3f}\t'
              f'T-Discriminator Loss {t_d_outputs.loss.item():.3f}\t'
              f'S-Discriminator Loss {s_d_outputs.loss.item():.3f}\t'
              f'Soft Logits Loss {soft_logits_loss.item():.3f}\t'
              f'Hidden Loss {hidden_layers_loss.item():.3f}\t'
              f'Attention Loss {atten_layers_loss.item():.3f}\t'
              f'Total Loss {total_loss.item():.3f}\t')

        return total_loss

    def _eval(self, data_parallel=True):
        """ Only used to test the correctness of inference-phase of QuantizedElectra """
        self.model.eval()  # evaluation mode
        model = self.model.to(self.device)
        if data_parallel:  # use Data Parallelism with Multi-GPU
            model = nn.DataParallel(model)

        results = []  # prediction results
        iter_bar = tqdm(self.data_iter, desc='Iter (loss=X.XXX)')
        global_step = 0
        for batch in iter_bar:
            global_step += 1
            batch = [t.to(self.device) for t in batch]
            with torch.no_grad():  # evaluation without gradient calculation
                loss = self.get_loss(model, batch, global_step).mean()  # mean() for Data
            results.append(loss)

            iter_bar.set_description('Iter(loss=%5.3f)' % loss)
        return results

def main(train_cfg='config/electra_pretrain.json',
         model_cfg='config/electra_small.json',
         data_file='../tbc/books_large_all.txt',
         model_file=None,
         data_parallel=True,
         vocab='../uncased_L-12_H-768_A-12/vocab.txt',
         log_dir='../exp/electra/pretrain/runs',
         save_dir='../exp/electra/pretrain',
         max_len=128,
         max_pred=20,
         mask_prob=0.15,
         quantize=False):

    check_dirs_exist([log_dir, save_dir])

    train_cfg = ElectraConfig().from_json_file(train_cfg)
    model_cfg = ElectraConfig().from_json_file(model_cfg)

    set_seeds(train_cfg.seed)

    tokenizer = tokenization.FullTokenizer(vocab_file=vocab, do_lower_case=True)
    tokenize = lambda x: tokenizer.tokenize(tokenizer.convert_to_unicode(x))

    pipeline = [
        Preprocess4Pretrain(
            max_pred, mask_prob, list(tokenizer.vocab.keys()), tokenizer.convert_tokens_to_ids, max_len
        )
    ]

    data_iter = SentPairDataLoader(data_file, train_cfg.batch_size, tokenize, max_len, pipeline=pipeline)

    # Get distilled-electra and quantized-distilled-electra
    generator = ElectraForMaskedLM.from_pretrained('google/electra-small-generator')
    t_discriminator = ElectraForPreTraining.from_pretrained('google/electra-base-discriminator')
    s_discriminator = QuantizedElectraForPreTraining(model_cfg) if quantize else ElectraForPreTraining
    s_discriminator = s_discriminator.from_pretrained('google/electra-small-discriminator', config=model_cfg)  # model
    # config is used for model "QuantizedElectraForPreTraining"
    model = DistillElectraForPreTraining(generator, t_discriminator, s_discriminator, model_cfg)

    optimizer = optim.optim4GPU(train_cfg, model)
    writer = SummaryWriter(log_dir=log_dir) # for tensorboardX

    base_trainer_args = (train_cfg, model_cfg, model, data_iter, None, optimizer, save_dir, get_device())
    trainer = QuantizedDistillElectraTrainer(writer, *base_trainer_args)
    trainer.train(model_file, None, data_parallel)
    trainer._eval()

if __name__ == '__main__':
    fire.Fire(main)


import torch
import os

import numpy as np
import pandas as pd
from sklearn import metrics

def cal_acc_auc(loader, model, device, dtype):
    with torch.no_grad():
        acc_lst = []
        auc_lst = []
        for X, y in loader:
            X = X.to(device=device, dtype=dtype)
            y = y.to(device='cpu', dtype=torch.long)
            scores = model(X)
            _, predict_labels = scores.max(dim=1)
            predict_labels = predict_labels.to(device='cpu', dtype=torch.long)
            acc_lst.append(metrics.accuracy_score(y, predict_labels))
            auc_lst.append(metrics.roc_auc_score(y, predict_labels))
        acc = np.mean(acc_lst)
        auc = np.mean(auc_lst)
    return acc, auc

def clip_gradient(optimizer, grad_clip):
    """
    Clips gradients computed during backpropagation to avoid explosion of gradients.

    :param optimizer: optimizer with the gradients to be clipped
    :param grad_clip: clip value
    :return:
    """
    for group in optimizer.param_groups:
        for param in group['params']:
            if param.grad is not None:
                param.grad.data.clamp_(-grad_clip, grad_clip)

def save_best(model, model_type, file_name, PATH='../model/'):
    if not os.path.exists(PATH):
        os.mkdir(PATH)
    if not os.path.exists(PATH + file_name + '/'):
        os.mkdir(PATH + file_name + '/')
    # for future training
    # state = {'eooch': epoch,
    #          'epochs_since_improvement': epochs_since_improvement,
    #          'model': model.state_dict(),
    #          'optimizer': optimizer.state_dict(),
    #          'val_acc': val_acc,
    #          'val_auc': val_auc}
    # filename = model_type + '_checkpoint_epoch_' + str(epoch) + '.pt'
    # torch.save(state, PATH + filename)
    model_name = PATH + file_name + '/' + 'Best_' + model_type + '.pt'
    torch.save(model, model_name)

def load_best_model(model_type, file_name, PATH='../model/'):
    print("\nInitializing the best {} model..................".format(model_type))
    return torch.load(PATH + file_name + '/' + 'Best_' + model_type + '.pt')
    # torch.load('../model/CNN_checkpoint_epoch_0.pth.tar')

def to_csv(file_name, model_type, test_acc, test_auc,
           result_csv_name='scores_all_models.csv', PATH_CSV='../results/'):
    if not os.path.exists(PATH_CSV):
        os.mkdir(PATH_CSV)
    print('Save results into ', PATH_CSV)
    df = pd.DataFrame({'file_name': [file_name], 'model_type': [model_type],
                       'test_acc': [test_acc], 'test_auc': [test_auc]})
    path_file = PATH_CSV + result_csv_name
    if os.path.isfile(path_file):
        old_df = pd.read_csv(path_file)
        new_df = old_df[old_df['file_name'] != file_name]
        new_df = new_df.append(df, ignore_index=True)
        new_df.to_csv(path_file, header=['file_name', 'model_type', 'test_acc', 'test_auc'], index=False)
    else:
        df.to_csv(path_file, index=False)


"""
"Dumb" utility executables

"""

import sys
import subprocess

def seconds_from_string(duration):
    """
    Return a float (seconds) from something like 00:15:32.33
    """
    hours = float(duration.split(':')[0])
    minutes = float(duration.split(':')[1])
    seconds = float(duration.split(':')[2])
    duration_seconds = (((hours * 60) + minutes) * 60) + seconds
    return duration_seconds

def status_bar(process):
    """
    This is a little gross, but it'll get us a status bar
    """
    fps = None
    duration = None
    while True:
        line = process.stdout.readline().strip()

        if line == '' and process.poll() is not None:
            break
        if fps is None or duration is None:
            if "Stream #" in line and " Video: " in line:
                fps = [s for s in line.split(',') if "fps" in s][0].strip(' fps')

            if "Duration: " in line:
                dur = line.split('Duration: ')[1].split(',')[0].strip()
                duration = seconds_from_string(duration=dur)

        else:
            if 'frame=' in line:
                cur_frame = line.split('frame=')[1].split('fps=')[0].strip()
                end_frame = float(duration) * float(fps.strip())
                pctg = (float(cur_frame) / float(end_frame))

                sys.stdout.write('\r')
                i = int(pctg * 20.0)
                sys.stdout.write("%s : [%-20s] %d%%" % (
                    'Transcode', '=' * i, int(pctg * 100)))  # pylint: disable=unicode-format-string
                sys.stdout.flush()

    # For display politeness
    sys.stdout.write('\r')
    sys.stdout.write("%s : [%-20s] %d%%" % ('Transcode', '=' * 20, 100))  # pylint: disable=unicode-format-string
    sys.stdout.flush()
    print('')

def probe_video(VideoFileObject):
    """
    Use ffprobe to determine video metadata
    """
    ffprobe_comm = 'ffprobe -hide_banner ' + VideoFileObject.filepath

    p = subprocess.Popen(
        ffprobe_comm,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
        shell=True
    )

    for line in iter(p.stdout.readline, b''):
        line = line.decode('utf-8')
        if "Duration: " in line:
            # Duration
            vid_duration = line.split('Duration: ')[1].split(',')[0].strip()

            if vid_duration != 'N/A':
                VideoFileObject.duration = seconds_from_string(
                    duration=vid_duration
                )
                # Bitrate
                try:
                    VideoFileObject.bitrate = float(line.split('bitrate: ')[1].strip().split()[0])
                except ValueError:
                    pass

        elif "Stream #" in line and 'Video: ' in line:
            codec_array = line.strip().split(',')
            for c in codec_array:
                # Resolution
                if len(c.split('x')) == 2:
                    if '[' not in c:
                        VideoFileObject.resolution = c.strip()
                    else:
                        VideoFileObject.resolution = c.strip().split(' ')[0]
    return VideoFileObject


import requests
import pandas as pd
import multiprocessing as mp
import settings

iex_p_api_token = settings.get_iex_p_api_token()
iex_s_api_token = settings.get_iex_s_api_token()

base_url = 'https://sandbox.iexapis.com/stable/'

quote_keep_keys = ('open',
                   'high',
                   'low',
                   'close',
                   'previousClose',
                   'volume',
                   'previousVolume',
                   'avgTotalVolume',
                   'changePercent')

def filter_for_keys(d, keep_keys):
    """filters out unnecessary keys from dictionary

    Args:
        d (dictionary): original dictionary
        keep_keys (iterable): any iterable that contains the keys we want to keep

    Returns:
        dictionary: contains only needed keeps
    """
    return {key: d[key] for key in keep_keys}

def get_eod_equities_data(ticker):
    """get quote for specific ticker

    Args:
        ticker (string): stock ticker

    Returns:
        dictionary that contains necessary data
    """
    url = base_url + f'stock/{ticker}/quote?token={iex_p_api_token}'
    json_response = requests.get(url).json()

    return filter_for_keys(json_response, quote_keep_keys)

def get_length_list(l, length):
    """ separates list l into lists of length

    Args:
        l (list):
        length (int):

    Yields:
        list: of length
    """
    for i in range(0, len(l), length):
        yield l[i:i + length]

def get_eod_equities_data(tickers):

    d = {}
    for l_tickers in get_length_list(tickers, 100):
        s_symbols = ','.join(l_tickers)

        url = base_url + \
            f'stock/market/batch?symbols={s_symbols}&types=quote&token={iex_p_api_token}'

        json_response = requests.get(url).json()

        d_partial = {key: filter_for_keys(
            value['quote'], quote_keep_keys) for key, value in json_response.items()}

        d.update(d_partial)
    return d